From ee2a41c9c4445faca0b5722d118c157c65c27136 Mon Sep 17 00:00:00 2001 From: nvauto <70000568+nvauto@users.noreply.github.com> Date: Mon, 25 Nov 2024 06:14:41 +0000 Subject: [PATCH 01/12] Init version 25.02.0-SNAPSHOT Signed-off-by: nvauto <70000568+nvauto@users.noreply.github.com> --- examples/SQL+DF-Examples/tpcds/README.md | 2 +- examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml | 2 +- .../RAPIDS-accelerated-UDFs/src/main/cpp/CMakeLists.txt | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/SQL+DF-Examples/tpcds/README.md b/examples/SQL+DF-Examples/tpcds/README.md index be5c2823..838277c3 100644 --- a/examples/SQL+DF-Examples/tpcds/README.md +++ b/examples/SQL+DF-Examples/tpcds/README.md @@ -18,7 +18,7 @@ This notebook can be opened and executed using standard It can also be opened and evaluated on hosted Notebook environments. Use the link below to launch on Google Colab and connect it to a [GPU instance](https://research.google.com/colaboratory/faq.html). - + Open In Colab diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml index 09829d80..6d493099 100644 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml @@ -25,7 +25,7 @@ user defined functions for use with the RAPIDS Accelerator for Apache Spark - 24.12.0-SNAPSHOT + 25.02.0-SNAPSHOT 1.8 diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/cpp/CMakeLists.txt b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/cpp/CMakeLists.txt index 1dbd631e..fda3ba8d 100755 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/cpp/CMakeLists.txt +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/src/main/cpp/CMakeLists.txt @@ -16,7 +16,7 @@ cmake_minimum_required(VERSION 3.28.6 FATAL_ERROR) -file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-24.12/RAPIDS.cmake +file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-25.02/RAPIDS.cmake ${CMAKE_BINARY_DIR}/RAPIDS.cmake) include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) @@ -32,7 +32,7 @@ if(DEFINED GPU_ARCHS) endif() rapids_cuda_init_architectures(UDFEXAMPLESJNI) -project(UDFEXAMPLESJNI VERSION 24.12.0 LANGUAGES C CXX CUDA) +project(UDFEXAMPLESJNI VERSION 25.02.0 LANGUAGES C CXX CUDA) option(PER_THREAD_DEFAULT_STREAM "Build with per-thread default stream" OFF) option(BUILD_UDF_BENCHMARKS "Build the benchmarks" OFF) @@ -84,10 +84,10 @@ set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -w --expt-extended-lambda --expt-relax set(CUDA_USE_STATIC_CUDA_RUNTIME ON) rapids_cpm_init() -rapids_cpm_find(cudf 24.12.00 +rapids_cpm_find(cudf 25.02.00 CPM_ARGS GIT_REPOSITORY https://github.com/rapidsai/cudf.git - GIT_TAG branch-24.12 + GIT_TAG branch-25.02 GIT_SHALLOW TRUE SOURCE_SUBDIR cpp OPTIONS "BUILD_TESTS OFF" From 6798507a2907c3aecfd9622ff3ec392af30e6d0e Mon Sep 17 00:00:00 2001 From: YanxuanLiu <104543031+YanxuanLiu@users.noreply.github.com> Date: Wed, 27 Nov 2024 08:43:35 +0800 Subject: [PATCH 02/12] add license header and check workflow (#473) Signed-off-by: YanxuanLiu --- .github/workflows/license-header-check.yml | 54 +++++++++++++++++++ dockerfile/gpu_executor_template.yaml | 14 +++++ .../xgboost-examples/csp/databricks/init.sh | 14 +++++ .../models_config/fashion_mnist/config.pbtxt | 14 +++++ .../models_config/housing_model/config.pbtxt | 14 +++++ .../Spark-DL/dl_inference/requirements.txt | 14 +++++ .../models_config/mnist_model/config.pbtxt | 14 +++++ .../models_config/resnet50/config.pbtxt | 14 +++++ .../Spark-DL/dl_inference/tf_requirements.txt | 14 +++++ .../dl_inference/torch_requirements.txt | 14 +++++ .../agaricus/python/com/__init__.py | 14 +++++ .../agaricus/python/com/nvidia/__init__.py | 14 +++++ .../python/com/nvidia/spark/__init__.py | 14 +++++ .../com/nvidia/spark/examples/__init__.py | 14 +++++ .../spark/examples/agaricus/__init__.py | 14 +++++ .../aggregator/assembly/assembly-no-scala.xml | 14 +++++ .../mortgage/python/com/__init__.py | 14 +++++ .../mortgage/python/com/nvidia/__init__.py | 14 +++++ .../python/com/nvidia/spark/__init__.py | 14 +++++ .../com/nvidia/spark/examples/__init__.py | 14 +++++ .../spark/examples/mortgage/__init__.py | 14 +++++ .../XGBoost-Examples/pack_pyspark_example.sh | 14 +++++ .../taxi/python/com/__init__.py | 14 +++++ .../taxi/python/com/nvidia/__init__.py | 14 +++++ .../taxi/python/com/nvidia/spark/__init__.py | 14 +++++ .../com/nvidia/spark/examples/__init__.py | 14 +++++ .../nvidia/spark/examples/taxi/__init__.py | 14 +++++ .../utility/python/com/__init__.py | 14 +++++ .../utility/python/com/nvidia/__init__.py | 14 +++++ .../python/com/nvidia/spark/__init__.py | 14 +++++ .../com/nvidia/spark/examples/__init__.py | 14 +++++ .../nvidia/spark/examples/utility/__init__.py | 14 +++++ scripts/building/python_build.sh | 14 +++++ scripts/encoding-sample/repartition.py | 14 +++++ scripts/encoding-sample/run.sh | 14 +++++ scripts/encoding-sample/truncate-model.py | 14 +++++ scripts/encoding/python/com/__init__.py | 14 +++++ .../encoding/python/com/nvidia/__init__.py | 14 +++++ .../python/com/nvidia/spark/__init__.py | 14 +++++ .../com/nvidia/spark/encoding/__init__.py | 14 +++++ .../nvidia/spark/encoding/criteo/__init__.py | 14 +++++ .../nvidia/spark/encoding/utility/__init__.py | 14 +++++ 42 files changed, 628 insertions(+) create mode 100644 .github/workflows/license-header-check.yml diff --git a/.github/workflows/license-header-check.yml b/.github/workflows/license-header-check.yml new file mode 100644 index 00000000..a2347a0c --- /dev/null +++ b/.github/workflows/license-header-check.yml @@ -0,0 +1,54 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A workflow to check copyright/license header +name: license header check + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + license-header-check: + runs-on: ubuntu-latest + if: "!contains(github.event.pull_request.title, '[bot]')" + steps: + - name: Get checkout depth + run: | + echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 10 ))" >> $GITHUB_ENV + + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: ${{ env.PR_FETCH_DEPTH }} + + - name: license-header-check + uses: NVIDIA/spark-rapids-common/license-header-check@main + with: + included_file_patterns: | + *.sh, + *.java, + *.py, + *.pbtxt, + *Dockerfile*, + *Jenkinsfile*, + *.yml, + *.yaml, + *.cpp, + *.hpp, + *.txt, + *.cu, + *.scala, + *.ini, + *.xml diff --git a/dockerfile/gpu_executor_template.yaml b/dockerfile/gpu_executor_template.yaml index 35d2f39e..6784e590 100644 --- a/dockerfile/gpu_executor_template.yaml +++ b/dockerfile/gpu_executor_template.yaml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + apiVersion: v1 kind: Pod spec: diff --git a/docs/get-started/xgboost-examples/csp/databricks/init.sh b/docs/get-started/xgboost-examples/csp/databricks/init.sh index fc415b2d..b9302f1c 100644 --- a/docs/get-started/xgboost-examples/csp/databricks/init.sh +++ b/docs/get-started/xgboost-examples/csp/databricks/init.sh @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + sudo rm -f /databricks/jars/spark--maven-trees--ml--10.x--xgboost-gpu--ml.dmlc--xgboost4j-gpu_2.12--ml.dmlc__xgboost4j-gpu_2.12__1.5.2.jar sudo rm -f /databricks/jars/spark--maven-trees--ml--10.x--xgboost-gpu--ml.dmlc--xgboost4j-spark-gpu_2.12--ml.dmlc__xgboost4j-spark-gpu_2.12__1.5.2.jar diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/fashion_mnist/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/fashion_mnist/config.pbtxt index 3554196f..d98b2a11 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/fashion_mnist/config.pbtxt +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/fashion_mnist/config.pbtxt @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + platform: "pytorch_libtorch" max_batch_size: 8192 input [ diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/housing_model/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/housing_model/config.pbtxt index 794ffd37..64deefca 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/housing_model/config.pbtxt +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/housing_model/config.pbtxt @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + platform: "pytorch_libtorch" max_batch_size: 8192 input [ diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/requirements.txt b/examples/ML+DL-Examples/Spark-DL/dl_inference/requirements.txt index abd6ad08..f30f172c 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/requirements.txt +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/requirements.txt @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + numpy pandas matplotlib diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/mnist_model/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/mnist_model/config.pbtxt index 76a1437e..cc9172f4 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/mnist_model/config.pbtxt +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/mnist_model/config.pbtxt @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + platform: "tensorflow_savedmodel" max_batch_size: 8192 diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/resnet50/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/resnet50/config.pbtxt index 76a1437e..cc9172f4 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/resnet50/config.pbtxt +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/resnet50/config.pbtxt @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + platform: "tensorflow_savedmodel" max_batch_size: 8192 diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tf_requirements.txt b/examples/ML+DL-Examples/Spark-DL/dl_inference/tf_requirements.txt index b78561bd..92bba846 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tf_requirements.txt +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tf_requirements.txt @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + -r requirements.txt tensorflow[and-cuda] tf-keras \ No newline at end of file diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/torch_requirements.txt b/examples/ML+DL-Examples/Spark-DL/dl_inference/torch_requirements.txt index 0f73b910..5f3b9017 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/torch_requirements.txt +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/torch_requirements.txt @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + -r requirements.txt torch torchvision diff --git a/examples/XGBoost-Examples/agaricus/python/com/__init__.py b/examples/XGBoost-Examples/agaricus/python/com/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/agaricus/python/com/__init__.py +++ b/examples/XGBoost-Examples/agaricus/python/com/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/agaricus/python/com/nvidia/__init__.py b/examples/XGBoost-Examples/agaricus/python/com/nvidia/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/agaricus/python/com/nvidia/__init__.py +++ b/examples/XGBoost-Examples/agaricus/python/com/nvidia/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/agaricus/python/com/nvidia/spark/__init__.py b/examples/XGBoost-Examples/agaricus/python/com/nvidia/spark/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/agaricus/python/com/nvidia/spark/__init__.py +++ b/examples/XGBoost-Examples/agaricus/python/com/nvidia/spark/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/agaricus/python/com/nvidia/spark/examples/__init__.py b/examples/XGBoost-Examples/agaricus/python/com/nvidia/spark/examples/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/agaricus/python/com/nvidia/spark/examples/__init__.py +++ b/examples/XGBoost-Examples/agaricus/python/com/nvidia/spark/examples/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/agaricus/python/com/nvidia/spark/examples/agaricus/__init__.py b/examples/XGBoost-Examples/agaricus/python/com/nvidia/spark/examples/agaricus/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/agaricus/python/com/nvidia/spark/examples/agaricus/__init__.py +++ b/examples/XGBoost-Examples/agaricus/python/com/nvidia/spark/examples/agaricus/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/aggregator/assembly/assembly-no-scala.xml b/examples/XGBoost-Examples/aggregator/assembly/assembly-no-scala.xml index fd18210e..fe4db9da 100644 --- a/examples/XGBoost-Examples/aggregator/assembly/assembly-no-scala.xml +++ b/examples/XGBoost-Examples/aggregator/assembly/assembly-no-scala.xml @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/mortgage/python/com/__init__.py b/examples/XGBoost-Examples/mortgage/python/com/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/mortgage/python/com/__init__.py +++ b/examples/XGBoost-Examples/mortgage/python/com/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/mortgage/python/com/nvidia/__init__.py b/examples/XGBoost-Examples/mortgage/python/com/nvidia/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/mortgage/python/com/nvidia/__init__.py +++ b/examples/XGBoost-Examples/mortgage/python/com/nvidia/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/mortgage/python/com/nvidia/spark/__init__.py b/examples/XGBoost-Examples/mortgage/python/com/nvidia/spark/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/mortgage/python/com/nvidia/spark/__init__.py +++ b/examples/XGBoost-Examples/mortgage/python/com/nvidia/spark/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/mortgage/python/com/nvidia/spark/examples/__init__.py b/examples/XGBoost-Examples/mortgage/python/com/nvidia/spark/examples/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/mortgage/python/com/nvidia/spark/examples/__init__.py +++ b/examples/XGBoost-Examples/mortgage/python/com/nvidia/spark/examples/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/mortgage/python/com/nvidia/spark/examples/mortgage/__init__.py b/examples/XGBoost-Examples/mortgage/python/com/nvidia/spark/examples/mortgage/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/mortgage/python/com/nvidia/spark/examples/mortgage/__init__.py +++ b/examples/XGBoost-Examples/mortgage/python/com/nvidia/spark/examples/mortgage/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/pack_pyspark_example.sh b/examples/XGBoost-Examples/pack_pyspark_example.sh index e446d27d..5f7564b2 100755 --- a/examples/XGBoost-Examples/pack_pyspark_example.sh +++ b/examples/XGBoost-Examples/pack_pyspark_example.sh @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Follow these steps to package the Python zip file rm -fr samples.zip cd agaricus/python ; zip -r ../../samples.zip com ; cd ../.. diff --git a/examples/XGBoost-Examples/taxi/python/com/__init__.py b/examples/XGBoost-Examples/taxi/python/com/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/taxi/python/com/__init__.py +++ b/examples/XGBoost-Examples/taxi/python/com/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/taxi/python/com/nvidia/__init__.py b/examples/XGBoost-Examples/taxi/python/com/nvidia/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/taxi/python/com/nvidia/__init__.py +++ b/examples/XGBoost-Examples/taxi/python/com/nvidia/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/taxi/python/com/nvidia/spark/__init__.py b/examples/XGBoost-Examples/taxi/python/com/nvidia/spark/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/taxi/python/com/nvidia/spark/__init__.py +++ b/examples/XGBoost-Examples/taxi/python/com/nvidia/spark/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/taxi/python/com/nvidia/spark/examples/__init__.py b/examples/XGBoost-Examples/taxi/python/com/nvidia/spark/examples/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/taxi/python/com/nvidia/spark/examples/__init__.py +++ b/examples/XGBoost-Examples/taxi/python/com/nvidia/spark/examples/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/taxi/python/com/nvidia/spark/examples/taxi/__init__.py b/examples/XGBoost-Examples/taxi/python/com/nvidia/spark/examples/taxi/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/taxi/python/com/nvidia/spark/examples/taxi/__init__.py +++ b/examples/XGBoost-Examples/taxi/python/com/nvidia/spark/examples/taxi/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/utility/python/com/__init__.py b/examples/XGBoost-Examples/utility/python/com/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/utility/python/com/__init__.py +++ b/examples/XGBoost-Examples/utility/python/com/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/utility/python/com/nvidia/__init__.py b/examples/XGBoost-Examples/utility/python/com/nvidia/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/utility/python/com/nvidia/__init__.py +++ b/examples/XGBoost-Examples/utility/python/com/nvidia/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/utility/python/com/nvidia/spark/__init__.py b/examples/XGBoost-Examples/utility/python/com/nvidia/spark/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/utility/python/com/nvidia/spark/__init__.py +++ b/examples/XGBoost-Examples/utility/python/com/nvidia/spark/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/utility/python/com/nvidia/spark/examples/__init__.py b/examples/XGBoost-Examples/utility/python/com/nvidia/spark/examples/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/utility/python/com/nvidia/spark/examples/__init__.py +++ b/examples/XGBoost-Examples/utility/python/com/nvidia/spark/examples/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/examples/XGBoost-Examples/utility/python/com/nvidia/spark/examples/utility/__init__.py b/examples/XGBoost-Examples/utility/python/com/nvidia/spark/examples/utility/__init__.py index e69de29b..0f6ef823 100644 --- a/examples/XGBoost-Examples/utility/python/com/nvidia/spark/examples/utility/__init__.py +++ b/examples/XGBoost-Examples/utility/python/com/nvidia/spark/examples/utility/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/scripts/building/python_build.sh b/scripts/building/python_build.sh index 033fa0bd..78e34908 100644 --- a/scripts/building/python_build.sh +++ b/scripts/building/python_build.sh @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Follow these steps to package the Python zip file cd ../../examples/XGBoost-Examples cd agaricus/python ; zip -r ../../samples.zip com ; cd ../.. diff --git a/scripts/encoding-sample/repartition.py b/scripts/encoding-sample/repartition.py index b9492ed6..af53380d 100644 --- a/scripts/encoding-sample/repartition.py +++ b/scripts/encoding-sample/repartition.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Note: Plase modify the data source options for your case. import sys diff --git a/scripts/encoding-sample/run.sh b/scripts/encoding-sample/run.sh index 6a3a97b6..18127692 100644 --- a/scripts/encoding-sample/run.sh +++ b/scripts/encoding-sample/run.sh @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # clear rm -f encoding.zip main.py rm -f raw-*.csv diff --git a/scripts/encoding-sample/truncate-model.py b/scripts/encoding-sample/truncate-model.py index e946e9df..0cde5026 100644 --- a/scripts/encoding-sample/truncate-model.py +++ b/scripts/encoding-sample/truncate-model.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import sys from pyspark.sql import SparkSession diff --git a/scripts/encoding/python/com/__init__.py b/scripts/encoding/python/com/__init__.py index e69de29b..0f6ef823 100644 --- a/scripts/encoding/python/com/__init__.py +++ b/scripts/encoding/python/com/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/scripts/encoding/python/com/nvidia/__init__.py b/scripts/encoding/python/com/nvidia/__init__.py index e69de29b..0f6ef823 100644 --- a/scripts/encoding/python/com/nvidia/__init__.py +++ b/scripts/encoding/python/com/nvidia/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/scripts/encoding/python/com/nvidia/spark/__init__.py b/scripts/encoding/python/com/nvidia/spark/__init__.py index e69de29b..0f6ef823 100644 --- a/scripts/encoding/python/com/nvidia/spark/__init__.py +++ b/scripts/encoding/python/com/nvidia/spark/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/scripts/encoding/python/com/nvidia/spark/encoding/__init__.py b/scripts/encoding/python/com/nvidia/spark/encoding/__init__.py index e69de29b..0f6ef823 100644 --- a/scripts/encoding/python/com/nvidia/spark/encoding/__init__.py +++ b/scripts/encoding/python/com/nvidia/spark/encoding/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/scripts/encoding/python/com/nvidia/spark/encoding/criteo/__init__.py b/scripts/encoding/python/com/nvidia/spark/encoding/criteo/__init__.py index e69de29b..0f6ef823 100644 --- a/scripts/encoding/python/com/nvidia/spark/encoding/criteo/__init__.py +++ b/scripts/encoding/python/com/nvidia/spark/encoding/criteo/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/scripts/encoding/python/com/nvidia/spark/encoding/utility/__init__.py b/scripts/encoding/python/com/nvidia/spark/encoding/utility/__init__.py index e69de29b..0f6ef823 100644 --- a/scripts/encoding/python/com/nvidia/spark/encoding/utility/__init__.py +++ b/scripts/encoding/python/com/nvidia/spark/encoding/utility/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + From e85423a74aad2c9f4952b957af6562d6ae4b5c4a Mon Sep 17 00:00:00 2001 From: YanxuanLiu <104543031+YanxuanLiu@users.noreply.github.com> Date: Wed, 27 Nov 2024 14:52:39 +0800 Subject: [PATCH 03/12] Fix bug: correct xml comment syntax of license header (#474) * add license header and check workflow Signed-off-by: YanxuanLiu * correct xml comment format Signed-off-by: YanxuanLiu --------- Signed-off-by: YanxuanLiu --- .../aggregator/assembly/assembly-no-scala.xml | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/examples/XGBoost-Examples/aggregator/assembly/assembly-no-scala.xml b/examples/XGBoost-Examples/aggregator/assembly/assembly-no-scala.xml index fe4db9da..6fe53faa 100644 --- a/examples/XGBoost-Examples/aggregator/assembly/assembly-no-scala.xml +++ b/examples/XGBoost-Examples/aggregator/assembly/assembly-no-scala.xml @@ -1,16 +1,18 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. + Date: Tue, 21 Jan 2025 13:46:33 +0800 Subject: [PATCH 04/12] Use common add to project [skip ci] (#485) follow up of https://github.com/NVIDIA/spark-rapids-common/issues/22 to avoid update action details for multiple `spark-rapids*` repos in the future Signed-off-by: Peixin Li --- .github/workflows/add-to-project.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml index 80f5d8a0..bd7bfc72 100644 --- a/.github/workflows/add-to-project.yml +++ b/.github/workflows/add-to-project.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,13 +23,11 @@ on: - opened jobs: - add-to-project: - if: github.repository == 'NVIDIA/spark-rapids-examples' - name: Add new issues and pull requests to project + Add-to-project: + if: github.repository_owner == 'NVIDIA' # avoid adding issues from forks runs-on: ubuntu-latest steps: - - uses: actions/add-to-project@v0.6.1 + - name: add-to-project + uses: NVIDIA/spark-rapids-common/add-to-project@main with: - project-url: https://github.com/orgs/NVIDIA/projects/4 - github-token: ${{ secrets.PROJECT_TOKEN }} - + token: ${{ secrets.PROJECT_TOKEN }} From ae292de6e771e56aae718f6c68ffee7be88fcff8 Mon Sep 17 00:00:00 2001 From: "Rishi C." <77904151+rishic3@users.noreply.github.com> Date: Tue, 4 Feb 2025 09:20:15 -0800 Subject: [PATCH 05/12] Support for Spark DL notebooks with PyTriton on Databricks/Dataproc (#483) ### Support for running DL Inference notebooks on CSP environments. - Refactored Triton sections to use PyTriton, a Python API for the Triton inference server which avoids Docker. Once this PR is merged, Triton sections no longer need to be skipped in the CI pipeline @YanxuanLiu . - Updated notebooks with instructions to run on Databricks/Dataproc - Updated Torch notebooks with best practices for ahead-of-time TensorRT compilation. - Cleaned up README, removing instructions to start Jupyter with PySpark (we need a cell to attach to standalone for CI/CD anyway, so hoping to reduce confusion for user). Notebook outputs are saved from running locally, but all notebooks were tested on Databricks/Dataproc. --------- Signed-off-by: Rishi Chandra --- .../Spark-DL/dl_inference/README.md | 107 +- .../dl_inference/databricks/README.md | 55 + .../databricks/setup/init_spark_dl.sh | 36 + .../databricks/setup/start_cluster.sh | 49 + .../Spark-DL/dl_inference/dataproc/README.md | 70 + .../dataproc/setup/init_spark_dl.sh | 60 + .../dataproc/setup/start_cluster.sh | 105 + .../conditional_generation_tf.ipynb | 1597 ++++------ .../conditional_generation_torch.ipynb | 2015 ++++++------ .../models_config/hf_generation_tf/1/model.py | 150 - .../hf_generation_tf/config.pbtxt | 52 - .../hf_generation_torch/1/model.py | 144 - .../hf_generation_torch/config.pbtxt | 52 - .../models_config/hf_pipeline_tf/1/model.py | 147 - .../models_config/hf_pipeline_tf/config.pbtxt | 57 - .../hf_pipeline_torch/1/model.py | 142 - .../hf_pipeline_torch/config.pbtxt | 57 - .../hf_transformer_torch/1/model.py | 137 - .../hf_transformer_torch/config.pbtxt | 52 - .../huggingface/pipelines_tf.ipynb | 1135 ++++--- .../huggingface/pipelines_torch.ipynb | 1074 ++++--- .../huggingface/pytriton_utils.py | 1 + .../sentence_transformers_torch.ipynb | 1079 ++++--- .../dl_inference/images/spark-pytriton.png | Bin 0 -> 112801 bytes .../pytorch/housing_regression_torch.ipynb | 1982 ++++++++++++ .../pytorch/image_classification_torch.ipynb | 1594 ++++++---- .../models_config/fashion_mnist/config.pbtxt | 30 - .../models_config/housing_model/config.pbtxt | 30 - .../dl_inference/pytorch/pytriton_utils.py | 1 + .../pytorch/regression_torch.ipynb | 2725 ----------------- .../Spark-DL/dl_inference/pytriton_utils.py | 130 + .../Spark-DL/dl_inference/requirements.txt | 9 +- .../tensorflow/image_classification_tf.ipynb | 1292 ++++---- .../tensorflow/keras-metadata_tf.ipynb | 1259 -------- ..._tf.ipynb => keras_preprocessing_tf.ipynb} | 1285 ++++---- .../tensorflow/keras_resnet50_tf.ipynb | 1415 +++++++++ .../models_config/feature_columns/1/model.py | 162 - .../feature_columns/config.pbtxt | 111 - .../models_config/mnist_model/config.pbtxt | 17 - .../models_config/resnet50/config.pbtxt | 17 - .../text_classification/1/model.py | 159 - .../text_classification/config.pbtxt | 51 - .../dl_inference/tensorflow/pytriton_utils.py | 1 + .../tensorflow/text_classification_tf.ipynb | 1713 ++++++----- 44 files changed, 11045 insertions(+), 11311 deletions(-) create mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/README.md create mode 100755 examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/init_spark_dl.sh create mode 100755 examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/start_cluster.sh create mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/README.md create mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/init_spark_dl.sh create mode 100755 examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/start_cluster.sh delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_tf/1/model.py delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_tf/config.pbtxt delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_torch/1/model.py delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_torch/config.pbtxt delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_tf/1/model.py delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_tf/config.pbtxt delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_torch/1/model.py delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_torch/config.pbtxt delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_transformer_torch/1/model.py delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_transformer_torch/config.pbtxt create mode 120000 examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pytriton_utils.py create mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/images/spark-pytriton.png create mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/housing_regression_torch.ipynb delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/fashion_mnist/config.pbtxt delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/housing_model/config.pbtxt create mode 120000 examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/pytriton_utils.py delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/regression_torch.ipynb create mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/pytriton_utils.py delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras-metadata_tf.ipynb rename examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/{feature_columns_tf.ipynb => keras_preprocessing_tf.ipynb} (56%) create mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_resnet50_tf.ipynb delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/feature_columns/1/model.py delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/feature_columns/config.pbtxt delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/mnist_model/config.pbtxt delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/resnet50/config.pbtxt delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/text_classification/1/model.py delete mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/text_classification/config.pbtxt create mode 120000 examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/pytriton_utils.py diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md b/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md index d704f299..034ee2af 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md @@ -1,11 +1,17 @@ -# Spark DL Inference Using External Frameworks +# Deep Learning Inference on Spark -Example notebooks for the [predict_batch_udf](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf) function introduced in Spark 3.4. +Example notebooks demonstrating **distributed deep learning inference** using the [predict_batch_udf](https://developer.nvidia.com/blog/distributed-deep-learning-made-easy-with-spark-3-4/) introduced in Spark 3.4.0. +These notebooks also demonstrate integration with [Triton Inference Server](https://docs.nvidia.com/deeplearning/triton-inference-server/user-guide/docs/index.html), an open-source, GPU-accelerated serving solution for DL. -## Overview +## Contents: +- [Overview](#overview) +- [Running Locally](#running-locally) +- [Running on Cloud](#running-on-cloud-environments) +- [Integration with Triton Inference Server](#inference-with-triton) -This directory contains notebooks for each DL framework (based on their own published examples). The goal is to demonstrate how models trained and saved on single-node machines can be easily used for parallel inferencing on Spark clusters. +## Overview +These notebooks demonstrate how models from external frameworks (Torch, Huggingface, Tensorflow) trained on single-worker machines can be used for large-scale distributed inference on Spark clusters. For example, a basic model trained in TensorFlow and saved on disk as "mnist_model" can be used in Spark as follows: ``` import numpy as np @@ -28,35 +34,33 @@ df = spark.read.parquet("mnist_data") predictions = df.withColumn("preds", mnist("data")).collect() ``` -In this simple case, the `predict_batch_fn` will use TensorFlow APIs to load the model and return a simple `predict` function which operates on numpy arrays. The `predict_batch_udf` will automatically convert the Spark DataFrame columns to the expected numpy inputs. +In this simple case, the `predict_batch_fn` will use TensorFlow APIs to load the model and return a simple `predict` function. The `predict_batch_udf` will handle the data conversion from Spark DataFrame columns into batched numpy inputs. + -All notebooks have been saved with sample outputs for quick browsing. -Here is a full list of the notebooks with their published example links: +#### Notebook List -| | Category | Notebook Name | Description | Link +Below is a full list of the notebooks with links to the examples they are based on. All notebooks have been saved with sample outputs for quick browsing. + +| | Framework | Notebook Name | Description | Link | ------------- | ------------- | ------------- | ------------- | ------------- | 1 | PyTorch | Image Classification | Training a model to predict clothing categories in FashionMNIST, including accelerated inference with Torch-TensorRT. | [Link](https://pytorch.org/tutorials/beginner/basics/quickstart_tutorial.html) -| 2 | PyTorch | Regression | Training a model to predict housing prices in the California Housing Dataset, including accelerated inference with Torch-TensorRT. | [Link](https://github.com/christianversloot/machine-learning-articles/blob/main/how-to-create-a-neural-network-for-regression-with-pytorch.md) +| 2 | PyTorch | Housing Regression | Training a model to predict housing prices in the California Housing Dataset, including accelerated inference with Torch-TensorRT. | [Link](https://github.com/christianversloot/machine-learning-articles/blob/main/how-to-create-a-neural-network-for-regression-with-pytorch.md) | 3 | Tensorflow | Image Classification | Training a model to predict hand-written digits in MNIST. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/save_and_load.ipynb) -| 4 | Tensorflow | Feature Columns | Training a model with preprocessing layers to predict likelihood of pet adoption in the PetFinder mini dataset. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/preprocessing_layers.ipynb) -| 5 | Tensorflow | Keras Metadata | Training ResNet-50 to perform flower recognition on Databricks. | [Link](https://docs.databricks.com/en/_extras/notebooks/source/deep-learning/keras-metadata.html) +| 4 | Tensorflow | Keras Preprocessing | Training a model with preprocessing layers to predict likelihood of pet adoption in the PetFinder mini dataset. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/preprocessing_layers.ipynb) +| 5 | Tensorflow | Keras Resnet50 | Training ResNet-50 to perform flower recognition from flower images. | [Link](https://docs.databricks.com/en/_extras/notebooks/source/deep-learning/keras-metadata.html) | 6 | Tensorflow | Text Classification | Training a model to perform sentiment analysis on the IMDB dataset. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/text_classification.ipynb) -| 7+8 | HuggingFace | Conditional Generation | Sentence translation using the T5 text-to-text transformer, with notebooks demoing both Torch and Tensorflow. | [Link](https://huggingface.co/docs/transformers/model_doc/t5#t5) -| 9+10 | HuggingFace | Pipelines | Sentiment analysis using Huggingface pipelines, with notebooks demoing both Torch and Tensorflow. | [Link](https://huggingface.co/docs/transformers/quicktour#pipeline-usage) -| 11 | HuggingFace | Sentence Transformers | Sentence embeddings using the SentenceTransformers framework in Torch. | [Link](https://huggingface.co/sentence-transformers) +| 7+8 | HuggingFace | Conditional Generation | Sentence translation using the T5 text-to-text transformer for both Torch and Tensorflow. | [Link](https://huggingface.co/docs/transformers/model_doc/t5#t5) +| 9+10 | HuggingFace | Pipelines | Sentiment analysis using Huggingface pipelines for both Torch and Tensorflow. | [Link](https://huggingface.co/docs/transformers/quicktour#pipeline-usage) +| 11 | HuggingFace | Sentence Transformers | Sentence embeddings using SentenceTransformers in Torch. | [Link](https://huggingface.co/sentence-transformers) -## Running the Notebooks +## Running Locally -If you want to run the notebooks yourself, please follow these instructions. - -**Notes**: -- The notebooks require a GPU environment for the executors. -- Please create separate environments for PyTorch and Tensorflow examples as specified below. This will avoid conflicts between the CUDA libraries bundled with their respective versions. The Huggingface examples will have a `_torch` or `_tf` suffix to specify the environment used. -- The PyTorch notebooks include model compilation and accelerated inference with TensorRT. While not included in the notebooks, Tensorflow also supports [integration with TensorRT](https://docs.nvidia.com/deeplearning/frameworks/tf-trt-user-guide/index.html), but may require downgrading the TF version. -- For demonstration purposes, these examples just use a local Spark Standalone cluster with a single executor, but you should be able to run them on any distributed Spark cluster. +To run the notebooks locally, please follow these instructions: #### Create environment +Each notebook has a suffix `_torch` or `_tf` specifying the environment used. + **For PyTorch:** ``` conda create -n spark-dl-torch python=3.11 @@ -70,36 +74,57 @@ conda activate spark-dl-tf pip install -r tf_requirements.txt ``` -#### Launch Jupyter + Spark +#### Start Cluster + +For demonstration, these instructions just use a local Standalone cluster with a single executor, but they can be run on any distributed Spark cluster. For cloud environments, see [below](#running-on-cloud-environments). +```shell +# Replace with your Spark installation path +export SPARK_HOME= ``` -# setup environment variables -export SPARK_HOME=/path/to/spark + +```shell +# Configure and start cluster export MASTER=spark://$(hostname):7077 export SPARK_WORKER_INSTANCES=1 export CORES_PER_WORKER=8 -export PYSPARK_DRIVER_PYTHON=jupyter -export PYSPARK_DRIVER_PYTHON_OPTS='lab' - -# start spark standalone cluster +export SPARK_WORKER_OPTS="-Dspark.worker.resource.gpu.amount=1 -Dspark.worker.resource.gpu.discoveryScript=$SPARK_HOME/examples/src/main/scripts/getGpusResources.sh" ${SPARK_HOME}/sbin/start-master.sh; ${SPARK_HOME}/sbin/start-worker.sh -c ${CORES_PER_WORKER} -m 16G ${MASTER} +``` -# start jupyter with pyspark -${SPARK_HOME}/bin/pyspark --master ${MASTER} \ ---driver-memory 8G \ ---executor-memory 8G \ ---conf spark.python.worker.reuse=True +The notebooks are ready to run! Each notebook has a cell to connect to the standalone cluster and create a SparkSession. -# BROWSE to localhost:8888 to view/run notebooks +**Notes**: +- Please create separate environments for PyTorch and Tensorflow notebooks as specified above. This will avoid conflicts between the CUDA libraries bundled with their respective versions. +- `requirements.txt` installs pyspark>=3.4.0. Make sure the installed PySpark version is compatible with your system's Spark installation. +- The notebooks require a GPU environment for the executors. +- The PyTorch notebooks include model compilation and accelerated inference with TensorRT. While not included in the notebooks, Tensorflow also supports [integration with TensorRT](https://docs.nvidia.com/deeplearning/frameworks/tf-trt-user-guide/index.html), but as of writing it is not supported in TF==2.17.0. -# stop spark standalone cluster -${SPARK_HOME}/sbin/stop-worker.sh; ${SPARK_HOME}/sbin/stop-master.sh +**Troubleshooting:** +If you encounter issues starting the Triton server, you may need to link your libstdc++ file to the conda environment, e.g.: +```shell +ln -sf /usr/lib/x86_64-linux-gnu/libstdc++.so.6 ${CONDA_PREFIX}/lib/libstdc++.so.6 ``` -## Triton Inference Server +## Running on Cloud Environments + +We also provide instructions to run the notebooks on CSP Spark environments. +See the instructions for [Databricks](databricks/README.md) and [GCP Dataproc](dataproc/README.md). + +## Inference with Triton + +The notebooks also demonstrate integration with the [Triton Inference Server](https://docs.nvidia.com/deeplearning/triton-inference-server/user-guide/docs/index.html), an open-source serving platform for deep learning models, which includes many [features and performance optimizations](https://docs.nvidia.com/deeplearning/triton-inference-server/user-guide/docs/index.html#triton-major-features) to streamline inference. +The notebooks use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like Python framework that handles communication with the Triton server. + +drawing -The example notebooks also demonstrate integration with [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. +The diagram above shows how Spark distributes inference tasks to run on the Triton Inference Server, with PyTriton handling request/response communication with the server. -**Note**: Some examples may require special configuration of server as highlighted in the notebooks. +The process looks like this: +- Distribute a PyTriton task across the Spark cluster, instructing each worker to launch a Triton server process. + - Use stage-level scheduling to ensure there is a 1:1 mapping between worker nodes and servers. +- Define a Triton inference function, which contains a client that binds to the local server on a given worker and sends inference requests. +- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark. +- Finally, distribute a shutdown signal to terminate the Triton server processes on each worker. -**Note**: for demonstration purposes, the Triton Inference Server integrations just launch the server in a docker container on the local host, so you will need to [install docker](https://docs.docker.com/engine/install/) on your local host. Most real-world deployments will likely be hosted on remote machines. +For more information on how PyTriton works, see the [PyTriton docs](https://triton-inference-server.github.io/pytriton/latest/high_level_design/). \ No newline at end of file diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/README.md b/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/README.md new file mode 100644 index 00000000..58c7d12b --- /dev/null +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/README.md @@ -0,0 +1,55 @@ +# Spark DL Inference on Databricks + +**Note**: fields in \ require user inputs. + +## Setup + +1. Install the latest [databricks-cli](https://docs.databricks.com/en/dev-tools/cli/tutorial.html) and configure for your workspace. + +2. Specify the path to your Databricks workspace: + ```shell + export WS_PATH= + + export NOTEBOOK_DEST=${WS_PATH}/spark-dl/notebook_torch.ipynb + export UTILS_DEST=${WS_PATH}/spark-dl/pytriton_utils.py + export INIT_DEST=${WS_PATH}/spark-dl/init_spark_dl.sh + ``` +3. Specify the local paths to the notebook you wish to run, the utils file, and the init script. + As an example for a PyTorch notebook: + ```shell + export NOTEBOOK_SRC= + export UTILS_SRC= + export INIT_SRC=$(pwd)/setup/init_spark_dl.sh + ``` +4. Specify the framework to torch or tf, corresponding to the notebook you wish to run. Continuing with the PyTorch example: + ```shell + export FRAMEWORK=torch + ``` + This will tell the init script which libraries to install on the cluster. + +5. Copy the files to the Databricks Workspace: + ```shell + databricks workspace import $NOTEBOOK_DEST --format JUPYTER --file $NOTEBOOK_SRC + databricks workspace import $UTILS_DEST --format AUTO --file $UTILS_SRC + databricks workspace import $INIT_DEST --format AUTO --file $INIT_SRC + ``` + +6. Launch the cluster with the provided script (note that the script specifies **Azure instances** by default; change as needed): + ```shell + cd setup + chmod +x start_cluster.sh + ./start_cluster.sh + ``` + + OR, start the cluster from the Databricks UI: + + - Go to `Compute > Create compute` and set the desired cluster settings. + - Integration with Triton inference server uses stage-level scheduling (Spark>=3.4.0). Make sure to: + - use a cluster with GPU resources + - set a value for `spark.executor.cores` + - ensure that `spark.executor.resource.gpu.amount` = 1 + - Under `Advanced Options > Init Scripts`, upload the init script from your workspace. + - Under environment variables, set `FRAMEWORK=torch` or `FRAMEWORK=tf` based on the notebook used. + - For Tensorflow notebooks, we recommend setting the environment variable `TF_GPU_ALLOCATOR=cuda_malloc_async` (especially for Huggingface LLM models), which enables the CUDA driver to implicity release unused memory from the pool. + +7. Navigate to the notebook in your workspace and attach it to the cluster. The default cluster name is `spark-dl-inference-$FRAMEWORK`. \ No newline at end of file diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/init_spark_dl.sh b/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/init_spark_dl.sh new file mode 100755 index 00000000..39673bf6 --- /dev/null +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/init_spark_dl.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright (c) 2025, NVIDIA CORPORATION. + +set -euxo pipefail + +# install requirements +sudo /databricks/python3/bin/pip3 install --upgrade pip + +if [[ "${FRAMEWORK}" == "torch" ]]; then + cat < temp_requirements.txt +datasets==3.* +transformers +urllib3<2 +nvidia-pytriton +torch +torchvision --extra-index-url https://download.pytorch.org/whl/cu121 +torch-tensorrt +tensorrt --extra-index-url https://download.pytorch.org/whl/cu121 +sentence_transformers +sentencepiece +nvidia-modelopt[all] --extra-index-url https://pypi.nvidia.com +EOF +elif [[ "${FRAMEWORK}" == "tf" ]]; then + cat < temp_requirements.txt +datasets==3.* +transformers +urllib3<2 +nvidia-pytriton +EOF +else + echo "Please export FRAMEWORK as torch or tf per README" + exit 1 +fi + +sudo /databricks/python3/bin/pip3 install --upgrade --force-reinstall -r temp_requirements.txt +rm temp_requirements.txt diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/start_cluster.sh b/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/start_cluster.sh new file mode 100755 index 00000000..98c1d5ac --- /dev/null +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/start_cluster.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright (c) 2025, NVIDIA CORPORATION. + +set -eo pipefail + +# configure arguments +if [[ -z ${INIT_DEST} ]]; then + echo "Please make sure INIT_DEST is exported per README.md" + exit 1 +fi + +if [[ -z ${FRAMEWORK} ]]; then + echo "Please make sure FRAMEWORK is exported to torch or tf per README.md" + exit 1 +fi + +json_config=$(cat < require user inputs. + +#### Setup GCloud CLI + +1. Install the latest [gcloud-cli](https://cloud.google.com/sdk/docs/install) and initialize with `gcloud init`. + +2. Configure the following settings: + ```shell + export PROJECT= + export DATAPROC_REGION= + export COMPUTE_REGION= + export COMPUTE_ZONE= + + gcloud config set project ${PROJECT} + gcloud config set dataproc/region ${DATAPROC_REGION} + gcloud config set compute/region ${COMPUTE_REGION} + gcloud config set compute/zone ${COMPUTE_ZONE} + ``` + +#### Copy files to GCS + +3. Create a GCS bucket if you don't already have one: + ```shell + export GCS_BUCKET= + + gcloud storage buckets create gs://${GCS_BUCKET} + ``` + +4. Specify the local path to the notebook(s) and copy to the GCS bucket. + As an example for a torch notebook: + ```shell + export SPARK_DL_HOME=${GCS_BUCKET}/spark-dl + + gcloud storage cp gs://${SPARK_DL_HOME}/notebooks/ + ``` + Repeat this step for any notebooks you wish to run. All notebooks under `gs://${SPARK_DL_HOME}/notebooks/` will be copied to the master node during initialization. + +5. Copy the utils file to the GCS bucket. + ```shell + gcloud storage cp gs://${SPARK_DL_HOME}/ + ``` + +#### Start cluster and run + +5. Specify the framework to use (torch or tf), which will determine what libraries to install on the cluster. For example: + ```shell + export FRAMEWORK=torch + ``` + Run the cluster startup script. The script will also retrieve and use the [spark-rapids initialization script](https://github.com/GoogleCloudDataproc/initialization-actions/blob/master/spark-rapids/spark-rapids.sh) to setup GPU resources. + ```shell + cd setup + chmod +x start_cluster.sh + ./start_cluster.sh + ``` + By default, the script creates a 4 node GPU cluster named `${USER}-spark-dl-inference-${FRAMEWORK}`. + +7. Browse to the Jupyter web UI: + - Go to `Dataproc` > `Clusters` > `(Cluster Name)` > `Web Interfaces` > `Jupyter/Lab` + + Or, get the link by running this command (under httpPorts > Jupyter/Lab): + ```shell + gcloud dataproc clusters describe ${CLUSTER_NAME} --region=${COMPUTE_REGION} + ``` + +8. Open and run the notebook interactively with the **Python 3 kernel**. +The notebooks can be found under `Local Disk/spark-dl-notebooks` on the master node (folder icon on the top left > Local Disk). \ No newline at end of file diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/init_spark_dl.sh b/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/init_spark_dl.sh new file mode 100644 index 00000000..264e3d5b --- /dev/null +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/init_spark_dl.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright (c) 2025, NVIDIA CORPORATION. + +set -euxo pipefail + +function get_metadata_attribute() { + local -r attribute_name=$1 + local -r default_value=$2 + /usr/share/google/get_metadata_value "attributes/${attribute_name}" || echo -n "${default_value}" +} + +SPARK_DL_HOME=$(get_metadata_attribute spark-dl-home UNSET) +if [[ ${SPARK_DL_HOME} == "UNSET" ]]; then + echo "Please set --metadata spark-dl-home" + exit 1 +fi + +GCS_BUCKET=$(get_metadata_attribute gcs-bucket UNSET) +if [[ ${GCS_BUCKET} == "UNSET" ]]; then + echo "Please set --metadata gcs-bucket" + exit 1 +fi + +REQUIREMENTS=$(get_metadata_attribute requirements UNSET) +if [[ ${REQUIREMENTS} == "UNSET" ]]; then + echo "Please set --metadata requirements" + exit 1 +fi + +# mount gcs bucket as fuse +export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s` +echo "deb https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list +curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - +sudo apt-get update +sudo apt-get install -y fuse gcsfuse +sudo mkdir -p /mnt/gcs +gcsfuse -o allow_other --implicit-dirs ${GCS_BUCKET} /mnt/gcs +sudo chmod -R 777 /mnt/gcs + +# install requirements +pip install --upgrade pip +echo "${REQUIREMENTS}" > temp_requirements.txt +pip install --upgrade --force-reinstall -r temp_requirements.txt +rm temp_requirements.txt + +# copy notebooks to master +ROLE=$(/usr/share/google/get_metadata_value attributes/dataproc-role) +if [[ "${ROLE}" == 'Master' ]]; then + if gsutil -q stat gs://${SPARK_DL_HOME}/notebooks/**; then + mkdir spark-dl-notebooks + gcloud storage cp -r gs://${SPARK_DL_HOME}/notebooks/* spark-dl-notebooks + gcloud storage cp gs://${SPARK_DL_HOME}/pytriton_utils.py spark-dl-notebooks/ + else + echo "Failed to retrieve notebooks from gs://${SPARK_DL_HOME}/notebooks/" + exit 1 + fi +fi + +sudo chmod -R a+rw /home/ +sudo systemctl daemon-reload diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/start_cluster.sh b/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/start_cluster.sh new file mode 100755 index 00000000..4e9a7791 --- /dev/null +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/start_cluster.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# Copyright (c) 2025, NVIDIA CORPORATION. + +set -eo pipefail + +# configure arguments +if [[ -z ${GCS_BUCKET} ]]; then + echo "Please export GCS_BUCKET per README.md" + exit 1 +fi + +if [[ -z ${FRAMEWORK} ]]; then + echo "Please export FRAMEWORK as 'torch' or 'tf'" + exit 1 +fi + +if [[ -z ${COMPUTE_REGION} ]]; then + COMPUTE_REGION=$(gcloud config get-value compute/region) + if [[ -z ${COMPUTE_REGION} ]]; then + echo "Please export COMPUTE_REGION per README.md or set it in gcloud config." + exit 1 + fi +fi + +SPARK_DL_HOME=${SPARK_DL_HOME:-${GCS_BUCKET}/spark-dl} + +# copy init script to gcs +gcloud storage cp init_spark_dl.sh gs://${SPARK_DL_HOME}/init/ +INIT_PATH=gs://${SPARK_DL_HOME}/init/init_spark_dl.sh + +# retrieve and upload spark-rapids initialization script to gcs +curl -LO https://raw.githubusercontent.com/GoogleCloudDataproc/initialization-actions/master/spark-rapids/spark-rapids.sh +# don't enable rapids plugin by default +sed -i '/spark.plugins=com.nvidia.spark.SQLPlugin/d' spark-rapids.sh +gcloud storage cp spark-rapids.sh gs://${SPARK_DL_HOME}/init/ +# rm spark-rapids.sh + +COMMON_REQUIREMENTS="numpy +pandas +matplotlib +portalocker +pyarrow +pydot +scikit-learn +huggingface +datasets==3.* +transformers +urllib3<2 +nvidia-pytriton" + +TORCH_REQUIREMENTS="${COMMON_REQUIREMENTS} +torch +torchvision --extra-index-url https://download.pytorch.org/whl/cu121 +torch-tensorrt +tensorrt --extra-index-url https://download.pytorch.org/whl/cu121 +sentence_transformers +sentencepiece +nvidia-modelopt[all] --extra-index-url https://pypi.nvidia.com" + +TF_REQUIREMENTS="${COMMON_REQUIREMENTS} +tensorflow[and-cuda] +tf-keras" + +cluster_name=${USER}-spark-dl-inference-${FRAMEWORK} +if [[ ${FRAMEWORK} == "torch" ]]; then + requirements=${TORCH_REQUIREMENTS} + echo "=========================================================" + echo "Starting PyTorch cluster ${cluster_name}" + echo "=========================================================" +elif [[ ${FRAMEWORK} == "tf" ]]; then + requirements=${TF_REQUIREMENTS} + echo "=========================================================" + echo "Starting Tensorflow cluster ${cluster_name}" + echo "=========================================================" +else + echo "Please export FRAMEWORK as torch or tf" + exit 1 +fi + +# start cluster if not already running +if gcloud dataproc clusters list | grep -q "${cluster_name}"; then + echo "Cluster ${cluster_name} already exists." +else + gcloud dataproc clusters create ${cluster_name} \ + --image-version=2.2-ubuntu \ + --region ${COMPUTE_REGION} \ + --master-machine-type n1-standard-16 \ + --num-workers 4 \ + --worker-min-cpu-platform="Intel Skylake" \ + --worker-machine-type n1-standard-16 \ + --master-accelerator type=nvidia-tesla-t4,count=1 \ + --worker-accelerator type=nvidia-tesla-t4,count=1 \ + --initialization-actions gs://${SPARK_DL_HOME}/init/spark-rapids.sh,${INIT_PATH} \ + --metadata gpu-driver-provider="NVIDIA" \ + --metadata gcs-bucket=${GCS_BUCKET} \ + --metadata spark-dl-home=${SPARK_DL_HOME} \ + --metadata requirements="${requirements}" \ + --worker-local-ssd-interface=NVME \ + --optional-components=JUPYTER \ + --bucket ${GCS_BUCKET} \ + --enable-component-gateway \ + --max-idle "60m" \ + --subnet=default \ + --no-shielded-secure-boot +fi diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_tf.ipynb index 3105e066..cdf6dc49 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_tf.ipynb @@ -5,9 +5,12 @@ "id": "777fc40d", "metadata": {}, "source": [ + "\n", + "\n", "# PySpark Huggingface Inferencing\n", - "## Conditional generation with Tensorflow\n", + "### Conditional generation with Tensorflow\n", "\n", + "In this notebook, we demonstrate distributed inference with the T5 transformer to perform sentence translation. \n", "From: https://huggingface.co/docs/transformers/model_doc/t5" ] }, @@ -16,9 +19,7 @@ "id": "05c79ac4-bf25-421e-b55e-020d6d9e15d5", "metadata": {}, "source": [ - "### Using TensorFlow\n", - "Note that cuFFT/cuDNN/cuBLAS registration errors are expected with `tf=2.17.0` and will not affect behavior, as noted in [this issue.](https://github.com/tensorflow/tensorflow/issues/62075) \n", - "This notebook does not demonstrate inference with TensorRT, as [TF-TRT](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/index.html#tensorrt-10) does not yet support `tf=2.17.0`. See the `pytorch` notebooks for TensorRT demos." + "Note that cuFFT/cuDNN/cuBLAS registration errors are expected (as of `tf=2.17.0`) and will not affect behavior, as noted in [this issue.](https://github.com/tensorflow/tensorflow/issues/62075)" ] }, { @@ -31,42 +32,28 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-11 00:16:59.451769: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2024-10-11 00:16:59.459246: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2024-10-11 00:16:59.467162: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2024-10-11 00:16:59.469569: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2024-10-11 00:16:59.475888: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "2025-01-27 12:01:14.829270: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2025-01-27 12:01:14.836118: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2025-01-27 12:01:14.843723: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2025-01-27 12:01:14.845951: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2025-01-27 12:01:14.851831: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2024-10-11 00:16:59.818338: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" + "2025-01-27 12:01:15.226235: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" ] } ], "source": [ - "from transformers import AutoTokenizer, TFT5ForConditionalGeneration" - ] - }, - { - "cell_type": "markdown", - "id": "5346a20c", - "metadata": {}, - "source": [ - "Enabling Huggingface tokenizer parallelism so that it is not automatically disabled with Python parallelism. See [this thread](https://github.com/huggingface/transformers/issues/5486) for more info. " - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "a1008e27", - "metadata": {}, - "outputs": [], - "source": [ + "from transformers import AutoTokenizer, TFT5ForConditionalGeneration\n", + "\n", + "# Manually enable Huggingface tokenizer parallelism to avoid disabling with PySpark parallelism.\n", + "# See (https://github.com/huggingface/transformers/issues/5486) for more info. \n", "import os\n", "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "275890d7", "metadata": {}, "outputs": [ @@ -76,6 +63,16 @@ "text": [ "2.17.0\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", + "I0000 00:00:1738008075.848429 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008075.871583 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008075.875015 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" + ] } ], "source": [ @@ -95,7 +92,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "2684fb41-9467-40c0-9d7e-a1cc867c5a3c", "metadata": {}, "outputs": [ @@ -103,7 +100,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-11 00:17:00.886565: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 46024 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n", + "I0000 00:00:1738008076.235518 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008076.238670 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008076.241422 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008076.340179 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008076.341199 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008076.342108 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2025-01-27 12:01:16.343039: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 43844 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n", "All PyTorch model weights were used when initializing TFT5ForConditionalGeneration.\n", "\n", "All the weights of TFT5ForConditionalGeneration were initialized from the PyTorch model.\n", @@ -128,7 +131,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "6eb2dfdb-0ad3-4d0f-81a4-268d92c53759", "metadata": {}, "outputs": [ @@ -137,26 +140,24 @@ "output_type": "stream", "text": [ "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", - "I0000 00:00:1728605822.106234 276792 service.cc:146] XLA service 0x7f53a8003630 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n", - "I0000 00:00:1728605822.106259 276792 service.cc:154] StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n", - "2024-10-11 00:17:02.108842: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", - "2024-10-11 00:17:02.117215: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8907\n", - "I0000 00:00:1728605822.137920 276792 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n" + "I0000 00:00:1738008077.718863 2973172 service.cc:146] XLA service 0x7370c0003170 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n", + "I0000 00:00:1738008077.718882 2973172 service.cc:154] StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n", + "2025-01-27 12:01:17.726803: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", + "2025-01-27 12:01:17.754699: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8907\n", + "I0000 00:00:1738008077.795877 2973172 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n" ] } ], "source": [ - "input_ids = tokenizer(input_sequences, \n", - " padding=\"longest\", \n", - " max_length=512,\n", - " truncation=True,\n", - " return_tensors=\"tf\").input_ids\n", - "outputs = model.generate(input_ids, max_length=20)" + "inputs = tokenizer(input_sequences, \n", + " padding=True,\n", + " return_tensors=\"tf\")\n", + "outputs = model.generate(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_length=128)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "720158d4-e0e0-4904-b096-e5aede756afd", "metadata": {}, "outputs": [ @@ -168,7 +169,7 @@ " 'HuggingFace ist ein Unternehmen']" ] }, - "execution_count": 6, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -177,63 +178,76 @@ "[tokenizer.decode(o, skip_special_tokens=True) for o in outputs]" ] }, + { + "cell_type": "markdown", + "id": "546eabe0", + "metadata": {}, + "source": [ + "## PySpark" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "68121304-f1df-466e-9347-c9d2b36a9b3a", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql.types import *\n", + "from pyspark import SparkConf\n", + "from pyspark.sql import SparkSession\n", + "from pyspark.sql.functions import pandas_udf, col, struct\n", + "from pyspark.ml.functions import predict_batch_udf" + ] + }, { "cell_type": "code", "execution_count": 7, - "id": "8d4b364b-13cb-48ea-a97a-ccfc9e408075", + "id": "2f6db1f0-7d68-4af7-8bd6-c9fa45906c61", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'tf'" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "model.framework" + "import json\n", + "import pandas as pd\n", + "import datasets\n", + "from datasets import load_dataset\n", + "datasets.disable_progress_bars()" ] }, { "cell_type": "markdown", - "id": "546eabe0", + "id": "0d636975", "metadata": {}, "source": [ - "## PySpark" + "Check the cluster environment to handle any platform-specific Spark configurations." ] }, { "cell_type": "code", "execution_count": 8, - "id": "2f6db1f0-7d68-4af7-8bd6-c9fa45906c61", + "id": "ca351245", "metadata": {}, "outputs": [], "source": [ - "import os\n", - "from pathlib import Path\n", - "from datasets import load_dataset" + "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n", + "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n", + "on_standalone = not (on_databricks or on_dataproc)" ] }, { - "cell_type": "code", - "execution_count": 9, - "id": "68121304-f1df-466e-9347-c9d2b36a9b3a", + "cell_type": "markdown", + "id": "d3199f8b", "metadata": {}, - "outputs": [], "source": [ - "from pyspark.sql.types import *\n", - "from pyspark.sql import SparkSession\n", - "from pyspark import SparkConf\n", - "import socket" + "#### Create Spark Session\n", + "\n", + "For local standalone clusters, we'll connect to the cluster and create the Spark Session. \n", + "For CSP environments, Spark will either be preconfigured (Databricks) or we'll need to create the Spark Session (Dataproc)." ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "id": "6279a849", "metadata": {}, "outputs": [ @@ -241,108 +255,128 @@ "name": "stderr", "output_type": "stream", "text": [ - "24/10/11 00:17:03 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "24/10/11 00:17:03 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "25/01/27 20:01:19 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/01/27 20:01:19 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "24/10/11 00:17:03 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" + "25/01/27 20:01:19 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "25/01/27 20:01:19 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" ] } ], "source": [ - "conda_env = os.environ.get(\"CONDA_PREFIX\")\n", - "hostname = socket.gethostname()\n", - "\n", "conf = SparkConf()\n", + "\n", "if 'spark' not in globals():\n", - " # If Spark is not already started with Jupyter, attach to Spark Standalone\n", - " import socket\n", - " hostname = socket.gethostname()\n", - " conf.setMaster(f\"spark://{hostname}:7077\") # assuming Master is on default port 7077\n", - "conf.set(\"spark.task.maxFailures\", \"1\")\n", - "conf.set(\"spark.driver.memory\", \"8g\")\n", - "conf.set(\"spark.executor.memory\", \"8g\")\n", - "conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled\", \"false\")\n", - "conf.set(\"spark.sql.pyspark.jvmStacktrace.enabled\", \"true\")\n", - "conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", - "conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"512\")\n", - "conf.set(\"spark.python.worker.reuse\", \"true\")\n", - "# Create Spark Session\n", + " if on_standalone:\n", + " import socket\n", + " \n", + " conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + " hostname = socket.gethostname()\n", + " conf.setMaster(f\"spark://{hostname}:7077\")\n", + " conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", + " conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_env}/lib:{conda_env}/lib/python3.11/site-packages/nvidia_pytriton.libs:$LD_LIBRARY_PATH\")\n", + " source = \"/usr/lib/x86_64-linux-gnu/libstdc++.so.6\"\n", + " target = f\"{conda_env}/lib/libstdc++.so.6\"\n", + " try:\n", + " if os.path.islink(target) or os.path.exists(target):\n", + " os.remove(target)\n", + " os.symlink(source, target)\n", + " except OSError as e:\n", + " print(f\"Error creating symlink: {e}\")\n", + " elif on_dataproc:\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conda_lib_path=\"/opt/conda/miniconda3/lib\"\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_lib_path}:$LD_LIBRARY_PATH\")\n", + " conf.set(\"spark.executorEnv.TF_GPU_ALLOCATOR\", \"cuda_malloc_async\")\n", + " conf.set(\"spark.executor.instances\", \"4\") # dataproc defaults to 2\n", + "\n", + " conf.set(\"spark.executor.cores\", \"8\")\n", + " conf.set(\"spark.task.resource.gpu.amount\", \"0.125\")\n", + " conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n", + " conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", + " conf.set(\"spark.python.worker.reuse\", \"true\")\n", + "\n", + "conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"1000\")\n", "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", "sc = spark.sparkContext" ] }, + { + "cell_type": "markdown", + "id": "7f311650", + "metadata": {}, + "source": [ + "Load the IMBD Movie Reviews dataset from Huggingface." + ] + }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "id": "b8453111-d068-49bb-ab91-8ae3d8bcdb7a", "metadata": {}, "outputs": [], "source": [ - "# load IMDB reviews (test) dataset\n", - "data = load_dataset(\"imdb\", split=\"test\")" + "dataset = load_dataset(\"imdb\", split=\"test\")\n", + "dataset = dataset.to_pandas().drop(columns=\"label\")" + ] + }, + { + "cell_type": "markdown", + "id": "6fd5b472-47e8-4804-9907-772793fedb2b", + "metadata": {}, + "source": [ + "### Create PySpark DataFrame" ] }, { "cell_type": "code", - "execution_count": 12, - "id": "7ad01d4a", + "execution_count": 11, + "id": "d24d9404-0269-476e-a9dd-1842667c915a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "25000" + "StructType([StructField('text', StringType(), True)])" ] }, - "execution_count": 12, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "lines = []\n", - "for example in data:\n", - " lines.append([example[\"text\"].split(\".\")[0]])\n", - "\n", - "len(lines)" - ] - }, - { - "cell_type": "markdown", - "id": "6fd5b472-47e8-4804-9907-772793fedb2b", - "metadata": {}, - "source": [ - "### Create PySpark DataFrame" + "df = spark.createDataFrame(dataset).repartition(8)\n", + "df.schema" ] }, { "cell_type": "code", - "execution_count": 13, - "id": "d24d9404-0269-476e-a9dd-1842667c915a", + "execution_count": 12, + "id": "c76314b7", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "StructType([StructField('lines', StringType(), True)])" + "25000" ] }, - "execution_count": 13, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df = spark.createDataFrame(lines, ['lines']).repartition(8)\n", - "df.schema" + "df.count()" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "id": "4384c762-1f79-4f60-876c-94b1f552e8fb", "metadata": {}, "outputs": [ @@ -350,16 +384,16 @@ "name": "stderr", "output_type": "stream", "text": [ - " \r" + "25/01/27 20:01:27 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] }, { "data": { "text/plain": [ - "[Row(lines='(Some Spoilers) Dull as dishwater slasher flick that has this deranged homeless man Harry, Darwyn Swalve, out murdering real-estate agent all over the city of L')]" + "[Row(text=\"Anyone remember the first CKY, CKY2K etc..? Back when it was about making crazy cool stuff, rather than watching Bam Margera act like a douchebag, spoiled 5 year old, super/rock-star wannabe.

The show used to be awesome, however, Bam's fame and wealth has led him to believe, that we now enjoy him acting childish and idiotic, more than actual cool stuff, that used to be in ex. CKY2K.

The acts are so repetitive, there's like nothing new, except annoying stupidity and rehearsed comments... The only things we see is Bam Margera, so busy showing us how much he doesn't care, how much money he got or whatsoever.

I really got nothing much left to say except, give us back CKY2K, cause Bam suck..

I enjoy watching Steve-o, Knoxville etc. a thousand times more.\")]" ] }, - "execution_count": 14, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -378,67 +412,44 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "id": "e7eec8ec-4126-4890-b957-025809fad67d", "metadata": {}, - "outputs": [], - "source": [ - "df.write.mode(\"overwrite\").parquet(\"imdb_test\")" - ] - }, - { - "cell_type": "markdown", - "id": "304e1fc8-42a3-47dd-b3c0-47efd5be1040", - "metadata": {}, - "source": [ - "### Check arrow memory configuration" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "20554ea5-01be-4a30-8607-db5d87786fec", - "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/27 20:01:27 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + ] + } + ], "source": [ - "if int(spark.conf.get(\"spark.sql.execution.arrow.maxRecordsPerBatch\")) > 512:\n", - " print(\"Decreasing `spark.sql.execution.arrow.maxRecordsPerBatch` to ensure the vectorized reader won't run out of memory\")\n", - " spark.conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"512\")\n", - "assert len(df.head()) > 0, \"`df` should not be empty\"" + "data_path = \"spark-dl-datasets/imdb_test\"\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " data_path = \"dbfs:/FileStore/\" + data_path\n", + "\n", + "df.write.mode(\"overwrite\").parquet(data_path)" ] }, { "cell_type": "markdown", - "id": "06a4ecab-c9d9-466f-ba49-902ad1fd5488", + "id": "078425e1", "metadata": {}, "source": [ - "## Inference using Spark DL API\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "e7a00479-1347-4de8-8431-faa77f8cdf4c", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import pandas as pd\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import col, pandas_udf, struct\n", - "from pyspark.sql.types import StringType" + "#### Load and preprocess DataFrame\n", + "\n", + "Define our preprocess function. We'll take the first sentence from each sample as our input for translation." ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 15, "id": "b9a0889a-35b4-493a-8197-1146fc7efd53", "metadata": {}, "outputs": [], "source": [ - "# only use first sentence and add prefix for conditional generation\n", "def preprocess(text: pd.Series, prefix: str = \"\") -> pd.Series:\n", " @pandas_udf(\"string\")\n", " def _preprocess(text: pd.Series) -> pd.Series:\n", @@ -448,7 +459,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 16, "id": "c483e4d4-9ab1-416f-a766-694e17490fd3", "metadata": {}, "outputs": [ @@ -456,130 +467,109 @@ "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| lines|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| This is so overly clichéd you'll want to switch it off after the first 45 minutes|\n", - "| I am a big fan of The ABC Movies of the Week genre|\n", - "|In the early 1990's \"Step-by-Step\" came as a tedious combination of the ultra-cheesy \"Full House\" and the long-defunc...|\n", - "|When The Spirits Within was released, all you heard from Final Fantasy fans was how awful the movie was because it di...|\n", - "| I like to think of myself as a bad movie connoisseur|\n", - "|This film did well at the box office, and the producers of this mess thought the stars had such good chemistry in thi...|\n", - "|Following the pleasingly atmospheric original and the amusingly silly second one, this incredibly dull, slow, and une...|\n", - "| I like CKY and Viva La Bam, so I couldn't resist this when I saw it for £1|\n", - "| I have read all of the reviews for this direct to video movie|\n", - "|Yes, it was an awful movie, but there was a song near the beginning of the movie, I think, called \"I got a Woody\" or ...|\n", - "| This was the most uninteresting horror flick I have seen to date|\n", - "|I don't know if this exceptionally dull movie was intended as an unofficial sequel to 'The French Connection\", but it...|\n", - "|Heart of Darkness Movie Review Could a book that is well known for its eloquent wording and complicated concepts ever...|\n", - "| A bad movie ABOUT a bad movie|\n", - "|Apart from the fact that this film was made ( I suppose it seemed a good idea at the time considering BOTTOM was so p...|\n", - "|Watching this movie, you just have to ask: What were they thinking? There are so many noticeably bad parts of this mo...|\n", - "| OK, lets start with the best|\n", - "| Anna Christie (Greta Garbo) returns to see her father Chris (George F Marion) after 15 years|\n", - "| C|\n", - "| Tom and Jerry are transporting goods via airplane to Africa|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", + "+----------------------------------------------------------------------------------------------------+\n", + "| text|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "|Doesn't anyone bother to check where this kind of sludge comes from before blathering on about it...|\n", + "|There were two things I hated about WASTED : The directing and the script . I know I`m opening my...|\n", + "|I'm rather surprised that anybody found this film touching or moving.

The basic premis...|\n", + "|Cultural Vandalism Is the new Hallmark production of Gulliver's Travels an act of cultural vandal...|\n", + "|I was at Wrestlemania VI in Toronto as a 10 year old, and the event I saw then was pretty differe...|\n", + "|This movie has been done before. It is basically a unoriginal combo of \"Napoleon Dynamite\" and \"S...|\n", + "|[ as a new resolution for this year 2005, i decide to write a comment for each movie I saw in the...|\n", + "|This movie is over hyped!! I am sad to say that I manage to watch the first 15 minutes of this mo...|\n", + "|This show had a promising start as sort of the opposite of 'Oceans 11' but has developed into a s...|\n", + "|MINOR PLOT SPOILERS AHEAD!!!

How did such talented actors get involved in such mindles...|\n", + "|There is not one character on this sitcom with any redeeming qualities. They are all self-centere...|\n", + "|Tommy Lee Jones was the best Woodroe and no one can play Woodroe F. Call better than he. Not only...|\n", + "|My wife rented this movie and then conveniently never got to see it. If I ever want to torture he...|\n", + "|This is one of those star-filled over-the-top comedies that could a) be hysterical, or b) wish th...|\n", + "|This excruciatingly boring and unfunny movie made me think that Chaplin was the real Hitler, as o...|\n", + "|you will likely be sorely disappointed by this sequel that's not a sequel.AWIL is a classic.but t...|\n", + "|If I was British, I would be embarrassed by this portrayal of incompetence. A protection agent of...|\n", + "|One of those movies in which there are no big twists whatsoever and you can predict pretty much w...|\n", + "|This show is like watching someone who is in training to someday host a show. There are some good...|\n", + "|Sigh. I'm baffled when I see a short like this get attention and assignments and whatnot. I saw t...|\n", + "+----------------------------------------------------------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] - }, - { - "data": { - "text/plain": [ - "100" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ - "# only use first N examples, since this is slow\n", - "df = spark.read.parquet(\"imdb_test\").limit(100)\n", - "df.show(truncate=120)\n", - "df.count()" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "831bc52c-a5c6-4c29-a6da-0566b5167773", - "metadata": {}, - "outputs": [], - "source": [ - "# only use first 100 rows, since generation takes a while\n", - "df1 = df.withColumn(\"input\", preprocess(col(\"lines\"), \"Translate English to German: \")).select(\"input\").limit(100).cache()" + "# Limit to N rows, since this can be slow\n", + "df = spark.read.parquet(data_path).limit(256).repartition(8)\n", + "df.show(truncate=100)" ] }, { - "cell_type": "code", - "execution_count": 21, - "id": "46dac59c-5a54-4576-91e0-279c8b375b95", + "cell_type": "markdown", + "id": "a9f8e538", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "100" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "df1.count()" + "Append a prefix to tell the model to translate English to French:" ] }, { "cell_type": "code", - "execution_count": 22, - "id": "fef1d846-5852-4762-8527-602f32c0d7cd", + "execution_count": 17, + "id": "831bc52c-a5c6-4c29-a6da-0566b5167773", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| input|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| Translate English to German: This is so overly clichéd you'll want to switch it off after the first 45 minutes|\n", - "| Translate English to German: I am a big fan of The ABC Movies of the Week genre|\n", - "|Translate English to German: In the early 1990's \"Step-by-Step\" came as a tedious combination of the ultra-cheesy \"Fu...|\n", - "|Translate English to German: When The Spirits Within was released, all you heard from Final Fantasy fans was how awfu...|\n", - "| Translate English to German: I like to think of myself as a bad movie connoisseur|\n", - "|Translate English to German: This film did well at the box office, and the producers of this mess thought the stars h...|\n", - "|Translate English to German: Following the pleasingly atmospheric original and the amusingly silly second one, this i...|\n", - "| Translate English to German: I like CKY and Viva La Bam, so I couldn't resist this when I saw it for £1|\n", - "| Translate English to German: I have read all of the reviews for this direct to video movie|\n", - "|Translate English to German: Yes, it was an awful movie, but there was a song near the beginning of the movie, I thin...|\n", - "| Translate English to German: This was the most uninteresting horror flick I have seen to date|\n", - "|Translate English to German: I don't know if this exceptionally dull movie was intended as an unofficial sequel to 'T...|\n", - "|Translate English to German: Heart of Darkness Movie Review Could a book that is well known for its eloquent wording ...|\n", - "| Translate English to German: A bad movie ABOUT a bad movie|\n", - "|Translate English to German: Apart from the fact that this film was made ( I suppose it seemed a good idea at the tim...|\n", - "|Translate English to German: Watching this movie, you just have to ask: What were they thinking? There are so many no...|\n", - "| Translate English to German: OK, lets start with the best|\n", - "|Translate English to German: Anna Christie (Greta Garbo) returns to see her father Chris (George F Marion) after 15 y...|\n", - "| Translate English to German: C|\n", - "| Translate English to German: Tom and Jerry are transporting goods via airplane to Africa|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", + "+----------------------------------------------------------------------------------------------------+\n", + "| input|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "|translate English to French: Doesn't anyone bother to check where this kind of sludge comes from ...|\n", + "|translate English to French: There were two things I hated about WASTED : The directing and the s...|\n", + "| translate English to French: I'm rather surprised that anybody found this film touching or moving|\n", + "|translate English to French: Cultural Vandalism Is the new Hallmark production of Gulliver's Trav...|\n", + "|translate English to French: I was at Wrestlemania VI in Toronto as a 10 year old, and the event ...|\n", + "| translate English to French: This movie has been done before|\n", + "|translate English to French: [ as a new resolution for this year 2005, i decide to write a commen...|\n", + "|translate English to French: This movie is over hyped!! I am sad to say that I manage to watch th...|\n", + "|translate English to French: This show had a promising start as sort of the opposite of 'Oceans 1...|\n", + "|translate English to French: MINOR PLOT SPOILERS AHEAD!!!

How did such talented actors...|\n", + "| translate English to French: There is not one character on this sitcom with any redeeming qualities|\n", + "| translate English to French: Tommy Lee Jones was the best Woodroe and no one can play Woodroe F|\n", + "| translate English to French: My wife rented this movie and then conveniently never got to see it|\n", + "|translate English to French: This is one of those star-filled over-the-top comedies that could a)...|\n", + "|translate English to French: This excruciatingly boring and unfunny movie made me think that Chap...|\n", + "|translate English to French: you will likely be sorely disappointed by this sequel that's not a s...|\n", + "|translate English to French: If I was British, I would be embarrassed by this portrayal of incomp...|\n", + "|translate English to French: One of those movies in which there are no big twists whatsoever and ...|\n", + "|translate English to French: This show is like watching someone who is in training to someday hos...|\n", + "| translate English to French: Sigh|\n", + "+----------------------------------------------------------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] } ], "source": [ - "df1.show(truncate=120)" + "input_df = df.select(preprocess(col(\"text\"), \"translate English to French: \").alias(\"input\")).cache()\n", + "input_df.show(truncate=100)" + ] + }, + { + "cell_type": "markdown", + "id": "ec53a65c", + "metadata": {}, + "source": [ + "## Inference using Spark DL API\n", + "\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "\n", + "- predict_batch_fn uses Tensorflow APIs to load the model and return a predict function which operates on numpy arrays \n", + "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 18, "id": "e7ae69d3-70c2-4765-928f-c96a7ba59829", "metadata": {}, "outputs": [], @@ -590,6 +580,7 @@ " from transformers import TFT5ForConditionalGeneration, AutoTokenizer\n", "\n", " # Enable GPU memory growth\n", + " print(\"initializing model\")\n", " gpus = tf.config.experimental.list_physical_devices('GPU')\n", " if gpus:\n", " try:\n", @@ -602,15 +593,15 @@ " tokenizer = AutoTokenizer.from_pretrained(\"google-t5/t5-small\")\n", "\n", " def predict(inputs):\n", - " flattened = np.squeeze(inputs).tolist() # convert 2d numpy array of string into flattened python list\n", - " input_ids = tokenizer(flattened, \n", - " padding=\"longest\", \n", - " max_length=512,\n", - " return_tensors=\"tf\").input_ids\n", - " output_ids = model.generate(input_ids, max_length=20)\n", - " string_outputs = np.array([tokenizer.decode(o, skip_special_tokens=True) for o in output_ids])\n", + " flattened = np.squeeze(inputs).tolist()\n", + " inputs = tokenizer(flattened, \n", + " padding=True, \n", + " return_tensors=\"tf\")\n", + " outputs = model.generate(input_ids=inputs[\"input_ids\"],\n", + " attention_mask=inputs[\"attention_mask\"],\n", + " max_length=128)\n", + " string_outputs = np.array([tokenizer.decode(o, skip_special_tokens=True) for o in outputs])\n", " print(\"predict: {}\".format(len(flattened)))\n", - "\n", " return string_outputs\n", " \n", " return predict" @@ -618,19 +609,19 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 19, "id": "36684f59-d947-43f8-a2e8-c7a423764e88", "metadata": {}, "outputs": [], "source": [ "generate = predict_batch_udf(predict_batch_fn,\n", " return_type=StringType(),\n", - " batch_size=10)" + " batch_size=32)" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 20, "id": "6a01c855-8fa1-4765-a3a5-2c9dd872df10", "metadata": {}, "outputs": [ @@ -638,15 +629,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 21:> (0 + 1) / 1]\r" + "[Stage 24:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 9.39 ms, sys: 2.14 ms, total: 11.5 ms\n", - "Wall time: 11.4 s\n" + "CPU times: user 10.9 ms, sys: 10.1 ms, total: 21 ms\n", + "Wall time: 19.3 s\n" ] }, { @@ -660,13 +651,13 @@ "source": [ "%%time\n", "# first pass caches model/fn\n", - "preds = df1.withColumn(\"preds\", generate(struct(\"input\")))\n", + "preds = input_df.withColumn(\"preds\", generate(struct(\"input\")))\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 21, "id": "d912d4b0-cd0b-44ea-859a-b23455cc2700", "metadata": {}, "outputs": [ @@ -674,15 +665,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 23:> (0 + 1) / 1]\r" + "[Stage 27:==================================================> (7 + 1) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.62 ms, sys: 4.01 ms, total: 7.64 ms\n", - "Wall time: 8.53 s\n" + "CPU times: user 7.89 ms, sys: 4.57 ms, total: 12.5 ms\n", + "Wall time: 12 s\n" ] }, { @@ -695,13 +686,13 @@ ], "source": [ "%%time\n", - "preds = df1.withColumn(\"preds\", generate(\"input\"))\n", + "preds = input_df.withColumn(\"preds\", generate(\"input\"))\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 22, "id": "5fe3d88b-30f7-468f-8db8-1f4118d0f26c", "metadata": {}, "outputs": [ @@ -709,15 +700,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 25:> (0 + 1) / 1]\r" + "[Stage 30:==================================================> (7 + 1) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 5.37 ms, sys: 2.51 ms, total: 7.88 ms\n", - "Wall time: 8.52 s\n" + "CPU times: user 6.32 ms, sys: 5.64 ms, total: 12 ms\n", + "Wall time: 12.2 s\n" ] }, { @@ -730,13 +721,13 @@ ], "source": [ "%%time\n", - "preds = df1.withColumn(\"preds\", generate(col(\"input\")))\n", + "preds = input_df.withColumn(\"preds\", generate(col(\"input\")))\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 23, "id": "4ad9b365-4b9a-438e-8fdf-47da55cb1cf4", "metadata": {}, "outputs": [ @@ -744,37 +735,37 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 27:> (0 + 1) / 1]\r" + "[Stage 33:> (0 + 1) / 1]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "| input| preds|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "|Translate English to German: This is so overly clichéd yo...| Das ist so übertrieben klischeehaft, dass Sie es nach den|\n", - "|Translate English to German: I am a big fan of The ABC Mo...| Ich bin ein großer Fan von The ABC Movies of the Week|\n", - "|Translate English to German: In the early 1990's \"Step-by...| Anfang der 1990er Jahre kam \"Step-by-Step\" als müh|\n", - "|Translate English to German: When The Spirits Within was ...|Als The Spirits Within veröffentlicht wurde, hörten Sie v...|\n", - "|Translate English to German: I like to think of myself as...| Ich halte mich gerne als schlechter Filmliebhaber|\n", - "|Translate English to German: This film did well at the bo...|Dieser Film hat sich gut an der Boxoffice ereignet, und d...|\n", - "|Translate English to German: Following the pleasingly atm...|Nach dem erfreulich stimmungsvollen Original und dem amüsant|\n", - "|Translate English to German: I like CKY and Viva La Bam, ...| Ich mag CKY und Viva La Bam, also konnte ich mich nicht|\n", - "|Translate English to German: I have read all of the revie...| Ich habe alle Rezensionen zu diesem direkten Film gelesen.|\n", - "|Translate English to German: Yes, it was an awful movie, ...| Ja, es war ein schrecklicher Film, aber es gab|\n", - "|Translate English to German: This was the most uninterest...|Dies war der größte Horrorfilm, den ich bisher gesehen habe.|\n", - "|Translate English to German: I don't know if this excepti...|Ich weiß nicht, ob dieser außergewöhnlich langweilige Fil...|\n", - "|Translate English to German: Heart of Darkness Movie Revi...|Herz der Dunkelheit Film Review Kann ein Buch, das für se...|\n", - "| Translate English to German: A bad movie ABOUT a bad movie| Ein schlechter Film ABOUT a bad movie|\n", - "|Translate English to German: Apart from the fact that thi...| Dieser Film wurde zwar fertiggestellt, aber es schien mir |\n", - "|Translate English to German: Watching this movie, you jus...|Wenn man diesen Film anschaut, muss man einfach fragen: W...|\n", - "| Translate English to German: OK, lets start with the best| OK, lets start with the best|\n", - "|Translate English to German: Anna Christie (Greta Garbo) ...| Anna Christie (Greta Garbo) kehrt nach 15 Jahren zurück,|\n", - "| Translate English to German: C| C|\n", - "|Translate English to German: Tom and Jerry are transporti...|Tom und Jerry transportieren Güter über Flugzeug nach Afrika|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "| input| preds|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "|translate English to French: Doesn't anyone bot...|Ne s'ennuie-t-il pas de vérifier où viennent ce...|\n", + "|translate English to French: There were two thi...|Il y avait deux choses que j'ai hâte de voir : ...|\n", + "|translate English to French: I'm rather surpris...|Je suis plutôt surpris que quelqu'un ait trouvé...|\n", + "|translate English to French: Cultural Vandalism...|Vandalisme culturel La nouvelle production Hall...|\n", + "|translate English to French: I was at Wrestlema...|J'étais à Wrestlemania VI à Toronto en 10 ans, ...|\n", + "|translate English to French: This movie has bee...| Ce film a été réalisé avant|\n", + "|translate English to French: [ as a new resolut...|[ en tant que nouvelle résolution pour cette an...|\n", + "|translate English to French: This movie is over...|Je suis triste de dire que je parviens à regard...|\n", + "|translate English to French: This show had a pr...|Ce spectacle a eu un début prometteur en l'espè...|\n", + "|translate English to French: MINOR PLOT SPOILER...|br />br /> Comment ces acteurs talentueux ont-i...|\n", + "|translate English to French: There is not one c...|Il n'y a pas d'un personnage sur ce sitcom ayan...|\n", + "|translate English to French: Tommy Lee Jones wa...|Tommy Lee Jones était le meilleur Woodroe et pe...|\n", + "|translate English to French: My wife rented thi...|Ma femme a loué ce film et n'a jamais pu le voi...|\n", + "|translate English to French: This is one of tho...|C’est l’une des comédies en étoiles à l’étoile ...|\n", + "|translate English to French: This excruciatingl...|Ce film excruciant ennuyant et infaillible m’a ...|\n", + "|translate English to French: you will likely be...|Vous serez probablement très déçu par cette séq...|\n", + "|translate English to French: If I was British, ...|Si j'étais britannique, je seraitis embarrassé ...|\n", + "|translate English to French: One of those movie...|Un des films dans lesquels il n'y a pas de gros...|\n", + "|translate English to French: This show is like ...|Ce spectacle ressemble à l'observation d'une pe...|\n", + "| translate English to French: Sigh| Pesée|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] @@ -788,66 +779,22 @@ } ], "source": [ - "preds.show(truncate=60)" + "preds.show(truncate=50)" ] }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 24, "id": "1eb0c83b-d91b-4f8c-a5e7-c35f55c88108", "metadata": {}, "outputs": [], "source": [ - "# only use first 100 rows, since generation takes a while\n", - "df2 = df.withColumn(\"input\", preprocess(col(\"lines\"), \"Translate English to French: \")).select(\"input\").limit(100).cache()" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "id": "054f94fd-fe79-41e7-b1c7-6124083acc72", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| input|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| Translate English to French: This is so overly clichéd you'll want to switch it off after the first 45 minutes|\n", - "| Translate English to French: I am a big fan of The ABC Movies of the Week genre|\n", - "|Translate English to French: In the early 1990's \"Step-by-Step\" came as a tedious combination of the ultra-cheesy \"Fu...|\n", - "|Translate English to French: When The Spirits Within was released, all you heard from Final Fantasy fans was how awfu...|\n", - "| Translate English to French: I like to think of myself as a bad movie connoisseur|\n", - "|Translate English to French: This film did well at the box office, and the producers of this mess thought the stars h...|\n", - "|Translate English to French: Following the pleasingly atmospheric original and the amusingly silly second one, this i...|\n", - "| Translate English to French: I like CKY and Viva La Bam, so I couldn't resist this when I saw it for £1|\n", - "| Translate English to French: I have read all of the reviews for this direct to video movie|\n", - "|Translate English to French: Yes, it was an awful movie, but there was a song near the beginning of the movie, I thin...|\n", - "| Translate English to French: This was the most uninteresting horror flick I have seen to date|\n", - "|Translate English to French: I don't know if this exceptionally dull movie was intended as an unofficial sequel to 'T...|\n", - "|Translate English to French: Heart of Darkness Movie Review Could a book that is well known for its eloquent wording ...|\n", - "| Translate English to French: A bad movie ABOUT a bad movie|\n", - "|Translate English to French: Apart from the fact that this film was made ( I suppose it seemed a good idea at the tim...|\n", - "|Translate English to French: Watching this movie, you just have to ask: What were they thinking? There are so many no...|\n", - "| Translate English to French: OK, lets start with the best|\n", - "|Translate English to French: Anna Christie (Greta Garbo) returns to see her father Chris (George F Marion) after 15 y...|\n", - "| Translate English to French: C|\n", - "| Translate English to French: Tom and Jerry are transporting goods via airplane to Africa|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "only showing top 20 rows\n", - "\n" - ] - } - ], - "source": [ - "df2.show(truncate=120)" + "input_df2 = df.select(preprocess(col(\"text\"), \"translate English to German: \").alias(\"input\")).cache()" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 25, "id": "6f6b70f9-188a-402b-9143-78a5788140e4", "metadata": {}, "outputs": [ @@ -855,15 +802,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 33:> (0 + 1) / 1]\r" + "[Stage 36:==================================================> (7 + 1) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.9 ms, sys: 5.97 ms, total: 8.87 ms\n", - "Wall time: 11.7 s\n" + "CPU times: user 4.68 ms, sys: 8.71 ms, total: 13.4 ms\n", + "Wall time: 16.1 s\n" ] }, { @@ -877,13 +824,13 @@ "source": [ "%%time\n", "# first pass caches model/fn\n", - "preds = df2.withColumn(\"preds\", generate(struct(\"input\")))\n", + "preds = input_df2.withColumn(\"preds\", generate(struct(\"input\")))\n", "result = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 26, "id": "031a6a5e-7999-4653-b394-19ed478d8c96", "metadata": {}, "outputs": [ @@ -891,15 +838,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 35:> (0 + 1) / 1]\r" + "[Stage 39:==============> (2 + 6) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 4.41 ms, sys: 1.59 ms, total: 5.99 ms\n", - "Wall time: 8.23 s\n" + "CPU times: user 5.51 ms, sys: 5.43 ms, total: 10.9 ms\n", + "Wall time: 11.6 s\n" ] }, { @@ -912,13 +859,13 @@ ], "source": [ "%%time\n", - "preds = df2.withColumn(\"preds\", generate(\"input\"))\n", + "preds = input_df2.withColumn(\"preds\", generate(\"input\"))\n", "result = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 27, "id": "229b6515-82f6-4e9c-90f0-a9c3cfb26301", "metadata": {}, "outputs": [ @@ -926,15 +873,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 37:> (0 + 1) / 1]\r" + "[Stage 42:==============> (2 + 6) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 5.46 ms, sys: 1.17 ms, total: 6.63 ms\n", - "Wall time: 8.08 s\n" + "CPU times: user 8.25 ms, sys: 3.17 ms, total: 11.4 ms\n", + "Wall time: 11.6 s\n" ] }, { @@ -947,13 +894,13 @@ ], "source": [ "%%time\n", - "preds = df2.withColumn(\"preds\", generate(col(\"input\")))\n", + "preds = input_df2.withColumn(\"preds\", generate(col(\"input\")))\n", "result = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 28, "id": "8be750ac-fa39-452e-bb4c-c2270bc2f70d", "metadata": {}, "outputs": [ @@ -961,37 +908,37 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 39:> (0 + 1) / 1]\r" + "[Stage 45:> (0 + 1) / 1]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "| input| preds|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "|Translate English to French: This is so overly clichéd yo...| Vous ne pouvez pas en tirer d'un tel cliché|\n", - "|Translate English to French: I am a big fan of The ABC Mo...| Je suis un grand fan du genre The ABC Movies of the Week|\n", - "|Translate English to French: In the early 1990's \"Step-by...| Au début des années 1990, «Step-by-Step» a été une|\n", - "|Translate English to French: When The Spirits Within was ...|Lorsque The Spirits Within a été publié, tout ce que vous...|\n", - "|Translate English to French: I like to think of myself as...| Je me considère comme un mauvais réalisateur de films|\n", - "|Translate English to French: This film did well at the bo...| Ce film a bien avancé à la salle de cinéma et|\n", - "|Translate English to French: Following the pleasingly atm...|Après l'original agréablement atmosphérique et la seconde...|\n", - "|Translate English to French: I like CKY and Viva La Bam, ...| Je m'aime CKY et Viva La Bam, |\n", - "|Translate English to French: I have read all of the revie...| J'ai lu tous les commentaires pour ce film direct à vidéo|\n", - "|Translate English to French: Yes, it was an awful movie, ...| Oui, c'était un film terrible, mais il y avait une chanson|\n", - "|Translate English to French: This was the most uninterest...| Ce fut le plus inquiétant et le plus inquiétant d'h|\n", - "|Translate English to French: I don't know if this excepti...|Je ne sais pas si ce film extrêmement tacheté était desti...|\n", - "|Translate English to French: Heart of Darkness Movie Revi...| Un livre connu pour son éloquence et ses concepts complexes|\n", - "| Translate English to French: A bad movie ABOUT a bad movie| Un mauvais film ABOUT a bad movie|\n", - "|Translate English to French: Apart from the fact that thi...|En plus du fait que ce film a été réalisé (je suppose quil s|\n", - "|Translate English to French: Watching this movie, you jus...|Vous devez simplement vous demander : « Que pense-t-il? » Il|\n", - "| Translate English to French: OK, lets start with the best| OK, s'il y a lieu de commencer par le meilleur|\n", - "|Translate English to French: Anna Christie (Greta Garbo) ...|Anna Christie (Greta Garbo) retourne pour voir son père C...|\n", - "| Translate English to French: C| C|\n", - "|Translate English to French: Tom and Jerry are transporti...| Tom et Jerry transportent des marchandises par avion en |\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "| input| preds|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "|translate English to German: Doesn't anyone bot...|Warum hat man sich nicht angeschaut, woher der ...|\n", + "|translate English to German: There were two thi...|Es gab zwei Dinge, die ich hat an WASTED gehass...|\n", + "|translate English to German: I'm rather surpris...|Ich bin ziemlich überrascht, dass jemand diesen...|\n", + "|translate English to German: Cultural Vandalism...|Kultureller Vandalismus Ist die neue Hallmark-P...|\n", + "|translate English to German: I was at Wrestlema...|Ich war als 10 Jahre alt bei Wrestlemania VI in...|\n", + "|translate English to German: This movie has bee...| Dieser Film wurde bereits vorgenommen|\n", + "|translate English to German: [ as a new resolut...|[ als neue Entschließung für dieses Jahr 2005, ...|\n", + "|translate English to German: This movie is over...|Ich hoffe, dass ich die ersten 15 Minuten diese...|\n", + "|translate English to German: This show had a pr...|Diese Show hatte einen vielversprechenden Start...|\n", + "|translate English to German: MINOR PLOT SPOILER...|br />br />Wie haben sich so talentierte Schausp...|\n", + "|translate English to German: There is not one c...|Es gibt keinen Charakter auf dieser Seite mit i...|\n", + "|translate English to German: Tommy Lee Jones wa...|Tommy Lee Jones war der beste Woodroe und niema...|\n", + "|translate English to German: My wife rented thi...|Meine Frau hat diesen Film vermietet und dann b...|\n", + "|translate English to German: This is one of tho...|Dies ist eines der Sterne-gefüllten über-the-to...|\n", + "|translate English to German: This excruciatingl...|Dieser schreckliche langweilige und unfunnelnde...|\n", + "|translate English to German: you will likely be...|Sie werden wahrscheinlich ernsthaft enttäuscht ...|\n", + "|translate English to German: If I was British, ...|Wenn ich Britisch wäre, wäre ich beschämt über ...|\n", + "|translate English to German: One of those movie...|Einer der Filme, in denen es keine großen Drehu...|\n", + "|translate English to German: This show is like ...|Diese Show ist wie ein jemanden, der in Ausbild...|\n", + "| translate English to German: Sigh| Segnen|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] @@ -1005,416 +952,253 @@ } ], "source": [ - "preds.show(truncate=60)" + "preds.show(truncate=50)" ] }, { "cell_type": "markdown", - "id": "bcabb2a8-3880-46ec-8e01-5a10f71fe83d", + "id": "f5803188", "metadata": {}, "source": [ - "### Using Triton Inference Server\n", + "## Using Triton Inference Server\n", + "In this section, we demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. \n", + "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server. \n", + "\n", + "The process looks like this:\n", + "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n", + "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n", + "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n", + "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n", "\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment. " + "\"drawing\"" ] }, { - "cell_type": "markdown", - "id": "5d98fa52-7665-49bf-865a-feec86effe23", + "cell_type": "code", + "execution_count": 29, + "id": "6d09f972", "metadata": {}, + "outputs": [], "source": [ - "This notebook uses the [Python backend with a custom execution environment](https://github.com/triton-inference-server/python_backend#creating-custom-execution-environments) with the compatible versions of Python/Numpy for Triton 24.08, using a conda-pack environment created as follows:\n", - "```\n", - "conda create -n huggingface-tf -c conda-forge python=3.10.0\n", - "conda activate huggingface-tf\n", - "\n", - "export PYTHONNOUSERSITE=True\n", - "pip install numpy==1.26.4 tensorflow[and-cuda] tf-keras transformers conda-pack \n", - "\n", - "conda-pack # huggingface-tf.tar.gz\n", - "```" + "from functools import partial" ] }, { - "cell_type": "code", - "execution_count": 35, - "id": "b858cf85-82e6-41ef-905b-d8c5d6fea492", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], + "cell_type": "markdown", + "id": "2964ffee", + "metadata": {}, "source": [ - "import os" + "Import the utility functions from pytriton_utils.py:" ] }, { "cell_type": "code", - "execution_count": 36, - "id": "05ce7c77-d562-45e8-89bb-cd656aba5a5f", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 30, + "id": "f1083dc8", + "metadata": {}, "outputs": [], "source": [ - "%%bash\n", - "# copy custom model to expected layout for Triton\n", - "rm -rf models\n", - "mkdir -p models\n", - "cp -r models_config/hf_generation_tf models\n", + "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "# add custom execution environment\n", - "cp huggingface-tf.tar.gz models" + "from pytriton_utils import (\n", + " use_stage_level_scheduling,\n", + " find_ports,\n", + " start_triton,\n", + " stop_triton\n", + ")" ] }, { "cell_type": "markdown", - "id": "a552865c-5dad-4f25-8834-f41e253ac2f6", - "metadata": { - "tags": [] - }, + "id": "066c8695", + "metadata": {}, "source": [ - "#### Start Triton Server on each executor" + "Define the Triton Server function:" ] }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 31, "id": "afd00b7e-8150-4c95-a2e4-037e9c90f92a", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "data": { - "text/plain": [ - "[True]" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], + "metadata": {}, + "outputs": [], "source": [ - "num_executors = 1\n", - "triton_models_dir = \"{}/models\".format(os.getcwd())\n", - "huggingface_cache_dir = \"{}/.cache/huggingface\".format(os.path.expanduser('~'))\n", - "nodeRDD = sc.parallelize(list(range(num_executors)), num_executors)\n", - "\n", - "def start_triton(it):\n", - " import docker\n", + "def triton_server(ports):\n", " import time\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " if containers:\n", - " print(\">>>> containers: {}\".format([c.short_id for c in containers]))\n", - " else:\n", + " import signal\n", + " import numpy as np\n", + " import tensorflow as tf\n", + " from transformers import TFT5ForConditionalGeneration, AutoTokenizer\n", + " from pytriton.decorators import batch\n", + " from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n", + " from pytriton.triton import Triton, TritonConfig\n", + " from pyspark import TaskContext\n", + "\n", + " print(f\"SERVER: Initializing Conditional Generation model on worker {TaskContext.get().partitionId()}.\")\n", + "\n", + " # Enable GPU memory growth\n", + " gpus = tf.config.experimental.list_physical_devices('GPU')\n", + " if gpus:\n", " try:\n", - " container=client.containers.run(\n", - " \"nvcr.io/nvidia/tritonserver:24.08-py3\", \"tritonserver --model-repository=/models\",\n", - " detach=True,\n", - " device_requests=[docker.types.DeviceRequest(device_ids=[\"0\"], capabilities=[['gpu']])],\n", - " environment=[\n", - " \"TRANSFORMERS_CACHE=/cache\"\n", - " ],\n", - " name=\"spark-triton\",\n", - " network_mode=\"host\",\n", - " remove=True,\n", - " shm_size=\"1G\",\n", - " volumes={\n", - " triton_models_dir: {\"bind\": \"/models\", \"mode\": \"ro\"},\n", - " huggingface_cache_dir: {\"bind\": \"/cache\", \"mode\": \"rw\"}\n", - " }\n", - " )\n", - " print(\">>>> starting triton: {}\".format(container.short_id))\n", - " except Exception as e:\n", - " print(\">>>> failed to start triton: {}\".format(e))\n", - " # wait for triton to be running\n", - " time.sleep(15)\n", - " client = grpcclient.InferenceServerClient(\"localhost:8001\")\n", - " ready = False\n", - " while not ready:\n", - " try:\n", - " ready = client.is_server_ready()\n", - " except Exception as e:\n", - " time.sleep(5)\n", + " for gpu in gpus:\n", + " tf.config.experimental.set_memory_growth(gpu, True)\n", + " except RuntimeError as e:\n", + " print(e)\n", + " \n", + " tokenizer = AutoTokenizer.from_pretrained(\"google-t5/t5-small\")\n", + " model = TFT5ForConditionalGeneration.from_pretrained(\"google-t5/t5-small\")\n", + "\n", + " @batch\n", + " def _infer_fn(**inputs):\n", + " sentences = np.squeeze(inputs[\"text\"]).tolist()\n", + " print(f\"SERVER: Received batch of size {len(sentences)}\")\n", + " decoded_sentences = [s.decode(\"utf-8\") for s in sentences]\n", + " inputs = tokenizer(decoded_sentences,\n", + " padding=True,\n", + " return_tensors=\"tf\")\n", + " output_ids = model.generate(input_ids=inputs[\"input_ids\"],\n", + " attention_mask=inputs[\"attention_mask\"],\n", + " max_length=128)\n", + " outputs = np.array([[tokenizer.decode(o, skip_special_tokens=True)] for o in output_ids])\n", + " return {\n", + " \"translations\": outputs,\n", + " }\n", "\n", - " return [True]\n", + " workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n", + " triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n", + " with Triton(config=triton_conf, workspace=workspace_path) as triton:\n", + " triton.bind(\n", + " model_name=\"ConditionalGeneration\",\n", + " infer_func=_infer_fn,\n", + " inputs=[\n", + " Tensor(name=\"text\", dtype=object, shape=(-1,)),\n", + " ],\n", + " outputs=[\n", + " Tensor(name=\"translations\", dtype=object, shape=(-1,)),\n", + " ],\n", + " config=ModelConfig(\n", + " max_batch_size=64,\n", + " batcher=DynamicBatcher(max_queue_delay_microseconds=5000), # 5ms\n", + " ),\n", + " strict=True,\n", + " )\n", "\n", - "nodeRDD.barrier().mapPartitions(start_triton).collect()" + " def _stop_triton(signum, frame):\n", + " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", + " triton.stop()\n", + "\n", + " signal.signal(signal.SIGTERM, _stop_triton)\n", + "\n", + " print(\"SERVER: Serving inference\")\n", + " triton.serve()" ] }, { "cell_type": "markdown", - "id": "528d2df6-49fc-4be7-a534-a087dfe31c84", + "id": "527da1b0", "metadata": {}, "source": [ - "#### Run inference" + "#### Start Triton servers" ] }, { - "cell_type": "code", - "execution_count": 38, - "id": "1a997c33-5202-466d-8304-b8c30f32978f", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "import pandas as pd\n", - "from functools import partial\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import col, pandas_udf, struct\n", - "from pyspark.sql.types import StringType" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "id": "9dea1875-6b95-4fc0-926d-a625a441b33d", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], + "cell_type": "markdown", + "id": "96b35b50", + "metadata": {}, "source": [ - "# only use first N examples, since this is slow\n", - "df = spark.read.parquet(\"imdb_test\").limit(100).cache()" + "**Specify the number of nodes in the cluster.** \n", + "Following the README, the example standalone cluster uses 1 node. The example Databricks/Dataproc cluster scripts use 4 nodes by default. " ] }, { "cell_type": "code", - "execution_count": 40, - "id": "5d6c54e7-534d-406f-b8e6-fd592efd0ab2", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 32, + "id": "7c4855ca", + "metadata": {}, "outputs": [], "source": [ - "# only use first sentence and add prefix for conditional generation\n", - "def preprocess(text: pd.Series, prefix: str = \"\") -> pd.Series:\n", - " @pandas_udf(\"string\")\n", - " def _preprocess(text: pd.Series) -> pd.Series:\n", - " return pd.Series([prefix + s.split(\".\")[0] for s in text])\n", - " return _preprocess(text)" + "# Change based on cluster setup\n", + "num_nodes = 1 if on_standalone else 4" ] }, { - "cell_type": "code", - "execution_count": 41, - "id": "dc1bbbe3-4232-49e5-80f6-99976524b73b", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], + "cell_type": "markdown", + "id": "52f7e397", + "metadata": {}, "source": [ - "# only use first 100 rows, since generation takes a while\n", - "df1 = df.withColumn(\"input\", preprocess(col(\"lines\"), \"Translate English to German: \")).select(\"input\").limit(100)" + "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " ] }, { "cell_type": "code", - "execution_count": 42, - "id": "5d10c61c-6102-4d19-8dd6-0c7b5b65343e", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 33, + "id": "3d522f30", + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| input|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| Translate English to German: This is so overly clichéd you'll want to switch it off after the first 45 minutes|\n", - "| Translate English to German: I am a big fan of The ABC Movies of the Week genre|\n", - "|Translate English to German: In the early 1990's \"Step-by-Step\" came as a tedious combination of the ultra-cheesy \"Fu...|\n", - "|Translate English to German: When The Spirits Within was released, all you heard from Final Fantasy fans was how awfu...|\n", - "| Translate English to German: I like to think of myself as a bad movie connoisseur|\n", - "|Translate English to German: This film did well at the box office, and the producers of this mess thought the stars h...|\n", - "|Translate English to German: Following the pleasingly atmospheric original and the amusingly silly second one, this i...|\n", - "| Translate English to German: I like CKY and Viva La Bam, so I couldn't resist this when I saw it for £1|\n", - "| Translate English to German: I have read all of the reviews for this direct to video movie|\n", - "|Translate English to German: Yes, it was an awful movie, but there was a song near the beginning of the movie, I thin...|\n", - "| Translate English to German: This was the most uninteresting horror flick I have seen to date|\n", - "|Translate English to German: I don't know if this exceptionally dull movie was intended as an unofficial sequel to 'T...|\n", - "|Translate English to German: Heart of Darkness Movie Review Could a book that is well known for its eloquent wording ...|\n", - "| Translate English to German: A bad movie ABOUT a bad movie|\n", - "|Translate English to German: Apart from the fact that this film was made ( I suppose it seemed a good idea at the tim...|\n", - "|Translate English to German: Watching this movie, you just have to ask: What were they thinking? There are so many no...|\n", - "| Translate English to German: OK, lets start with the best|\n", - "|Translate English to German: Anna Christie (Greta Garbo) returns to see her father Chris (George F Marion) after 15 y...|\n", - "| Translate English to German: C|\n", - "| Translate English to German: Tom and Jerry are transporting goods via airplane to Africa|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "only showing top 20 rows\n", - "\n" + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" ] } ], "source": [ - "df1.show(truncate=120)" + "sc = spark.sparkContext\n", + "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" ] }, { - "cell_type": "code", - "execution_count": 43, - "id": "2e0907da-a5d9-4c3b-9db4-ce5e70ca9bb4", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "def triton_fn(triton_uri, model_name):\n", - " import numpy as np\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " np_types = {\n", - " \"BOOL\": np.dtype(np.bool_),\n", - " \"INT8\": np.dtype(np.int8),\n", - " \"INT16\": np.dtype(np.int16),\n", - " \"INT32\": np.dtype(np.int32),\n", - " \"INT64\": np.dtype(np.int64),\n", - " \"FP16\": np.dtype(np.float16),\n", - " \"FP32\": np.dtype(np.float32),\n", - " \"FP64\": np.dtype(np.float64),\n", - " \"FP64\": np.dtype(np.double),\n", - " \"BYTES\": np.dtype(object)\n", - " }\n", - "\n", - " client = grpcclient.InferenceServerClient(triton_uri)\n", - " model_meta = client.get_model_metadata(model_name)\n", - " \n", - " def predict(inputs):\n", - " if isinstance(inputs, np.ndarray):\n", - " # single ndarray input\n", - " request = [grpcclient.InferInput(model_meta.inputs[0].name, inputs.shape, model_meta.inputs[0].datatype)]\n", - " request[0].set_data_from_numpy(inputs.astype(np_types[model_meta.inputs[0].datatype]))\n", - " else:\n", - " # dict of multiple ndarray inputs\n", - " request = [grpcclient.InferInput(i.name, inputs[i.name].shape, i.datatype) for i in model_meta.inputs]\n", - " for i in request:\n", - " i.set_data_from_numpy(inputs[i.name()].astype(np_types[i.datatype()]))\n", - " \n", - " response = client.infer(model_name, inputs=request)\n", - " \n", - " if len(model_meta.outputs) > 1:\n", - " # return dictionary of numpy arrays\n", - " return {o.name: response.as_numpy(o.name) for o in model_meta.outputs}\n", - " else:\n", - " # return single numpy array\n", - " return response.as_numpy(model_meta.outputs[0].name)\n", - " \n", - " return predict" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "id": "9308bdd7-6f67-484d-8b51-dd1e1b2960ba", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], + "cell_type": "markdown", + "id": "f9cc80bf", + "metadata": {}, "source": [ - "generate = predict_batch_udf(partial(triton_fn, triton_uri=\"localhost:8001\", model_name=\"hf_generation_tf\"),\n", - " return_type=StringType(),\n", - " input_tensor_shapes=[[1]],\n", - " batch_size=100)" + "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." ] }, { "cell_type": "code", - "execution_count": 45, - "id": "38484ffd-370d-492b-8ca4-9eff9f242a9f", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 34, + "id": "3487a85d", + "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 45:> (0 + 1) / 1]\r" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 5.88 ms, sys: 3.96 ms, total: 9.84 ms\n", - "Wall time: 2.66 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" + "Using ports [7000, 7001, 7002]\n" ] } ], "source": [ - "%%time\n", - "# first pass caches model/fn\n", - "preds = df1.withColumn(\"preds\", generate(struct(\"input\")))\n", - "results = preds.collect()" + "model_name = \"ConditionalGeneration\"\n", + "ports = find_ports()\n", + "assert len(ports) == 3\n", + "print(f\"Using ports {ports}\")" ] }, { "cell_type": "code", - "execution_count": 46, - "id": "ebcb6699-3ac2-4529-ab0f-fab0a5e792da", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 35, + "id": "c605ab40", + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 47:> (0 + 1) / 1]\r" + "[Stage 46:> (0 + 1) / 1]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.82 ms, sys: 1.05 ms, total: 3.87 ms\n", - "Wall time: 1.03 s\n" + "Triton Server PIDs:\n", + " {\n", + " \"cb4ae00-lcedt\": 3010304\n", + "}\n" ] }, { @@ -1426,204 +1210,147 @@ } ], "source": [ - "%%time\n", - "preds = df1.withColumn(\"preds\", generate(\"input\"))\n", - "results = preds.collect()" + "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", + " ports=ports,\n", + " model_name=model_name)).collectAsMap()\n", + "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + ] + }, + { + "cell_type": "markdown", + "id": "3f284eb3", + "metadata": {}, + "source": [ + "#### Define client function" ] }, { "cell_type": "code", - "execution_count": 47, - "id": "e2ed18ad-d00b-472c-b2c3-047932f2105d", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 49:> (0 + 1) / 1]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.55 ms, sys: 2.49 ms, total: 4.03 ms\n", - "Wall time: 967 ms\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - } - ], + "execution_count": 36, + "id": "404c5091", + "metadata": {}, + "outputs": [], "source": [ - "%%time\n", - "preds = df1.withColumn(\"preds\", generate(col(\"input\")))\n", - "results = preds.collect()" + "url = f\"http://localhost:{ports[0]}\"" ] }, { "cell_type": "code", - "execution_count": 48, - "id": "0cd64a1c-beb8-47d5-ac6f-e8525bb61176", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 51:> (0 + 1) / 1]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "| input| preds|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "|Translate English to German: This is so overly clichéd yo...| Das ist so übertrieben klischeehaft, dass Sie es nach den|\n", - "|Translate English to German: I am a big fan of The ABC Mo...| Ich bin ein großer Fan von The ABC Movies of the Week|\n", - "|Translate English to German: In the early 1990's \"Step-by...| Anfang der 1990er Jahre kam \"Step-by-Step\" als müh|\n", - "|Translate English to German: When The Spirits Within was ...|Als The Spirits Within veröffentlicht wurde, hörten Sie v...|\n", - "|Translate English to German: I like to think of myself as...| Ich halte mich gerne als schlechter Filmliebhaber|\n", - "|Translate English to German: This film did well at the bo...|Dieser Film hat sich gut an der Boxoffice ereignet, und d...|\n", - "|Translate English to German: Following the pleasingly atm...|Nach dem erfreulich stimmungsvollen Original und dem amüsant|\n", - "|Translate English to German: I like CKY and Viva La Bam, ...| Ich mag CKY und Viva La Bam, also konnte ich mich nicht|\n", - "|Translate English to German: I have read all of the revie...| Ich habe alle Rezensionen zu diesem direkten Film gelesen.|\n", - "|Translate English to German: Yes, it was an awful movie, ...| Ja, es war ein schrecklicher Film, aber es gab|\n", - "|Translate English to German: This was the most uninterest...|Dies war der größte Horrorfilm, den ich bisher gesehen habe.|\n", - "|Translate English to German: I don't know if this excepti...|Ich weiß nicht, ob dieser außergewöhnlich langweilige Fil...|\n", - "|Translate English to German: Heart of Darkness Movie Revi...|Herz der Dunkelheit Film Review Kann ein Buch, das für se...|\n", - "| Translate English to German: A bad movie ABOUT a bad movie| Ein schlechter Film ABOUT a bad movie|\n", - "|Translate English to German: Apart from the fact that thi...| Dieser Film wurde zwar fertiggestellt, aber es schien mir |\n", - "|Translate English to German: Watching this movie, you jus...|Wenn man diesen Film anschaut, muss man einfach fragen: W...|\n", - "| Translate English to German: OK, lets start with the best| OK, lets start with the best|\n", - "|Translate English to German: Anna Christie (Greta Garbo) ...| Anna Christie (Greta Garbo) kehrt nach 15 Jahren zurück,|\n", - "| Translate English to German: C| C|\n", - "|Translate English to German: Tom and Jerry are transporti...|Tom und Jerry transportieren Güter über Flugzeug nach Afrika|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "only showing top 20 rows\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - } - ], + "execution_count": 37, + "id": "aff88b3f", + "metadata": {}, + "outputs": [], "source": [ - "preds.show(truncate=60)" + "def triton_fn(url, model_name):\n", + " import numpy as np\n", + " from pytriton.client import ModelClient\n", + "\n", + " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", + "\n", + " def infer_batch(inputs):\n", + " with ModelClient(url, model_name, inference_timeout_s=240) as client:\n", + " flattened = np.squeeze(inputs).tolist() \n", + " # Encode batch\n", + " encoded_batch = [[text.encode(\"utf-8\")] for text in flattened]\n", + " encoded_batch_np = np.array(encoded_batch, dtype=np.bytes_)\n", + " # Run inference\n", + " result_data = client.infer_batch(encoded_batch_np)\n", + " result_data = np.squeeze(result_data[\"translations\"], -1)\n", + " return result_data\n", + " \n", + " return infer_batch" + ] + }, + { + "cell_type": "markdown", + "id": "a85e2ceb", + "metadata": {}, + "source": [ + "#### Load and preprocess DataFrame" ] }, { "cell_type": "code", - "execution_count": 49, - "id": "af70fed8-0f2b-4ea7-841c-476afdf9b1c0", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 38, + "id": "2fa3664e", + "metadata": {}, + "outputs": [], + "source": [ + "def preprocess(text: pd.Series, prefix: str = \"\") -> pd.Series:\n", + " @pandas_udf(\"string\")\n", + " def _preprocess(text: pd.Series) -> pd.Series:\n", + " return pd.Series([prefix + s.split(\".\")[0] for s in text])\n", + " return _preprocess(text)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "5d6c54e7-534d-406f-b8e6-fd592efd0ab2", + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.parquet(data_path).limit(256).repartition(8)" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "dc1bbbe3-4232-49e5-80f6-99976524b73b", + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "24/10/11 00:18:52 WARN CacheManager: Asked to cache already cached data.\n" + "25/01/27 20:03:03 WARN CacheManager: Asked to cache already cached data.\n" ] } ], "source": [ - "# only use first 100 rows, since generation takes a while\n", - "df2 = df.withColumn(\"input\", preprocess(col(\"lines\"), \"Translate English to French: \")).select(\"input\").limit(100).cache()" + "input_df = df.select(preprocess(col(\"text\"), \"translate English to French: \").alias(\"input\")).cache()" + ] + }, + { + "cell_type": "markdown", + "id": "e71f07d4", + "metadata": {}, + "source": [ + "#### Run Inference" ] }, { "cell_type": "code", - "execution_count": 50, - "id": "ef075e10-e22c-4236-9e0b-cb47cf2d3d06", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| input|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| Translate English to French: This is so overly clichéd you'll want to switch it off after the first 45 minutes|\n", - "| Translate English to French: I am a big fan of The ABC Movies of the Week genre|\n", - "|Translate English to French: In the early 1990's \"Step-by-Step\" came as a tedious combination of the ultra-cheesy \"Fu...|\n", - "|Translate English to French: When The Spirits Within was released, all you heard from Final Fantasy fans was how awfu...|\n", - "| Translate English to French: I like to think of myself as a bad movie connoisseur|\n", - "|Translate English to French: This film did well at the box office, and the producers of this mess thought the stars h...|\n", - "|Translate English to French: Following the pleasingly atmospheric original and the amusingly silly second one, this i...|\n", - "| Translate English to French: I like CKY and Viva La Bam, so I couldn't resist this when I saw it for £1|\n", - "| Translate English to French: I have read all of the reviews for this direct to video movie|\n", - "|Translate English to French: Yes, it was an awful movie, but there was a song near the beginning of the movie, I thin...|\n", - "| Translate English to French: This was the most uninteresting horror flick I have seen to date|\n", - "|Translate English to French: I don't know if this exceptionally dull movie was intended as an unofficial sequel to 'T...|\n", - "|Translate English to French: Heart of Darkness Movie Review Could a book that is well known for its eloquent wording ...|\n", - "| Translate English to French: A bad movie ABOUT a bad movie|\n", - "|Translate English to French: Apart from the fact that this film was made ( I suppose it seemed a good idea at the tim...|\n", - "|Translate English to French: Watching this movie, you just have to ask: What were they thinking? There are so many no...|\n", - "| Translate English to French: OK, lets start with the best|\n", - "|Translate English to French: Anna Christie (Greta Garbo) returns to see her father Chris (George F Marion) after 15 y...|\n", - "| Translate English to French: C|\n", - "| Translate English to French: Tom and Jerry are transporting goods via airplane to Africa|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "only showing top 20 rows\n", - "\n" - ] - } - ], + "execution_count": 41, + "id": "5d10c61c-6102-4d19-8dd6-0c7b5b65343e", + "metadata": {}, + "outputs": [], "source": [ - "df2.show(truncate=120)" + "generate = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", + " return_type=StringType(),\n", + " input_tensor_shapes=[[1]],\n", + " batch_size=32)" ] }, { "cell_type": "code", - "execution_count": 51, - "id": "2e7e4af8-b815-4375-b851-8368309ee8e1", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 42, + "id": "2e0907da-a5d9-4c3b-9db4-ce5e70ca9bb4", + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 55:> (0 + 1) / 1]\r" + "[Stage 50:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.91 ms, sys: 1.34 ms, total: 5.25 ms\n", - "Wall time: 1.27 s\n" + "CPU times: user 9.24 ms, sys: 8.47 ms, total: 17.7 ms\n", + "Wall time: 27.2 s\n" ] }, { @@ -1636,33 +1363,30 @@ ], "source": [ "%%time\n", - "preds = df2.withColumn(\"preds\", generate(struct(\"input\")))\n", + "# first pass caches model/fn\n", + "preds = input_df.withColumn(\"preds\", generate(struct(\"input\")))\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 52, - "id": "7b0aefb0-a96b-4791-a23c-1ce9b24eb20c", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 43, + "id": "9308bdd7-6f67-484d-8b51-dd1e1b2960ba", + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 57:> (0 + 1) / 1]\r" + "[Stage 53:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 4.31 ms, sys: 0 ns, total: 4.31 ms\n", - "Wall time: 1 s\n" + "CPU times: user 9.13 ms, sys: 4.18 ms, total: 13.3 ms\n", + "Wall time: 20.7 s\n" ] }, { @@ -1675,33 +1399,29 @@ ], "source": [ "%%time\n", - "preds = df2.withColumn(\"preds\", generate(\"input\"))\n", + "preds = input_df.withColumn(\"preds\", generate(\"input\"))\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 53, - "id": "1214b75b-a373-4579-b4c6-0cb8627da776", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 44, + "id": "38484ffd-370d-492b-8ca4-9eff9f242a9f", + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 59:> (0 + 1) / 1]\r" + "[Stage 56:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.84 ms, sys: 1.31 ms, total: 4.15 ms\n", - "Wall time: 990 ms\n" + "CPU times: user 8.88 ms, sys: 5.1 ms, total: 14 ms\n", + "Wall time: 21 s\n" ] }, { @@ -1714,55 +1434,51 @@ ], "source": [ "%%time\n", - "preds = df2.withColumn(\"preds\", generate(col(\"input\")))\n", + "preds = input_df.withColumn(\"preds\", generate(col(\"input\")))\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 54, - "id": "c9dbd21f-9e37-4221-b765-80ba8c80b884", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 45, + "id": "ebcb6699-3ac2-4529-ab0f-fab0a5e792da", + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 61:> (0 + 1) / 1]\r" + "[Stage 59:> (0 + 1) / 1]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "| input| preds|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "|Translate English to French: This is so overly clichéd yo...| Vous ne pouvez pas en tirer d'un tel cliché|\n", - "|Translate English to French: I am a big fan of The ABC Mo...| Je suis un grand fan du genre The ABC Movies of the Week|\n", - "|Translate English to French: In the early 1990's \"Step-by...| Au début des années 1990, «Step-by-Step» a été une|\n", - "|Translate English to French: When The Spirits Within was ...|Lorsque The Spirits Within a été publié, tout ce que vous...|\n", - "|Translate English to French: I like to think of myself as...| Je me considère comme un mauvais réalisateur de films|\n", - "|Translate English to French: This film did well at the bo...| Ce film a bien avancé à la salle de cinéma et|\n", - "|Translate English to French: Following the pleasingly atm...|Après l'original agréablement atmosphérique et la seconde...|\n", - "|Translate English to French: I like CKY and Viva La Bam, ...| Je m'aime CKY et Viva La Bam, |\n", - "|Translate English to French: I have read all of the revie...| J'ai lu tous les commentaires pour ce film direct à vidéo|\n", - "|Translate English to French: Yes, it was an awful movie, ...| Oui, c'était un film terrible, mais il y avait une chanson|\n", - "|Translate English to French: This was the most uninterest...| Ce fut le plus inquiétant et le plus inquiétant d'h|\n", - "|Translate English to French: I don't know if this excepti...|Je ne sais pas si ce film extrêmement tacheté était desti...|\n", - "|Translate English to French: Heart of Darkness Movie Revi...| Un livre connu pour son éloquence et ses concepts complexes|\n", - "| Translate English to French: A bad movie ABOUT a bad movie| Un mauvais film ABOUT a bad movie|\n", - "|Translate English to French: Apart from the fact that thi...|En plus du fait que ce film a été réalisé (je suppose quil s|\n", - "|Translate English to French: Watching this movie, you jus...|Vous devez simplement vous demander : « Que pense-t-il? » Il|\n", - "| Translate English to French: OK, lets start with the best| OK, s'il y a lieu de commencer par le meilleur|\n", - "|Translate English to French: Anna Christie (Greta Garbo) ...|Anna Christie (Greta Garbo) retourne pour voir son père C...|\n", - "| Translate English to French: C| C|\n", - "|Translate English to French: Tom and Jerry are transporti...| Tom et Jerry transportent des marchandises par avion en |\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "| input| preds|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "|translate English to French: Doesn't anyone bot...|Ne s'ennuie-t-il pas de vérifier où viennent ce...|\n", + "|translate English to French: There were two thi...|Il y avait deux choses que j'ai hâte de voir : ...|\n", + "|translate English to French: I'm rather surpris...|Je suis plutôt surpris que quelqu'un ait trouvé...|\n", + "|translate English to French: Cultural Vandalism...|Vandalisme culturel La nouvelle production Hall...|\n", + "|translate English to French: I was at Wrestlema...|J'étais à Wrestlemania VI à Toronto en 10 ans, ...|\n", + "|translate English to French: This movie has bee...| Ce film a été réalisé avant|\n", + "|translate English to French: [ as a new resolut...|[ en tant que nouvelle résolution pour cette an...|\n", + "|translate English to French: This movie is over...|Je suis triste de dire que je parviens à regard...|\n", + "|translate English to French: This show had a pr...|Ce spectacle a eu un début prometteur en l'espè...|\n", + "|translate English to French: MINOR PLOT SPOILER...|br />br /> Comment ces acteurs talentueux ont-i...|\n", + "|translate English to French: There is not one c...|Il n'y a pas d'un personnage sur ce sitcom ayan...|\n", + "|translate English to French: Tommy Lee Jones wa...|Tommy Lee Jones était le meilleur Woodroe et pe...|\n", + "|translate English to French: My wife rented thi...|Ma femme a loué ce film et n'a jamais pu le voi...|\n", + "|translate English to French: This is one of tho...|C’est l’une des comédies en étoiles à l’étoile ...|\n", + "|translate English to French: This excruciatingl...|Ce film excruciant ennuyant et infaillible m’a ...|\n", + "|translate English to French: you will likely be...|Vous serez probablement très déçu par cette séq...|\n", + "|translate English to French: If I was British, ...|Si j'étais britannique, je seraitis embarrassé ...|\n", + "|translate English to French: One of those movie...|Un des films dans lesquels il n'y a pas de gros...|\n", + "|translate English to French: This show is like ...|Ce spectacle ressemble à l'observation d'une pe...|\n", + "| translate English to French: Sigh| Pesée|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] @@ -1776,7 +1492,7 @@ } ], "source": [ - "preds.show(truncate=60)" + "preds.show(truncate=50)" ] }, { @@ -1786,19 +1502,22 @@ "tags": [] }, "source": [ - "#### Stop Triton Server on each executor" + "#### Shut down server on each executor" ] }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 46, "id": "425d3b28-7705-45ba-8a18-ad34fc895219", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + }, { "name": "stderr", "output_type": "stream", @@ -1812,36 +1531,26 @@ "[True]" ] }, - "execution_count": 55, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "def stop_triton(it):\n", - " import docker\n", - " import time\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " print(\">>>> stopping containers: {}\".format([c.short_id for c in containers]))\n", - " if containers:\n", - " container=containers[0]\n", - " container.stop(timeout=120)\n", - "\n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(stop_triton).collect()" + "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", + "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" ] }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 47, "id": "2dec80ca-7a7c-46a9-97c0-7afb1572f5b9", "metadata": {}, "outputs": [], "source": [ - "spark.stop()" + "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n", + " spark.stop()" ] }, { diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_torch.ipynb index 94cb7df1..fe91335e 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_torch.ipynb @@ -2,58 +2,56 @@ "cells": [ { "cell_type": "markdown", - "id": "777fc40d", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "8f6659b4-88da-4207-8d32-2674da5383a0", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ - "# PySpark Huggingface Inferencing\n", - "## Conditional generation with PyTorch\n", + "\n", + "\n", + "# PySpark DL Inference\n", + "### Conditional generation with Huggingface\n", "\n", + "In this notebook, we demonstrate distributed inference with the T5 transformer to perform sentence translation. \n", "From: https://huggingface.co/docs/transformers/model_doc/t5" ] }, { "cell_type": "code", "execution_count": 1, - "id": "c0eed0e8", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n" - ] - } - ], - "source": [ - "from transformers import T5Tokenizer, T5ForConditionalGeneration" - ] - }, - { - "cell_type": "markdown", - "id": "041ca559", - "metadata": {}, - "source": [ - "Enabling Huggingface tokenizer parallelism so that it is not automatically disabled with Python parallelism. See [this thread](https://github.com/huggingface/transformers/issues/5486) for more info. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6695a3e5", "metadata": {}, "outputs": [], "source": [ + "from transformers import T5Tokenizer, T5ForConditionalGeneration\n", + "\n", + "# Manually enable Huggingface tokenizer parallelism to avoid disabling with PySpark parallelism.\n", + "# See (https://github.com/huggingface/transformers/issues/5486) for more info. \n", "import os\n", "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"" ] }, { "cell_type": "code", - "execution_count": null, - "id": "900d6506", + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n" + ] + } + ], "source": [ "tokenizer = T5Tokenizer.from_pretrained(\"google-t5/t5-small\")\n", "model = T5ForConditionalGeneration.from_pretrained(\"google-t5/t5-small\")\n", @@ -71,24 +69,20 @@ }, { "cell_type": "code", - "execution_count": 2, - "id": "73655aea", + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "input_ids = tokenizer(input_sequences, \n", - " padding=\"longest\", \n", - " max_length=512,\n", - " truncation=True,\n", - " return_tensors=\"pt\").input_ids\n", + "inputs = tokenizer(input_sequences,\n", + " padding=True, \n", + " return_tensors=\"pt\")\n", "\n", - "outputs = model.generate(input_ids, max_length=20)" + "outputs = model.generate(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_length=128)" ] }, { "cell_type": "code", - "execution_count": 3, - "id": "90e54262", + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -99,7 +93,7 @@ " 'HuggingFace ist ein Unternehmen']" ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -109,172 +103,247 @@ ] }, { - "cell_type": "code", - "execution_count": 4, - "id": "6b11c89a", + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'pt'" - ] + "source": [ + "## PySpark" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" + "inputWidgets": {}, + "nuid": "1b8dae4a-3bfc-4430-b28a-7350db5efed4", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" } - ], + }, + "outputs": [], + "source": [ + "from pyspark.sql.types import *\n", + "from pyspark import SparkConf\n", + "from pyspark.sql import SparkSession\n", + "from pyspark.sql.functions import pandas_udf, col, struct\n", + "from pyspark.ml.functions import predict_batch_udf" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a93a1424-e483-4d37-a719-32fabee3f285", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], "source": [ - "model.framework" + "import json\n", + "import pandas as pd\n", + "import datasets\n", + "from datasets import load_dataset\n", + "datasets.disable_progress_bars()" ] }, { "cell_type": "markdown", - "id": "546eabe0", "metadata": {}, "source": [ - "## PySpark" + "Check the cluster environment to handle any platform-specific Spark configurations." ] }, { "cell_type": "code", - "execution_count": 1, - "id": "2f6db1f0-7d68-4af7-8bd6-c9fa45906c61", + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ - "import os\n", - "from pathlib import Path\n", - "from datasets import load_dataset" + "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n", + "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n", + "on_standalone = not (on_databricks or on_dataproc)" ] }, { - "cell_type": "code", - "execution_count": 2, - "id": "68121304-f1df-466e-9347-c9d2b36a9b3a", + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "from pyspark.sql.types import *\n", - "from pyspark.sql import SparkSession\n", - "from pyspark import SparkConf" + "#### Create Spark Session\n", + "\n", + "For local standalone clusters, we'll connect to the cluster and create the Spark Session. \n", + "For CSP environments, Spark will either be preconfigured (Databricks) or we'll need to create the Spark Session (Dataproc)." ] }, { "cell_type": "code", - "execution_count": 3, - "id": "6279a849", + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "24/10/10 00:10:48 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "24/10/10 00:10:48 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "25/01/27 19:35:04 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/01/27 19:35:05 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "24/10/10 00:10:48 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" + "25/01/27 19:35:05 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "25/01/27 19:35:05 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" ] } ], "source": [ - "import os\n", - "conda_env = os.environ.get(\"CONDA_PREFIX\")\n", - "\n", "conf = SparkConf()\n", + "\n", "if 'spark' not in globals():\n", - " # If Spark is not already started with Jupyter, attach to Spark Standalone\n", - " import socket\n", - " hostname = socket.gethostname()\n", - " conf.setMaster(f\"spark://{hostname}:7077\") # assuming Master is on default port 7077\n", - "conf.set(\"spark.task.maxFailures\", \"1\")\n", - "conf.set(\"spark.driver.memory\", \"8g\")\n", - "conf.set(\"spark.executor.memory\", \"8g\")\n", - "conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled\", \"false\")\n", - "conf.set(\"spark.sql.pyspark.jvmStacktrace.enabled\", \"true\")\n", - "conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", - "conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"512\")\n", - "conf.set(\"spark.python.worker.reuse\", \"true\")\n", - "# Create Spark Session\n", + " if on_standalone:\n", + " import socket\n", + " conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + " hostname = socket.gethostname()\n", + " conf.setMaster(f\"spark://{hostname}:7077\")\n", + " conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", + " conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_env}/lib:{conda_env}/lib/python3.11/site-packages/nvidia_pytriton.libs:$LD_LIBRARY_PATH\")\n", + " elif on_dataproc:\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conda_lib_path=\"/opt/conda/miniconda3/lib\"\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_lib_path}:$LD_LIBRARY_PATH\")\n", + " conf.set(\"spark.executor.instances\", \"4\") # dataproc defaults to 2\n", + "\n", + " conf.set(\"spark.executor.cores\", \"8\")\n", + " conf.set(\"spark.task.resource.gpu.amount\", \"0.125\")\n", + " conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n", + " conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", + " conf.set(\"spark.python.worker.reuse\", \"true\")\n", + "\n", + "conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"1000\")\n", "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", "sc = spark.sparkContext" ] }, { - "cell_type": "code", - "execution_count": 4, - "id": "b8453111-d068-49bb-ab91-8ae3d8bcdb7a", - "metadata": {}, - "outputs": [], + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f08c37a5-fb0c-45f6-8630-d2af67831641", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ - "# load IMDB reviews (test) dataset\n", - "data = load_dataset(\"imdb\", split=\"test\")" + "Load the IMBD Movie Reviews dataset from Huggingface." ] }, { "cell_type": "code", - "execution_count": 5, - "id": "7ad01d4a", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "25000" - ] + "execution_count": 9, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" + "inputWidgets": {}, + "nuid": "f0ec30c9-365a-43c5-9c53-3497400ee548", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" } - ], + }, + "outputs": [], "source": [ - "lines = []\n", - "for example in data:\n", - " lines.append([example[\"text\"].split(\".\")[0]])\n", - "\n", - "len(lines)" + "dataset = load_dataset(\"imdb\", split=\"test\")\n", + "dataset = dataset.to_pandas().drop(columns=\"label\")" ] }, { "cell_type": "markdown", - "id": "6fd5b472-47e8-4804-9907-772793fedb2b", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "1e4269da-d2b3-46a5-9309-38a1ba825a47", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ - "### Create PySpark DataFrame" + "#### Create PySpark DataFrame" ] }, { "cell_type": "code", - "execution_count": 6, - "id": "d24d9404-0269-476e-a9dd-1842667c915a", - "metadata": {}, + "execution_count": 10, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "30dab34d-8e4b-4f30-b7c2-3dff49da018b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [ { "data": { "text/plain": [ - "StructType([StructField('lines', StringType(), True)])" + "StructType([StructField('text', StringType(), True)])" ] }, - "execution_count": 6, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df = spark.createDataFrame(lines, ['lines']).repartition(8)\n", + "df = spark.createDataFrame(dataset).repartition(8)\n", "df.schema" ] }, { "cell_type": "code", - "execution_count": 7, - "id": "4384c762-1f79-4f60-876c-94b1f552e8fb", - "metadata": {}, + "execution_count": 11, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "55c33cc0-5dfb-449c-ae79-80972fb04405", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [ { "name": "stderr", @@ -286,89 +355,130 @@ { "data": { "text/plain": [ - "[Row(lines='(Some Spoilers) Dull as dishwater slasher flick that has this deranged homeless man Harry, Darwyn Swalve, out murdering real-estate agent all over the city of L')]" + "25000" ] }, - "execution_count": 7, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df.take(1)" - ] - }, - { - "cell_type": "markdown", - "id": "42ba3513-82dd-47e7-8193-eb4389458757", - "metadata": {}, - "source": [ - "### Save the test dataset as parquet files" + "df.count()" ] }, { "cell_type": "code", - "execution_count": 8, - "id": "e7eec8ec-4126-4890-b957-025809fad67d", - "metadata": {}, - "outputs": [], - "source": [ - "df.write.mode(\"overwrite\").parquet(\"imdb_test\")" - ] - }, - { - "cell_type": "markdown", - "id": "304e1fc8-42a3-47dd-b3c0-47efd5be1040", - "metadata": {}, + "execution_count": 12, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "efd6d6d9-1c2c-4131-8df4-a3ef75c3fc57", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/27 19:35:12 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + ] + }, + { + "data": { + "text/plain": [ + "[Row(text=\"Anyone remember the first CKY, CKY2K etc..? Back when it was about making crazy cool stuff, rather than watching Bam Margera act like a douchebag, spoiled 5 year old, super/rock-star wannabe.

The show used to be awesome, however, Bam's fame and wealth has led him to believe, that we now enjoy him acting childish and idiotic, more than actual cool stuff, that used to be in ex. CKY2K.

The acts are so repetitive, there's like nothing new, except annoying stupidity and rehearsed comments... The only things we see is Bam Margera, so busy showing us how much he doesn't care, how much money he got or whatsoever.

I really got nothing much left to say except, give us back CKY2K, cause Bam suck..

I enjoy watching Steve-o, Knoxville etc. a thousand times more.\")]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "### Check arrow memory configuration" + "df.take(1)" ] }, { "cell_type": "code", - "execution_count": 9, - "id": "20554ea5-01be-4a30-8607-db5d87786fec", - "metadata": {}, - "outputs": [], + "execution_count": 13, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "65a5b258-1634-441e-8b36-29777e54592d", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/27 19:35:12 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + ] + } + ], "source": [ - "if int(spark.conf.get(\"spark.sql.execution.arrow.maxRecordsPerBatch\")) > 512:\n", - " print(\"Decreasing `spark.sql.execution.arrow.maxRecordsPerBatch` to ensure the vectorized reader won't run out of memory\")\n", - " spark.conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"512\")\n", - "assert len(df.head()) > 0, \"`df` should not be empty\"" + "data_path = \"spark-dl-datasets/imdb_test\"\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " data_path = \"dbfs:/FileStore/\" + data_path\n", + "\n", + "df.write.mode(\"overwrite\").parquet(data_path)" ] }, { "cell_type": "markdown", - "id": "06a4ecab-c9d9-466f-ba49-902ad1fd5488", - "metadata": {}, - "source": [ - "## Inference using Spark DL API\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "e7a00479-1347-4de8-8431-faa77f8cdf4c", "metadata": { - "tags": [] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "89b909f4-5732-428b-ad61-9a6c5cf94df2", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } }, - "outputs": [], "source": [ - "import pandas as pd\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import col, pandas_udf, struct\n", - "from pyspark.sql.types import StringType" + "#### Load and preprocess DataFrame\n", + "\n", + "Define our preprocess function. We'll take the first sentence from each sample as our input for translation." ] }, { "cell_type": "code", - "execution_count": 11, - "id": "b9a0889a-35b4-493a-8197-1146fc7efd53", - "metadata": {}, + "execution_count": 14, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "eb7e53d6-bbd0-48d2-a3be-36847275e2a9", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ - "# only use first sentence and add prefix for conditional generation\n", "def preprocess(text: pd.Series, prefix: str = \"\") -> pd.Series:\n", " @pandas_udf(\"string\")\n", " def _preprocess(text: pd.Series) -> pd.Series:\n", @@ -378,160 +488,185 @@ }, { "cell_type": "code", - "execution_count": 12, - "id": "c483e4d4-9ab1-416f-a766-694e17490fd3", - "metadata": {}, + "execution_count": 15, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "97eee1a4-9dc4-43b0-9578-6d7f8ff338bd", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| lines|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| This is so overly clichéd you'll want to switch it off after the first 45 minutes|\n", - "| I am a big fan of The ABC Movies of the Week genre|\n", - "|In the early 1990's \"Step-by-Step\" came as a tedious combination of the ultra-cheesy \"Full House\" and the long-defunc...|\n", - "|When The Spirits Within was released, all you heard from Final Fantasy fans was how awful the movie was because it di...|\n", - "| I like to think of myself as a bad movie connoisseur|\n", - "|This film did well at the box office, and the producers of this mess thought the stars had such good chemistry in thi...|\n", - "|Following the pleasingly atmospheric original and the amusingly silly second one, this incredibly dull, slow, and une...|\n", - "| I like CKY and Viva La Bam, so I couldn't resist this when I saw it for £1|\n", - "| I have read all of the reviews for this direct to video movie|\n", - "|Yes, it was an awful movie, but there was a song near the beginning of the movie, I think, called \"I got a Woody\" or ...|\n", - "| This was the most uninteresting horror flick I have seen to date|\n", - "|I don't know if this exceptionally dull movie was intended as an unofficial sequel to 'The French Connection\", but it...|\n", - "|Heart of Darkness Movie Review Could a book that is well known for its eloquent wording and complicated concepts ever...|\n", - "| A bad movie ABOUT a bad movie|\n", - "|Apart from the fact that this film was made ( I suppose it seemed a good idea at the time considering BOTTOM was so p...|\n", - "|Watching this movie, you just have to ask: What were they thinking? There are so many noticeably bad parts of this mo...|\n", - "| OK, lets start with the best|\n", - "| Anna Christie (Greta Garbo) returns to see her father Chris (George F Marion) after 15 years|\n", - "| C|\n", - "| Tom and Jerry are transporting goods via airplane to Africa|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", + "+----------------------------------------------------------------------------------------------------+\n", + "| text|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "|The only reason I'm even giving this movie a 4 is because it was made in to an episode of Mystery...|\n", + "|Awkward disaster mishmash has a team of scavengers coming across the overturned S.S. Poseidon, ho...|\n", + "|Here is a fantastic concept for a film - a series of meteors crash into a small town and the resu...|\n", + "|I walked out of the cinema having suffered this film after 30 mins. I left two friends pinned in ...|\n", + "|A wildly uneven film where the major problem is the uneasy mix of comedy and thriller. To me, the...|\n", + "|Leonard Rossiter and Frances de la Tour carry this film, not without a struggle, as the script wa...|\n", + "|A good cast... A good idea but turns out it is flawed as hypnosis is not allowed as evidence in c...|\n", + "|Yet again, I appear to be the only person on planet Earth who is capable of criticizing Japanese ...|\n", + "|As a serious horror fan, I get that certain marketing ploys are used to sell movies, especially t...|\n", + "|Upon writing this review I have difficulty trying to think of what to write about. Nothing much h...|\n", + "|Simply awful. I'm including a spoiler warning here only because of including a coupla jokes from ...|\n", + "|I am a fan of Ed Harris' work and I really had high expectations about this film. Having so good ...|\n", + "|Well...I like Patricia Kaas. She is a beautiful lady and an extremely gifted and versatile singer...|\n", + "|This is a new approach to comedy. It isn't funny.

The joke is that this, in and of its...|\n", + "|It's been mentioned by others the inane dialogue in this series and I agree.

If Mom an...|\n", + "|One of the most boring movies I've ever had to sit through, it's completely formulaic. Just a coo...|\n", + "|This movie was playing on Lifetime Movie Network last month and I decided to check it out. I watc...|\n", + "|1983's \"Frightmare\" is an odd little film. The director seems to be trying to combine the atmosph...|\n", + "|'Felony' is a B-movie. No doubt about it.

Of course, if you take a look at the cast li...|\n", + "|This movie defines the word \"confused\". All the actors stay true to the script. More's the pity, ...|\n", + "+----------------------------------------------------------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] - }, - { - "data": { - "text/plain": [ - "100" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ - "# only use first N examples, since this is slow\n", - "df = spark.read.parquet(\"imdb_test\").limit(100)\n", - "df.show(truncate=120)\n", - "df.count()" + "# Limit to N rows, since this can be slow\n", + "df = spark.read.parquet(data_path).limit(512).repartition(8)\n", + "df.show(truncate=100)" ] }, { - "cell_type": "code", - "execution_count": 13, - "id": "831bc52c-a5c6-4c29-a6da-0566b5167773", + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# only use first 100 rows, since generation takes a while\n", - "df1 = df.withColumn(\"input\", preprocess(col(\"lines\"), \"Translate English to German: \")).select(\"input\").limit(100).cache()" + "Append a prefix to tell the model to translate English to French:" ] }, { "cell_type": "code", - "execution_count": 14, - "id": "46dac59c-5a54-4576-91e0-279c8b375b95", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "100" - ] + "execution_count": 16, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" + "inputWidgets": {}, + "nuid": "fa14304d-b409-4d07-99ef-9da7c7c76158", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" } - ], - "source": [ - "df1.count()" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "fef1d846-5852-4762-8527-602f32c0d7cd", - "metadata": {}, + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| input|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| Translate English to German: This is so overly clichéd you'll want to switch it off after the first 45 minutes|\n", - "| Translate English to German: I am a big fan of The ABC Movies of the Week genre|\n", - "|Translate English to German: In the early 1990's \"Step-by-Step\" came as a tedious combination of the ultra-cheesy \"Fu...|\n", - "|Translate English to German: When The Spirits Within was released, all you heard from Final Fantasy fans was how awfu...|\n", - "| Translate English to German: I like to think of myself as a bad movie connoisseur|\n", - "|Translate English to German: This film did well at the box office, and the producers of this mess thought the stars h...|\n", - "|Translate English to German: Following the pleasingly atmospheric original and the amusingly silly second one, this i...|\n", - "| Translate English to German: I like CKY and Viva La Bam, so I couldn't resist this when I saw it for £1|\n", - "| Translate English to German: I have read all of the reviews for this direct to video movie|\n", - "|Translate English to German: Yes, it was an awful movie, but there was a song near the beginning of the movie, I thin...|\n", - "| Translate English to German: This was the most uninteresting horror flick I have seen to date|\n", - "|Translate English to German: I don't know if this exceptionally dull movie was intended as an unofficial sequel to 'T...|\n", - "|Translate English to German: Heart of Darkness Movie Review Could a book that is well known for its eloquent wording ...|\n", - "| Translate English to German: A bad movie ABOUT a bad movie|\n", - "|Translate English to German: Apart from the fact that this film was made ( I suppose it seemed a good idea at the tim...|\n", - "|Translate English to German: Watching this movie, you just have to ask: What were they thinking? There are so many no...|\n", - "| Translate English to German: OK, lets start with the best|\n", - "|Translate English to German: Anna Christie (Greta Garbo) returns to see her father Chris (George F Marion) after 15 y...|\n", - "| Translate English to German: C|\n", - "| Translate English to German: Tom and Jerry are transporting goods via airplane to Africa|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", + "+----------------------------------------------------------------------------------------------------+\n", + "| input|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "|translate English to French: The only reason I'm even giving this movie a 4 is because it was mad...|\n", + "|translate English to French: Awkward disaster mishmash has a team of scavengers coming across the...|\n", + "|translate English to French: Here is a fantastic concept for a film - a series of meteors crash i...|\n", + "| translate English to French: I walked out of the cinema having suffered this film after 30 mins|\n", + "|translate English to French: A wildly uneven film where the major problem is the uneasy mix of co...|\n", + "|translate English to French: Leonard Rossiter and Frances de la Tour carry this film, not without...|\n", + "| translate English to French: A good cast|\n", + "|translate English to French: Yet again, I appear to be the only person on planet Earth who is cap...|\n", + "|translate English to French: As a serious horror fan, I get that certain marketing ploys are used...|\n", + "|translate English to French: Upon writing this review I have difficulty trying to think of what t...|\n", + "| translate English to French: Simply awful|\n", + "|translate English to French: I am a fan of Ed Harris' work and I really had high expectations abo...|\n", + "| translate English to French: Well|\n", + "| translate English to French: This is a new approach to comedy|\n", + "|translate English to French: It's been mentioned by others the inane dialogue in this series and ...|\n", + "|translate English to French: One of the most boring movies I've ever had to sit through, it's com...|\n", + "|translate English to French: This movie was playing on Lifetime Movie Network last month and I de...|\n", + "| translate English to French: 1983's \"Frightmare\" is an odd little film|\n", + "| translate English to French: 'Felony' is a B-movie|\n", + "| translate English to French: This movie defines the word \"confused\"|\n", + "+----------------------------------------------------------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] } ], "source": [ - "df1.show(truncate=120)" + "input_df = df.select(preprocess(col(\"text\"), \"translate English to French: \").alias(\"input\")).cache()\n", + "input_df.show(truncate=100)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "bc9cbdd2-1ca6-48e4-a549-792b3726525b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "source": [ + "## Inference using Spark DL API\n", + "\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "\n", + "- predict_batch_fn uses PyTorch APIs to load the model and return a predict function which operates on numpy arrays \n", + "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" ] }, { "cell_type": "code", - "execution_count": 16, - "id": "e7ae69d3-70c2-4765-928f-c96a7ba59829", - "metadata": {}, + "execution_count": 17, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "adb81177-442d-42ab-b86d-d8792201b4c8", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "def predict_batch_fn():\n", " import numpy as np\n", + " import torch\n", " from transformers import T5ForConditionalGeneration, T5Tokenizer\n", + " from pyspark import TaskContext\n", "\n", - " model = T5ForConditionalGeneration.from_pretrained(\"t5-small\")\n", + " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + " print(f\"Initializing model on worker {TaskContext.get().partitionId()}, device {device}.\")\n", + " model = T5ForConditionalGeneration.from_pretrained(\"t5-small\").to(device)\n", " tokenizer = T5Tokenizer.from_pretrained(\"t5-small\")\n", "\n", " def predict(inputs):\n", - " flattened = np.squeeze(inputs).tolist() # convert 2d numpy array of string into flattened python list\n", - " input_ids = tokenizer(flattened, \n", - " padding=\"longest\", \n", - " max_length=128,\n", - " truncation=True,\n", - " return_tensors=\"pt\").input_ids\n", - " output_ids = model.generate(input_ids, max_length=20)\n", - " string_outputs = np.array([tokenizer.decode(o, skip_special_tokens=True) for o in output_ids])\n", + " flattened = np.squeeze(inputs).tolist()\n", + " inputs = tokenizer(flattened, \n", + " padding=True,\n", + " return_tensors=\"pt\").to(device)\n", + " outputs = model.generate(input_ids=inputs[\"input_ids\"],\n", + " attention_mask=inputs[\"attention_mask\"],\n", + " max_length=128)\n", + " string_outputs = np.array([tokenizer.decode(o, skip_special_tokens=True) for o in outputs])\n", " print(\"predict: {}\".format(len(flattened)))\n", - " \n", " return string_outputs\n", " \n", " return predict" @@ -539,35 +674,57 @@ }, { "cell_type": "code", - "execution_count": 17, - "id": "36684f59-d947-43f8-a2e8-c7a423764e88", - "metadata": {}, + "execution_count": 18, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "20aab3a1-2284-4c07-9ce1-a20cf54d88f3", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [ "generate = predict_batch_udf(predict_batch_fn,\n", " return_type=StringType(),\n", - " batch_size=10)" + " batch_size=32)" ] }, { "cell_type": "code", - "execution_count": 18, - "id": "6a01c855-8fa1-4765-a3a5-2c9dd872df10", - "metadata": {}, + "execution_count": 19, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a8d6f48e-09e7-4fc7-9d2f-1b68bc2976a7", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 21:> (0 + 1) / 1]\r" + "[Stage 24:====================================> (5 + 3) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 6.58 ms, sys: 4.68 ms, total: 11.3 ms\n", - "Wall time: 7.41 s\n" + "CPU times: user 8.37 ms, sys: 1.12 ms, total: 9.48 ms\n", + "Wall time: 6.8 s\n" ] }, { @@ -581,29 +738,40 @@ "source": [ "%%time\n", "# first pass caches model/fn\n", - "preds = df1.withColumn(\"preds\", generate(struct(\"input\")))\n", + "preds = input_df.withColumn(\"preds\", generate(struct(\"input\")))\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 19, - "id": "d912d4b0-cd0b-44ea-859a-b23455cc2700", - "metadata": {}, + "execution_count": 20, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "abe2271d-0077-48f6-98b1-93524dd86447", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 23:> (0 + 1) / 1]\r" + "[Stage 27:==================================================> (7 + 1) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 1.87 ms, sys: 1.8 ms, total: 3.67 ms\n", - "Wall time: 5.71 s\n" + "CPU times: user 6.1 ms, sys: 1.19 ms, total: 7.29 ms\n", + "Wall time: 3.9 s\n" ] }, { @@ -616,29 +784,40 @@ ], "source": [ "%%time\n", - "preds = df1.withColumn(\"preds\", generate(\"input\"))\n", + "preds = input_df.withColumn(\"preds\", generate(\"input\"))\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 20, - "id": "5fe3d88b-30f7-468f-8db8-1f4118d0f26c", - "metadata": {}, + "execution_count": 21, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "77623711-a742-4262-8839-16fc3ddd1af7", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 25:> (0 + 1) / 1]\r" + "[Stage 30:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.99 ms, sys: 1.42 ms, total: 4.42 ms\n", - "Wall time: 5.69 s\n" + "CPU times: user 1.79 ms, sys: 4.62 ms, total: 6.4 ms\n", + "Wall time: 3.88 s\n" ] }, { @@ -651,140 +830,98 @@ ], "source": [ "%%time\n", - "preds = df1.withColumn(\"preds\", generate(col(\"input\")))\n", + "preds = input_df.withColumn(\"preds\", generate(col(\"input\")))\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 21, - "id": "4ad9b365-4b9a-438e-8fdf-47da55cb1cf4", - "metadata": {}, + "execution_count": 22, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "f339c654-52fd-4992-b054-188dfb260e5d", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 27:> (0 + 1) / 1]\r" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "| input| preds|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "|Translate English to German: This is so overly clichéd yo...| Das ist so übertrieben klischeehaft, dass Sie es nach den|\n", - "|Translate English to German: I am a big fan of The ABC Mo...| Ich bin ein großer Fan von The ABC Movies of the Week|\n", - "|Translate English to German: In the early 1990's \"Step-by...| Anfang der 1990er Jahre kam \"Step-by-Step\" als müh|\n", - "|Translate English to German: When The Spirits Within was ...|Als The Spirits Within veröffentlicht wurde, hörten Sie v...|\n", - "|Translate English to German: I like to think of myself as...| Ich halte mich gerne als schlechter Filmliebhaber|\n", - "|Translate English to German: This film did well at the bo...|Dieser Film hat sich gut an der Boxoffice ereignet, und d...|\n", - "|Translate English to German: Following the pleasingly atm...|Nach dem erfreulich stimmungsvollen Original und dem amüsant|\n", - "|Translate English to German: I like CKY and Viva La Bam, ...| Ich mag CKY und Viva La Bam, also konnte ich mich nicht|\n", - "|Translate English to German: I have read all of the revie...| Ich habe alle Rezensionen zu diesem direkten Film gelesen.|\n", - "|Translate English to German: Yes, it was an awful movie, ...| Ja, es war ein schrecklicher Film, aber es gab|\n", - "|Translate English to German: This was the most uninterest...|Dies war der größte Horrorfilm, den ich bisher gesehen habe.|\n", - "|Translate English to German: I don't know if this excepti...|Ich weiß nicht, ob dieser außergewöhnlich langweilige Fil...|\n", - "|Translate English to German: Heart of Darkness Movie Revi...|Herz der Dunkelheit Film Review Kann ein Buch, das für se...|\n", - "| Translate English to German: A bad movie ABOUT a bad movie| Ein schlechter Film ABOUT a bad movie|\n", - "|Translate English to German: Apart from the fact that thi...| Dieser Film wurde zwar fertiggestellt, aber es schien mir|\n", - "|Translate English to German: Watching this movie, you jus...|Wenn man diesen Film anschaut, muss man einfach fragen: W...|\n", - "| Translate English to German: OK, lets start with the best| OK, lets start with the best|\n", - "|Translate English to German: Anna Christie (Greta Garbo) ...| Anna Christie (Greta Garbo) kehrt nach 15 Jahren zurück,|\n", - "| Translate English to German: C| C|\n", - "|Translate English to German: Tom and Jerry are transporti...|Tom und Jerry transportieren Güter über Flugzeug nach Afrika|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "| input| preds|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "|translate English to French: The only reason I'...|La seule raison pour laquelle je donne même ce ...|\n", + "|translate English to French: Awkward disaster m...|La mishmash d’Awkward a eu une équipe de scaven...|\n", + "|translate English to French: Here is a fantasti...|Voici un concept fantastique pour un film : une...|\n", + "|translate English to French: I walked out of th...|Je me suis rendu du cinéma après avoir subi ce ...|\n", + "|translate English to French: A wildly uneven fi...|Un film extrêmement inégal où le problème majeu...|\n", + "|translate English to French: Leonard Rossiter a...|Leonard Rossiter et Frances de la Tour mettent ...|\n", + "| translate English to French: A good cast| Une bonne étoile|\n", + "|translate English to French: Yet again, I appea...|Encore une fois, je semble être la seule person...|\n", + "|translate English to French: As a serious horro...|En tant que grand fan d'horreur, je peux obteni...|\n", + "|translate English to French: Upon writing this ...|la suite de cette étude, j'ai de la difficulté ...|\n", + "| translate English to French: Simply awful| Tout simplement terrible|\n", + "|translate English to French: I am a fan of Ed H...|Je suis un fan de l'oeuvre d'Ed Harris et j'ai ...|\n", + "| translate English to French: Well| Eh bien|\n", + "|translate English to French: This is a new appr...| Il s’agit d’une nouvelle approche de la comédie.|\n", + "|translate English to French: It's been mentione...|Il a été mentionné par d'autres le dialogue ina...|\n", + "|translate English to French: One of the most bo...|Un des films les plus ennuyeux que je n'ai jama...|\n", + "|translate English to French: This movie was pla...|Ce film jouait sur Lifetime Movie Network le mo...|\n", + "|translate English to French: 1983's \"Frightmare...|Le film \"Frightmare\" de 1983 est un petit film ...|\n", + "|translate English to French: 'Felony' is a B-movie| 'Felony' est un mouvement B|\n", + "|translate English to French: This movie defines...| Ce film définit le mot «confus»|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] } ], "source": [ - "preds.show(truncate=60)" + "preds.show(truncate=50)" ] }, { - "cell_type": "code", - "execution_count": 22, - "id": "1eb0c83b-d91b-4f8c-a5e7-c35f55c88108", + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# only use first 100 rows, since generation takes a while\n", - "df2 = df.withColumn(\"input\", preprocess(col(\"lines\"), \"Translate English to French: \")).select(\"input\").limit(100).cache()" + "Let's try English to German:" ] }, { "cell_type": "code", "execution_count": 23, - "id": "054f94fd-fe79-41e7-b1c7-6124083acc72", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| input|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| Translate English to French: This is so overly clichéd you'll want to switch it off after the first 45 minutes|\n", - "| Translate English to French: I am a big fan of The ABC Movies of the Week genre|\n", - "|Translate English to French: In the early 1990's \"Step-by-Step\" came as a tedious combination of the ultra-cheesy \"Fu...|\n", - "|Translate English to French: When The Spirits Within was released, all you heard from Final Fantasy fans was how awfu...|\n", - "| Translate English to French: I like to think of myself as a bad movie connoisseur|\n", - "|Translate English to French: This film did well at the box office, and the producers of this mess thought the stars h...|\n", - "|Translate English to French: Following the pleasingly atmospheric original and the amusingly silly second one, this i...|\n", - "| Translate English to French: I like CKY and Viva La Bam, so I couldn't resist this when I saw it for £1|\n", - "| Translate English to French: I have read all of the reviews for this direct to video movie|\n", - "|Translate English to French: Yes, it was an awful movie, but there was a song near the beginning of the movie, I thin...|\n", - "| Translate English to French: This was the most uninteresting horror flick I have seen to date|\n", - "|Translate English to French: I don't know if this exceptionally dull movie was intended as an unofficial sequel to 'T...|\n", - "|Translate English to French: Heart of Darkness Movie Review Could a book that is well known for its eloquent wording ...|\n", - "| Translate English to French: A bad movie ABOUT a bad movie|\n", - "|Translate English to French: Apart from the fact that this film was made ( I suppose it seemed a good idea at the tim...|\n", - "|Translate English to French: Watching this movie, you just have to ask: What were they thinking? There are so many no...|\n", - "| Translate English to French: OK, lets start with the best|\n", - "|Translate English to French: Anna Christie (Greta Garbo) returns to see her father Chris (George F Marion) after 15 y...|\n", - "| Translate English to French: C|\n", - "| Translate English to French: Tom and Jerry are transporting goods via airplane to Africa|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "only showing top 20 rows\n", - "\n" - ] - } - ], + "outputs": [], "source": [ - "df2.show(truncate=120)" + "input_df2 = df.select(preprocess(col(\"text\"), \"translate English to German: \").alias(\"input\")).cache()" ] }, { "cell_type": "code", "execution_count": 24, - "id": "6f6b70f9-188a-402b-9143-78a5788140e4", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 33:> (0 + 1) / 1]\r" + "[Stage 36:==================================================> (7 + 1) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.46 ms, sys: 2.2 ms, total: 4.67 ms\n", - "Wall time: 7.38 s\n" + "CPU times: user 6.63 ms, sys: 618 μs, total: 7.25 ms\n", + "Wall time: 4.04 s\n" ] }, { @@ -798,29 +935,28 @@ "source": [ "%%time\n", "# first pass caches model/fn\n", - "preds = df2.withColumn(\"preds\", generate(struct(\"input\")))\n", + "preds = input_df2.withColumn(\"preds\", generate(struct(\"input\")))\n", "result = preds.collect()" ] }, { "cell_type": "code", "execution_count": 25, - "id": "031a6a5e-7999-4653-b394-19ed478d8c96", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 35:> (0 + 1) / 1]\r" + "[Stage 39:====================================> (5 + 3) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.34 ms, sys: 1.13 ms, total: 4.47 ms\n", - "Wall time: 6.1 s\n" + "CPU times: user 6.29 ms, sys: 0 ns, total: 6.29 ms\n", + "Wall time: 3.69 s\n" ] }, { @@ -833,29 +969,28 @@ ], "source": [ "%%time\n", - "preds = df2.withColumn(\"preds\", generate(\"input\"))\n", + "preds = input_df2.withColumn(\"preds\", generate(\"input\"))\n", "result = preds.collect()" ] }, { "cell_type": "code", "execution_count": 26, - "id": "229b6515-82f6-4e9c-90f0-a9c3cfb26301", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 37:> (0 + 1) / 1]\r" + "[Stage 42:==================================================> (7 + 1) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 1.72 ms, sys: 2.89 ms, total: 4.6 ms\n", - "Wall time: 5.93 s\n" + "CPU times: user 5.03 ms, sys: 1.58 ms, total: 6.61 ms\n", + "Wall time: 3.7 s\n" ] }, { @@ -868,472 +1003,348 @@ ], "source": [ "%%time\n", - "preds = df2.withColumn(\"preds\", generate(col(\"input\")))\n", + "preds = input_df2.withColumn(\"preds\", generate(col(\"input\")))\n", "result = preds.collect()" ] }, { "cell_type": "code", "execution_count": 27, - "id": "8be750ac-fa39-452e-bb4c-c2270bc2f70d", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 39:> (0 + 1) / 1]\r" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "| input| preds|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "|Translate English to French: This is so overly clichéd yo...| Vous ne pouvez pas en tirer d'un tel cliché|\n", - "|Translate English to French: I am a big fan of The ABC Mo...| Je suis un grand fan du genre The ABC Movies of the Week|\n", - "|Translate English to French: In the early 1990's \"Step-by...| Au début des années 1990, «Step-by-Step» a été une|\n", - "|Translate English to French: When The Spirits Within was ...|Lorsque The Spirits Within a été publié, tout ce que vous...|\n", - "|Translate English to French: I like to think of myself as...| Je me considère comme un mauvais réalisateur de films|\n", - "|Translate English to French: This film did well at the bo...| Ce film a bien avancé à la salle de cinéma et|\n", - "|Translate English to French: Following the pleasingly atm...|Après l'original agréablement atmosphérique et la seconde...|\n", - "|Translate English to French: I like CKY and Viva La Bam, ...| Je m'aime CKY et Viva La Bam,|\n", - "|Translate English to French: I have read all of the revie...| J'ai lu tous les commentaires pour ce film direct à vidéo|\n", - "|Translate English to French: Yes, it was an awful movie, ...| Oui, c'était un film terrible, mais il y avait une chanson|\n", - "|Translate English to French: This was the most uninterest...| Ce fut le plus inquiétant et le plus inquiétant d'h|\n", - "|Translate English to French: I don't know if this excepti...|Je ne sais pas si ce film extrêmement tacheté était desti...|\n", - "|Translate English to French: Heart of Darkness Movie Revi...| Un livre connu pour son éloquence et ses concepts complexes|\n", - "| Translate English to French: A bad movie ABOUT a bad movie| Un mauvais film ABOUT a bad movie|\n", - "|Translate English to French: Apart from the fact that thi...|En plus du fait que ce film a été réalisé (je suppose quil s|\n", - "|Translate English to French: Watching this movie, you jus...|Vous devez simplement vous demander : « Que pense-t-il? » Il|\n", - "| Translate English to French: OK, lets start with the best| OK, s'il y a lieu de commencer par le meilleur|\n", - "|Translate English to French: Anna Christie (Greta Garbo) ...|Anna Christie (Greta Garbo) retourne pour voir son père C...|\n", - "| Translate English to French: C| C|\n", - "|Translate English to French: Tom and Jerry are transporti...| Tom et Jerry transportent des marchandises par avion en|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "| input| preds|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "|translate English to German: The only reason I'...|Der einzige Grund, warum ich sogar diesen Film ...|\n", + "|translate English to German: Awkward disaster m...|Awkward-Katastrophenmischmash hat ein Team von ...|\n", + "|translate English to German: Here is a fantasti...|Hier ist ein fantastisches Konzept für einen Fi...|\n", + "|translate English to German: I walked out of th...|Ich ging aus dem Kino, nachdem ich diesen Film ...|\n", + "|translate English to German: A wildly uneven fi...|Ein völlig ungleicher Film, in dem das Hauptpro...|\n", + "|translate English to German: Leonard Rossiter a...|Leonard Rossiter und Frances de la Tour tragen ...|\n", + "| translate English to German: A good cast| Gutes Casting|\n", + "|translate English to German: Yet again, I appea...|Ich scheine wieder einmal die einzige Person au...|\n", + "|translate English to German: As a serious horro...|Als ernsthafter Horrorfan erhalte ich, dass bes...|\n", + "|translate English to German: Upon writing this ...|Ich habe Schwierigkeiten, mich an die Regeln zu...|\n", + "| translate English to German: Simply awful| Einfach schrecklich|\n", + "|translate English to German: I am a fan of Ed H...|Ich bin ein Fan von Ed Harris' Arbeit und hatte...|\n", + "| translate English to German: Well| Nun|\n", + "|translate English to German: This is a new appr...| Das ist ein neuer Ansatz für die Komödie|\n", + "|translate English to German: It's been mentione...|Es wurde von anderen erwähnt, die unangenehme D...|\n", + "|translate English to German: One of the most bo...|Einer der langwierigen Filme, die ich jemals du...|\n", + "|translate English to German: This movie was pla...|Dieser Film spielte im letzten Monat auf Lifeti...|\n", + "|translate English to German: 1983's \"Frightmare...| 1983 ist \"Frightmare\" ein merkwürdiger Film|\n", + "|translate English to German: 'Felony' is a B-movie| 'Felony' ist ein B-Film|\n", + "|translate English to German: This movie defines...| Dieser Film definiert das Wort \"verwirrt\"|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] } ], "source": [ - "preds.show(truncate=60)" + "preds.show(truncate=50)" ] }, { "cell_type": "markdown", - "id": "bcabb2a8-3880-46ec-8e01-5a10f71fe83d", - "metadata": {}, - "source": [ - "### Using Triton Inference Server\n", - "\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment." - ] - }, - { - "cell_type": "markdown", - "id": "5d98fa52-7665-49bf-865a-feec86effe23", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a79a6f3a-cc34-46a4-aadd-16870423fffa", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "source": [ - "This notebook uses the [Python backend with a custom execution environment](https://github.com/triton-inference-server/python_backend#creating-custom-execution-environments) with the compatible versions of Python/Numpy for Triton 24.08, using a conda-pack environment created as follows:\n", - "```\n", - "conda create -n huggingface-torch -c conda-forge python=3.10.0\n", - "conda activate huggingface-torch\n", + "## Using Triton Inference Server\n", + "In this section, we demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. \n", + "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server. \n", "\n", - "export PYTHONNOUSERSITE=True\n", - "pip install numpy==1.26.4 conda-pack sentencepiece sentence_transformers transformers\n", + "The process looks like this:\n", + "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n", + "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n", + "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n", + "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n", "\n", - "conda-pack # huggingface-torch.tar.gz\n", - "```" + "\"drawing\"" ] }, { "cell_type": "code", "execution_count": 28, - "id": "b858cf85-82e6-41ef-905b-d8c5d6fea492", "metadata": { - "tags": [ - "TRITON" - ] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "1e73757e-a451-4835-98e0-257ccf7a9025", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } }, "outputs": [], "source": [ - "import os" + "from functools import partial" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Import the utility functions from pytriton_utils.py:" ] }, { "cell_type": "code", "execution_count": 29, - "id": "05ce7c77-d562-45e8-89bb-cd656aba5a5f", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ - "%%bash\n", - "# copy custom model to expected layout for Triton\n", - "rm -rf models\n", - "mkdir -p models\n", - "cp -r models_config/hf_generation_torch models\n", + "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "# add custom execution environment\n", - "cp huggingface-torch.tar.gz models" + "from pytriton_utils import (\n", + " use_stage_level_scheduling,\n", + " find_ports,\n", + " start_triton,\n", + " stop_triton\n", + ")" ] }, { "cell_type": "markdown", - "id": "a552865c-5dad-4f25-8834-f41e253ac2f6", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ - "#### Start Triton Server on each executor" + "Define the Triton Server function:" ] }, { "cell_type": "code", "execution_count": 30, - "id": "afd00b7e-8150-4c95-a2e4-037e9c90f92a", "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "data": { - "text/plain": [ - "[True]" - ] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" + "inputWidgets": {}, + "nuid": "71b1cb49-3d8f-4eeb-937a-c0c334bd2947", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" } - ], + }, + "outputs": [], "source": [ - "num_executors = 1\n", - "triton_models_dir = \"{}/models\".format(os.getcwd())\n", - "huggingface_cache_dir = \"{}/.cache/huggingface\".format(os.path.expanduser('~'))\n", - "nodeRDD = sc.parallelize(list(range(num_executors)), num_executors)\n", - "\n", - "def start_triton(it):\n", - " import docker\n", + "def triton_server(ports):\n", " import time\n", - " import tritonclient.grpc as grpcclient\n", + " import signal\n", + " import numpy as np\n", + " import torch\n", + " from transformers import T5Tokenizer, T5ForConditionalGeneration\n", + " from pytriton.decorators import batch\n", + " from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n", + " from pytriton.triton import Triton, TritonConfig\n", + " from pyspark import TaskContext\n", + "\n", + " print(f\"SERVER: Initializing Conditional Generation model on worker {TaskContext.get().partitionId()}.\")\n", + " tokenizer = T5Tokenizer.from_pretrained(\"t5-small\")\n", + " model = T5ForConditionalGeneration.from_pretrained(\"t5-small\")\n", " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " if containers:\n", - " print(\">>>> containers: {}\".format([c.short_id for c in containers]))\n", - " else:\n", - " container=client.containers.run(\n", - " \"nvcr.io/nvidia/tritonserver:24.08-py3\", \"tritonserver --model-repository=/models\",\n", - " detach=True,\n", - " device_requests=[docker.types.DeviceRequest(device_ids=[\"0\"], capabilities=[['gpu']])],\n", - " environment=[\n", - " \"TRANSFORMERS_CACHE=/cache\"\n", + " DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + " print(f\"SERVER: Using {DEVICE} device.\")\n", + " model = model.to(DEVICE)\n", + "\n", + " @batch\n", + " def _infer_fn(**inputs):\n", + " sentences = np.squeeze(inputs[\"text\"]).tolist()\n", + " print(f\"SERVER: Received batch of size {len(sentences)}\")\n", + " decoded_sentences = [s.decode(\"utf-8\") for s in sentences]\n", + " inputs = tokenizer(decoded_sentences,\n", + " padding=True,\n", + " return_tensors=\"pt\").to(DEVICE)\n", + " output_ids = model.generate(input_ids=inputs[\"input_ids\"],\n", + " attention_mask=inputs[\"attention_mask\"],\n", + " max_length=128)\n", + " outputs = np.array([[tokenizer.decode(o, skip_special_tokens=True)] for o in output_ids])\n", + " return {\n", + " \"translations\": outputs,\n", + " }\n", + "\n", + " workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n", + " triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n", + " with Triton(config=triton_conf, workspace=workspace_path) as triton:\n", + " triton.bind(\n", + " model_name=\"ConditionalGeneration\",\n", + " infer_func=_infer_fn,\n", + " inputs=[\n", + " Tensor(name=\"text\", dtype=object, shape=(-1,)),\n", " ],\n", - " name=\"spark-triton\",\n", - " network_mode=\"host\",\n", - " remove=True,\n", - " shm_size=\"1G\",\n", - " volumes={\n", - " triton_models_dir: {\"bind\": \"/models\", \"mode\": \"ro\"},\n", - " huggingface_cache_dir: {\"bind\": \"/cache\", \"mode\": \"rw\"}\n", - " }\n", + " outputs=[\n", + " Tensor(name=\"translations\", dtype=object, shape=(-1,)),\n", + " ],\n", + " config=ModelConfig(\n", + " max_batch_size=64,\n", + " batcher=DynamicBatcher(max_queue_delay_microseconds=5000), # 5ms\n", + " ),\n", + " strict=True,\n", " )\n", - " print(\">>>> starting triton: {}\".format(container.short_id))\n", "\n", - " # wait for triton to be running\n", - " time.sleep(15)\n", - " client = grpcclient.InferenceServerClient(\"localhost:8001\")\n", - " ready = False\n", - " while not ready:\n", - " try:\n", - " ready = client.is_server_ready()\n", - " except Exception as e:\n", - " time.sleep(5)\n", + " def _stop_triton(signum, frame):\n", + " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", + " triton.stop()\n", "\n", - " return [True]\n", + " signal.signal(signal.SIGTERM, _stop_triton)\n", "\n", - "nodeRDD.barrier().mapPartitions(start_triton).collect()" + " print(\"SERVER: Serving inference\")\n", + " triton.serve()" ] }, { "cell_type": "markdown", - "id": "528d2df6-49fc-4be7-a534-a087dfe31c84", - "metadata": {}, - "source": [ - "#### Run inference" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "id": "1a997c33-5202-466d-8304-b8c30f32978f", "metadata": { - "tags": [ - "TRITON" - ] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "1bf14846-15a3-4bc8-b0c5-ce71680d3550", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } }, - "outputs": [], "source": [ - "import pandas as pd\n", - "from functools import partial\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import col, pandas_udf, struct\n", - "from pyspark.sql.types import StringType" + "#### Start Triton servers" ] }, { - "cell_type": "code", - "execution_count": 32, - "id": "9dea1875-6b95-4fc0-926d-a625a441b33d", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], + "cell_type": "markdown", + "metadata": {}, "source": [ - "# only use first N examples, since this is slow\n", - "df = spark.read.parquet(\"imdb_test\").limit(100).cache()" + "**Specify the number of nodes in the cluster.** \n", + "Following the README, the example standalone cluster uses 1 node. The example Databricks/Dataproc cluster scripts use 4 nodes by default. " ] }, { "cell_type": "code", - "execution_count": 33, - "id": "5d6c54e7-534d-406f-b8e6-fd592efd0ab2", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 31, + "metadata": {}, "outputs": [], "source": [ - "# only use first sentence and add prefix for conditional generation\n", - "def preprocess(text: pd.Series, prefix: str = \"\") -> pd.Series:\n", - " @pandas_udf(\"string\")\n", - " def _preprocess(text: pd.Series) -> pd.Series:\n", - " return pd.Series([prefix + s.split(\".\")[0] for s in text])\n", - " return _preprocess(text)" + "# Change based on cluster setup\n", + "num_nodes = 1 if on_standalone else 4" ] }, { - "cell_type": "code", - "execution_count": 34, - "id": "dc1bbbe3-4232-49e5-80f6-99976524b73b", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], + "cell_type": "markdown", + "metadata": {}, "source": [ - "# only use first 100 rows, since generation takes a while\n", - "df1 = df.withColumn(\"input\", preprocess(col(\"lines\"), \"Translate English to German: \")).select(\"input\").limit(100)" + "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " ] }, { "cell_type": "code", - "execution_count": 35, - "id": "5d10c61c-6102-4d19-8dd6-0c7b5b65343e", + "execution_count": 32, "metadata": { - "tags": [ - "TRITON" - ] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "5bf1fafc-d9c9-4fd7-901d-da97cf4ff496", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| input|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| Translate English to German: This is so overly clichéd you'll want to switch it off after the first 45 minutes|\n", - "| Translate English to German: I am a big fan of The ABC Movies of the Week genre|\n", - "|Translate English to German: In the early 1990's \"Step-by-Step\" came as a tedious combination of the ultra-cheesy \"Fu...|\n", - "|Translate English to German: When The Spirits Within was released, all you heard from Final Fantasy fans was how awfu...|\n", - "| Translate English to German: I like to think of myself as a bad movie connoisseur|\n", - "|Translate English to German: This film did well at the box office, and the producers of this mess thought the stars h...|\n", - "|Translate English to German: Following the pleasingly atmospheric original and the amusingly silly second one, this i...|\n", - "| Translate English to German: I like CKY and Viva La Bam, so I couldn't resist this when I saw it for £1|\n", - "| Translate English to German: I have read all of the reviews for this direct to video movie|\n", - "|Translate English to German: Yes, it was an awful movie, but there was a song near the beginning of the movie, I thin...|\n", - "| Translate English to German: This was the most uninteresting horror flick I have seen to date|\n", - "|Translate English to German: I don't know if this exceptionally dull movie was intended as an unofficial sequel to 'T...|\n", - "|Translate English to German: Heart of Darkness Movie Review Could a book that is well known for its eloquent wording ...|\n", - "| Translate English to German: A bad movie ABOUT a bad movie|\n", - "|Translate English to German: Apart from the fact that this film was made ( I suppose it seemed a good idea at the tim...|\n", - "|Translate English to German: Watching this movie, you just have to ask: What were they thinking? There are so many no...|\n", - "| Translate English to German: OK, lets start with the best|\n", - "|Translate English to German: Anna Christie (Greta Garbo) returns to see her father Chris (George F Marion) after 15 y...|\n", - "| Translate English to German: C|\n", - "| Translate English to German: Tom and Jerry are transporting goods via airplane to Africa|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "only showing top 20 rows\n", - "\n" + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" ] } ], "source": [ - "df1.show(truncate=120)" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "id": "2e0907da-a5d9-4c3b-9db4-ce5e70ca9bb4", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "def triton_fn(triton_uri, model_name):\n", - " import numpy as np\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " np_types = {\n", - " \"BOOL\": np.dtype(np.bool8),\n", - " \"INT8\": np.dtype(np.int8),\n", - " \"INT16\": np.dtype(np.int16),\n", - " \"INT32\": np.dtype(np.int32),\n", - " \"INT64\": np.dtype(np.int64),\n", - " \"FP16\": np.dtype(np.float16),\n", - " \"FP32\": np.dtype(np.float32),\n", - " \"FP64\": np.dtype(np.float64),\n", - " \"FP64\": np.dtype(np.double),\n", - " \"BYTES\": np.dtype(object)\n", - " }\n", - "\n", - " client = grpcclient.InferenceServerClient(triton_uri)\n", - " model_meta = client.get_model_metadata(model_name)\n", - " \n", - " def predict(inputs):\n", - " if isinstance(inputs, np.ndarray):\n", - " # single ndarray input\n", - " request = [grpcclient.InferInput(model_meta.inputs[0].name, inputs.shape, model_meta.inputs[0].datatype)]\n", - " request[0].set_data_from_numpy(inputs.astype(np_types[model_meta.inputs[0].datatype]))\n", - " else:\n", - " # dict of multiple ndarray inputs\n", - " request = [grpcclient.InferInput(i.name, inputs[i.name].shape, i.datatype) for i in model_meta.inputs]\n", - " for i in request:\n", - " i.set_data_from_numpy(inputs[i.name()].astype(np_types[i.datatype()]))\n", - " \n", - " response = client.infer(model_name, inputs=request)\n", - " \n", - " if len(model_meta.outputs) > 1:\n", - " # return dictionary of numpy arrays\n", - " return {o.name: response.as_numpy(o.name) for o in model_meta.outputs}\n", - " else:\n", - " # return single numpy array\n", - " return response.as_numpy(model_meta.outputs[0].name)\n", - " \n", - " return predict" + "sc = spark.sparkContext\n", + "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" ] }, { - "cell_type": "code", - "execution_count": 37, - "id": "9308bdd7-6f67-484d-8b51-dd1e1b2960ba", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], + "cell_type": "markdown", + "metadata": {}, "source": [ - "generate = predict_batch_udf(partial(triton_fn, triton_uri=\"localhost:8001\", model_name=\"hf_generation_torch\"),\n", - " return_type=StringType(),\n", - " input_tensor_shapes=[[1]],\n", - " batch_size=100)" + "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." ] }, { "cell_type": "code", - "execution_count": 38, - "id": "38484ffd-370d-492b-8ca4-9eff9f242a9f", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 33, + "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 45:> (0 + 1) / 1]\r" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 4.61 ms, sys: 1.26 ms, total: 5.87 ms\n", - "Wall time: 2.04 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" + "Using ports [7000, 7001, 7002]\n" ] } ], "source": [ - "%%time\n", - "# first pass caches model/fn\n", - "preds = df1.withColumn(\"preds\", generate(struct(\"input\")))\n", - "results = preds.collect()" + "model_name = \"ConditionalGeneration\"\n", + "ports = find_ports()\n", + "assert len(ports) == 3\n", + "print(f\"Using ports {ports}\")" ] }, { "cell_type": "code", - "execution_count": 39, - "id": "ebcb6699-3ac2-4529-ab0f-fab0a5e792da", + "execution_count": null, "metadata": { - "tags": [ - "TRITON" - ] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "289b08fa-7916-44ea-8fe5-28821451db6b", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 47:> (0 + 1) / 1]\r" + "[Stage 46:> (0 + 1) / 1]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.16 ms, sys: 641 μs, total: 3.8 ms\n", - "Wall time: 1.58 s\n" + "Triton Server PIDs:\n", + " {\n", + " \"cb4ae00-lcedt\": 2950716\n", + "}\n" ] }, { @@ -1345,190 +1356,221 @@ } ], "source": [ - "%%time\n", - "preds = df1.withColumn(\"preds\", generate(\"input\"))\n", - "results = preds.collect()" + "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", + " ports=ports,\n", + " model_name=model_name)).collectAsMap()\n", + "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Define client function" ] }, { "cell_type": "code", - "execution_count": 40, - "id": "e2ed18ad-d00b-472c-b2c3-047932f2105d", + "execution_count": 35, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{ports[0]}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 36, "metadata": { - "tags": [ - "TRITON" - ] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "e203eb19-166d-4177-aa87-fd31b7e3c90e", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 49:> (0 + 1) / 1]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 1.91 ms, sys: 2.38 ms, total: 4.29 ms\n", - "Wall time: 1.75 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] + "outputs": [], + "source": [ + "def triton_fn(url, model_name):\n", + " import numpy as np\n", + " from pytriton.client import ModelClient\n", + "\n", + " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", + "\n", + " def infer_batch(inputs):\n", + " with ModelClient(url, model_name, inference_timeout_s=240) as client:\n", + " flattened = np.squeeze(inputs).tolist() \n", + " # Encode batch\n", + " encoded_batch = [[text.encode(\"utf-8\")] for text in flattened]\n", + " encoded_batch_np = np.array(encoded_batch, dtype=np.bytes_)\n", + " # Run inference\n", + " result_data = client.infer_batch(encoded_batch_np)\n", + " result_data = np.squeeze(result_data[\"translations\"], -1)\n", + " return result_data\n", + " \n", + " return infer_batch" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "1b6b2a05-aea4-4e4d-a87d-0a6bd5ab554c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" } - ], + }, "source": [ - "%%time\n", - "preds = df1.withColumn(\"preds\", generate(col(\"input\")))\n", - "results = preds.collect()" + "#### Load and preprocess DataFrame" ] }, { "cell_type": "code", - "execution_count": 41, - "id": "0cd64a1c-beb8-47d5-ac6f-e8525bb61176", + "execution_count": 37, "metadata": { - "tags": [ - "TRITON" - ] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "a5e83230-5178-4fec-bba2-0e69be40e68c", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "| input| preds|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "|Translate English to German: This is so overly clichéd yo...| Das ist so übertrieben klischeehaft, dass Sie es nach den|\n", - "|Translate English to German: I am a big fan of The ABC Mo...| Ich bin ein großer Fan von The ABC Movies of the Week|\n", - "|Translate English to German: In the early 1990's \"Step-by...| Anfang der 1990er Jahre kam \"Step-by-Step\" als müh|\n", - "|Translate English to German: When The Spirits Within was ...|Als The Spirits Within veröffentlicht wurde, hörten Sie v...|\n", - "|Translate English to German: I like to think of myself as...| Ich halte mich gerne als schlechter Filmliebhaber|\n", - "|Translate English to German: This film did well at the bo...|Dieser Film hat sich gut an der Boxoffice ereignet, und d...|\n", - "|Translate English to German: Following the pleasingly atm...|Nach dem erfreulich stimmungsvollen Original und dem amüsant|\n", - "|Translate English to German: I like CKY and Viva La Bam, ...| Ich mag CKY und Viva La Bam, also konnte ich mich nicht|\n", - "|Translate English to German: I have read all of the revie...| Ich habe alle Rezensionen zu diesem direkten Film gelesen.|\n", - "|Translate English to German: Yes, it was an awful movie, ...| Ja, es war ein schrecklicher Film, aber es gab|\n", - "|Translate English to German: This was the most uninterest...|Dies war der größte Horrorfilm, den ich bisher gesehen habe.|\n", - "|Translate English to German: I don't know if this excepti...|Ich weiß nicht, ob dieser außergewöhnlich langweilige Fil...|\n", - "|Translate English to German: Heart of Darkness Movie Revi...|Herz der Dunkelheit Film Review Kann ein Buch, das für se...|\n", - "| Translate English to German: A bad movie ABOUT a bad movie| Ein schlechter Film ABOUT a bad movie|\n", - "|Translate English to German: Apart from the fact that thi...| Dieser Film wurde zwar fertiggestellt, aber es schien mir|\n", - "|Translate English to German: Watching this movie, you jus...|Wenn man diesen Film anschaut, muss man einfach fragen: W...|\n", - "| Translate English to German: OK, lets start with the best| OK, lets start with the best|\n", - "|Translate English to German: Anna Christie (Greta Garbo) ...| Anna Christie (Greta Garbo) kehrt nach 15 Jahren zurück,|\n", - "| Translate English to German: C| C|\n", - "|Translate English to German: Tom and Jerry are transporti...|Tom und Jerry transportieren Güter über Flugzeug nach Afrika|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "only showing top 20 rows\n", - "\n" - ] + "outputs": [], + "source": [ + "def preprocess(text: pd.Series, prefix: str = \"\") -> pd.Series:\n", + " @pandas_udf(\"string\")\n", + " def _preprocess(text: pd.Series) -> pd.Series:\n", + " return pd.Series([prefix + s.split(\".\")[0] for s in text])\n", + " return _preprocess(text)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "aad299b0-34bb-4edb-b1e4-cd0c82bb7455", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" } - ], + }, + "outputs": [], "source": [ - "preds.show(truncate=60)" + "df = spark.read.parquet(data_path).limit(512).repartition(8)" ] }, { "cell_type": "code", - "execution_count": 42, - "id": "af70fed8-0f2b-4ea7-841c-476afdf9b1c0", + "execution_count": 39, "metadata": { - "tags": [ - "TRITON" - ] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "7934a6fc-57bc-4104-a52c-076351e77cbe", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "24/10/10 00:12:21 WARN CacheManager: Asked to cache already cached data.\n" + "25/01/27 19:35:45 WARN CacheManager: Asked to cache already cached data.\n" ] } ], "source": [ - "# only use first 100 rows, since generation takes a while\n", - "df2 = df.withColumn(\"input\", preprocess(col(\"lines\"), \"Translate English to French: \")).select(\"input\").limit(100).cache()" + "input_df = df.select(preprocess(col(\"text\"), \"translate English to French: \").alias(\"input\")).cache()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Run Inference" ] }, { "cell_type": "code", - "execution_count": 43, - "id": "ef075e10-e22c-4236-9e0b-cb47cf2d3d06", + "execution_count": 40, "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| input|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| Translate English to French: This is so overly clichéd you'll want to switch it off after the first 45 minutes|\n", - "| Translate English to French: I am a big fan of The ABC Movies of the Week genre|\n", - "|Translate English to French: In the early 1990's \"Step-by-Step\" came as a tedious combination of the ultra-cheesy \"Fu...|\n", - "|Translate English to French: When The Spirits Within was released, all you heard from Final Fantasy fans was how awfu...|\n", - "| Translate English to French: I like to think of myself as a bad movie connoisseur|\n", - "|Translate English to French: This film did well at the box office, and the producers of this mess thought the stars h...|\n", - "|Translate English to French: Following the pleasingly atmospheric original and the amusingly silly second one, this i...|\n", - "| Translate English to French: I like CKY and Viva La Bam, so I couldn't resist this when I saw it for £1|\n", - "| Translate English to French: I have read all of the reviews for this direct to video movie|\n", - "|Translate English to French: Yes, it was an awful movie, but there was a song near the beginning of the movie, I thin...|\n", - "| Translate English to French: This was the most uninteresting horror flick I have seen to date|\n", - "|Translate English to French: I don't know if this exceptionally dull movie was intended as an unofficial sequel to 'T...|\n", - "|Translate English to French: Heart of Darkness Movie Review Could a book that is well known for its eloquent wording ...|\n", - "| Translate English to French: A bad movie ABOUT a bad movie|\n", - "|Translate English to French: Apart from the fact that this film was made ( I suppose it seemed a good idea at the tim...|\n", - "|Translate English to French: Watching this movie, you just have to ask: What were they thinking? There are so many no...|\n", - "| Translate English to French: OK, lets start with the best|\n", - "|Translate English to French: Anna Christie (Greta Garbo) returns to see her father Chris (George F Marion) after 15 y...|\n", - "| Translate English to French: C|\n", - "| Translate English to French: Tom and Jerry are transporting goods via airplane to Africa|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "only showing top 20 rows\n", - "\n" - ] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "be692f4a-cf86-4cf4-9530-7c62e479cacd", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" } - ], + }, + "outputs": [], "source": [ - "df2.show(truncate=120)" + "generate = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", + " return_type=StringType(),\n", + " input_tensor_shapes=[[1]],\n", + " batch_size=32)" ] }, { "cell_type": "code", - "execution_count": 44, - "id": "2e7e4af8-b815-4375-b851-8368309ee8e1", + "execution_count": 41, "metadata": { - "tags": [ - "TRITON" - ] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "0f6229ef-01c8-43c9-a259-c5df6a18d689", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 55:> (0 + 1) / 1]\r" + "[Stage 50:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.4 ms, sys: 2.75 ms, total: 6.14 ms\n", - "Wall time: 1.96 s\n" + "CPU times: user 8.21 ms, sys: 3.06 ms, total: 11.3 ms\n", + "Wall time: 4.43 s\n" ] }, { @@ -1541,33 +1583,41 @@ ], "source": [ "%%time\n", - "preds = df2.withColumn(\"preds\", generate(struct(\"input\")))\n", + "# first pass caches model/fn\n", + "preds = input_df.withColumn(\"preds\", generate(struct(\"input\")))\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 45, - "id": "7b0aefb0-a96b-4791-a23c-1ce9b24eb20c", + "execution_count": 42, "metadata": { - "tags": [ - "TRITON" - ] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "5a543b4c-8b29-4f61-9773-2639bbc7f728", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 57:> (0 + 1) / 1]\r" + "[Stage 53:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.76 ms, sys: 897 μs, total: 4.66 ms\n", - "Wall time: 1.61 s\n" + "CPU times: user 6.47 ms, sys: 910 μs, total: 7.38 ms\n", + "Wall time: 4.24 s\n" ] }, { @@ -1580,33 +1630,40 @@ ], "source": [ "%%time\n", - "preds = df2.withColumn(\"preds\", generate(\"input\"))\n", + "preds = input_df.withColumn(\"preds\", generate(\"input\"))\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 46, - "id": "1214b75b-a373-4579-b4c6-0cb8627da776", + "execution_count": 43, "metadata": { - "tags": [ - "TRITON" - ] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "4c0cfc4e-ef0a-435e-9fdf-72b72b6def93", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 59:> (0 + 1) / 1]\r" + "[Stage 56:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.61 ms, sys: 2.26 ms, total: 4.87 ms\n", - "Wall time: 1.67 s\n" + "CPU times: user 5.84 ms, sys: 1.27 ms, total: 7.11 ms\n", + "Wall time: 4.29 s\n" ] }, { @@ -1619,77 +1676,107 @@ ], "source": [ "%%time\n", - "preds = df2.withColumn(\"preds\", generate(col(\"input\")))\n", + "preds = input_df.withColumn(\"preds\", generate(col(\"input\")))\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 47, - "id": "c9dbd21f-9e37-4221-b765-80ba8c80b884", + "execution_count": 44, "metadata": { - "tags": [ - "TRITON" - ] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "2d756e2e-8b60-43cb-b5f9-e27de11be24d", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "| input| preds|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "|Translate English to French: This is so overly clichéd yo...| Vous ne pouvez pas en tirer d'un tel cliché|\n", - "|Translate English to French: I am a big fan of The ABC Mo...| Je suis un grand fan du genre The ABC Movies of the Week|\n", - "|Translate English to French: In the early 1990's \"Step-by...| Au début des années 1990, «Step-by-Step» a été une|\n", - "|Translate English to French: When The Spirits Within was ...|Lorsque The Spirits Within a été publié, tout ce que vous...|\n", - "|Translate English to French: I like to think of myself as...| Je me considère comme un mauvais réalisateur de films|\n", - "|Translate English to French: This film did well at the bo...| Ce film a bien avancé à la salle de cinéma et|\n", - "|Translate English to French: Following the pleasingly atm...|Après l'original agréablement atmosphérique et la seconde...|\n", - "|Translate English to French: I like CKY and Viva La Bam, ...| Je m'aime CKY et Viva La Bam,|\n", - "|Translate English to French: I have read all of the revie...| J'ai lu tous les commentaires pour ce film direct à vidéo|\n", - "|Translate English to French: Yes, it was an awful movie, ...| Oui, c'était un film terrible, mais il y avait une chanson|\n", - "|Translate English to French: This was the most uninterest...| Ce fut le plus inquiétant et le plus inquiétant d'h|\n", - "|Translate English to French: I don't know if this excepti...|Je ne sais pas si ce film extrêmement tacheté était desti...|\n", - "|Translate English to French: Heart of Darkness Movie Revi...| Un livre connu pour son éloquence et ses concepts complexes|\n", - "| Translate English to French: A bad movie ABOUT a bad movie| Un mauvais film ABOUT a bad movie|\n", - "|Translate English to French: Apart from the fact that thi...|En plus du fait que ce film a été réalisé (je suppose quil s|\n", - "|Translate English to French: Watching this movie, you jus...|Vous devez simplement vous demander : « Que pense-t-il? » Il|\n", - "| Translate English to French: OK, lets start with the best| OK, s'il y a lieu de commencer par le meilleur|\n", - "|Translate English to French: Anna Christie (Greta Garbo) ...|Anna Christie (Greta Garbo) retourne pour voir son père C...|\n", - "| Translate English to French: C| C|\n", - "|Translate English to French: Tom and Jerry are transporti...| Tom et Jerry transportent des marchandises par avion en|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "| input| preds|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "|translate English to French: The only reason I'...|La seule raison pour laquelle je donne même ce ...|\n", + "|translate English to French: Awkward disaster m...|La mishmash d’Awkward a eu une équipe de scaven...|\n", + "|translate English to French: Here is a fantasti...|Voici un concept fantastique pour un film : une...|\n", + "|translate English to French: I walked out of th...|Je me suis rendu du cinéma après avoir subi ce ...|\n", + "|translate English to French: A wildly uneven fi...|Un film extrêmement inégal où le problème majeu...|\n", + "|translate English to French: Leonard Rossiter a...|Leonard Rossiter et Frances de la Tour mettent ...|\n", + "| translate English to French: A good cast| Une bonne étoile|\n", + "|translate English to French: Yet again, I appea...|Encore une fois, je semble être la seule person...|\n", + "|translate English to French: As a serious horro...|En tant que grand fan d'horreur, je peux obteni...|\n", + "|translate English to French: Upon writing this ...|la suite de cette étude, j'ai de la difficulté ...|\n", + "| translate English to French: Simply awful| Tout simplement terrible|\n", + "|translate English to French: I am a fan of Ed H...|Je suis un fan de l'oeuvre d'Ed Harris et j'ai ...|\n", + "| translate English to French: Well| Eh bien|\n", + "|translate English to French: This is a new appr...| Il s’agit d’une nouvelle approche de la comédie.|\n", + "|translate English to French: It's been mentione...|Il a été mentionné par d'autres le dialogue ina...|\n", + "|translate English to French: One of the most bo...|Un des films les plus ennuyeux que je n'ai jama...|\n", + "|translate English to French: This movie was pla...|Ce film jouait sur Lifetime Movie Network le mo...|\n", + "|translate English to French: 1983's \"Frightmare...|Le film \"Frightmare\" de 1983 est un petit film ...|\n", + "|translate English to French: 'Felony' is a B-movie| 'Felony' est un mouvement B|\n", + "|translate English to French: This movie defines...| Ce film définit le mot «confus»|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] } ], "source": [ - "preds.show(truncate=60)" + "preds.show(truncate=50)" ] }, { "cell_type": "markdown", - "id": "919e3113-64dd-482a-9233-6607b3f63c1e", "metadata": { - "tags": [] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "86ae68d4-57da-41d9-91b4-625ef9465d60", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } }, "source": [ - "#### Stop Triton Server on each executor" + "#### Shut down servers on each executor" ] }, { "cell_type": "code", - "execution_count": 48, - "id": "425d3b28-7705-45ba-8a18-ad34fc895219", + "execution_count": 45, "metadata": { - "tags": [ - "TRITON" - ] + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "16fd4601-f6d5-4ddf-9b5e-d918ab0adf3a", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } }, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + }, { "name": "stderr", "output_type": "stream", @@ -1703,48 +1790,64 @@ "[True]" ] }, - "execution_count": 48, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "def stop_triton(it):\n", - " import docker\n", - " import time\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " print(\">>>> stopping containers: {}\".format([c.short_id for c in containers]))\n", - " if containers:\n", - " container=containers[0]\n", - " container.stop(timeout=120)\n", - "\n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(stop_triton).collect()" + "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", + "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" ] }, { "cell_type": "code", - "execution_count": 49, - "id": "2dec80ca-7a7c-46a9-97c0-7afb1572f5b9", + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ - "spark.stop()" + "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n", + " spark.stop()" ] }, { "cell_type": "code", "execution_count": null, - "id": "f43118ab-fc0a-4f64-a126-4302e615654a", - "metadata": {}, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "008c3e50-d321-4431-a9ab-919b35d1b042", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, "outputs": [], "source": [] } ], "metadata": { + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "environmentMetadata": null, + "language": "python", + "notebookMetadata": { + "mostRecentlyExecutedCommandWithImplicitDF": { + "commandId": 421988607303514, + "dataframes": [ + "_sqldf" + ] + }, + "pythonIndentUnit": 4 + }, + "notebookName": "spark-triton-db.ipynb", + "widgets": {} + }, "kernelspec": { "display_name": "spark-dl-torch", "language": "python", @@ -1764,5 +1867,5 @@ } }, "nbformat": 4, - "nbformat_minor": 5 + "nbformat_minor": 4 } diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_tf/1/model.py b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_tf/1/model.py deleted file mode 100644 index b788c893..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_tf/1/model.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import numpy as np -import json - -# triton_python_backend_utils is available in every Triton Python model. You -# need to use this module to create inference requests and responses. It also -# contains some utility functions for extracting information from model_config -# and converting Triton input/output types to numpy types. -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """Your Python model must use the same class name. Every Python model - that is created must have "TritonPythonModel" as the class name. - """ - - def initialize(self, args): - """`initialize` is called only once when the model is being loaded. - Implementing `initialize` function is optional. This function allows - the model to intialize any state associated with this model. - - Parameters - ---------- - args : dict - Both keys and values are strings. The dictionary keys and values are: - * model_config: A JSON string containing the model configuration - * model_instance_kind: A string containing model instance kind - * model_instance_device_id: A string containing model instance device ID - * model_repository: Model repository path - * model_version: Model version - * model_name: Model name - """ - import tensorflow as tf - # Enable GPU memory growth - gpus = tf.config.experimental.list_physical_devices('GPU') - if gpus: - try: - for gpu in gpus: - tf.config.experimental.set_memory_growth(gpu, True) - except RuntimeError as e: - print(e) - - print(tf.__version__) - - from transformers import AutoTokenizer, TFT5ForConditionalGeneration - - self.tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small") - self.model = TFT5ForConditionalGeneration.from_pretrained("google-t5/t5-small") - - # You must parse model_config. JSON string is not parsed here - self.model_config = model_config = json.loads(args['model_config']) - - # Get output configuration - output_config = pb_utils.get_output_config_by_name(model_config, "output") - - # Convert Triton types to numpy types - self.output_dtype = pb_utils.triton_string_to_numpy(output_config['data_type']) - - def execute(self, requests): - """`execute` MUST be implemented in every Python model. `execute` - function receives a list of pb_utils.InferenceRequest as the only - argument. This function is called when an inference request is made - for this model. Depending on the batching configuration (e.g. Dynamic - Batching) used, `requests` may contain multiple requests. Every - Python model, must create one pb_utils.InferenceResponse for every - pb_utils.InferenceRequest in `requests`. If there is an error, you can - set the error argument when creating a pb_utils.InferenceResponse - - Parameters - ---------- - requests : list - A list of pb_utils.InferenceRequest - - Returns - ------- - list - A list of pb_utils.InferenceResponse. The length of this list must - be the same as `requests` - """ - - output_dtype = self.output_dtype - - responses = [] - - # Every Python backend must iterate over everyone of the requests - # and create a pb_utils.InferenceResponse for each of them. - for request in requests: - # Get input numpy - sentence_input = pb_utils.get_input_tensor_by_name(request, "input") - sentences = list(sentence_input.as_numpy()) - sentences = np.squeeze(sentences, -1).tolist() - sentences = [s.decode('utf-8') for s in sentences] - - input_ids = self.tokenizer(sentences, - padding="longest", - max_length=512, - truncation=True, - return_tensors="tf").input_ids - output_ids = self.model.generate(input_ids, max_length=20) - outputs = np.array([self.tokenizer.decode(o, skip_special_tokens=True) for o in output_ids]) - - # Create output tensors. You need pb_utils.Tensor - # objects to create pb_utils.InferenceResponse. - output_tensor = pb_utils.Tensor("output", outputs.astype(output_dtype)) - - # Create InferenceResponse. You can set an error here in case - # there was a problem with handling this inference request. - # Below is an example of how you can set errors in inference - # response: - # - # pb_utils.InferenceResponse( - # output_tensors=..., TritonError("An error occured")) - inference_response = pb_utils.InferenceResponse(output_tensors=[output_tensor]) - responses.append(inference_response) - - # You should return a list of pb_utils.InferenceResponse. Length - # of this list must match the length of `requests` list. - return responses - - def finalize(self): - """`finalize` is called only once when the model is being unloaded. - Implementing `finalize` function is OPTIONAL. This function allows - the model to perform any necessary clean ups before exit. - """ - print('Cleaning up...') diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_tf/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_tf/config.pbtxt deleted file mode 100644 index 88b87130..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_tf/config.pbtxt +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "hf_generation_tf" -backend: "python" -max_batch_size: 8192 - -input [ - { - name: "input" - data_type: TYPE_STRING - dims: [1] - } -] -output [ - { - name: "output" - data_type: TYPE_STRING - dims: [1] - } -] - -instance_group [{ kind: KIND_GPU }] - -parameters: { - key: "EXECUTION_ENV_PATH", - value: {string_value: "$$TRITON_MODEL_DIRECTORY/../huggingface-tf.tar.gz"} -} - diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_torch/1/model.py b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_torch/1/model.py deleted file mode 100644 index 8e9604da..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_torch/1/model.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import numpy as np -import json - -# triton_python_backend_utils is available in every Triton Python model. You -# need to use this module to create inference requests and responses. It also -# contains some utility functions for extracting information from model_config -# and converting Triton input/output types to numpy types. -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """Your Python model must use the same class name. Every Python model - that is created must have "TritonPythonModel" as the class name. - """ - - def initialize(self, args): - """`initialize` is called only once when the model is being loaded. - Implementing `initialize` function is optional. This function allows - the model to intialize any state associated with this model. - - Parameters - ---------- - args : dict - Both keys and values are strings. The dictionary keys and values are: - * model_config: A JSON string containing the model configuration - * model_instance_kind: A string containing model instance kind - * model_instance_device_id: A string containing model instance device ID - * model_repository: Model repository path - * model_version: Model version - * model_name: Model name - """ - import torch - print("torch: {}".format(torch.__version__)) - print("cuda: {}".format(torch.cuda.is_available())) - - import transformers - print("transformers: {}".format(transformers.__version__)) - - from transformers import T5Tokenizer, T5ForConditionalGeneration - self.tokenizer = T5Tokenizer.from_pretrained("t5-small") - self.model = T5ForConditionalGeneration.from_pretrained("t5-small") - - # You must parse model_config. JSON string is not parsed here - self.model_config = model_config = json.loads(args['model_config']) - - # Get output configuration - output_config = pb_utils.get_output_config_by_name(model_config, "output") - - # Convert Triton types to numpy types - self.output_dtype = pb_utils.triton_string_to_numpy(output_config['data_type']) - - def execute(self, requests): - """`execute` MUST be implemented in every Python model. `execute` - function receives a list of pb_utils.InferenceRequest as the only - argument. This function is called when an inference request is made - for this model. Depending on the batching configuration (e.g. Dynamic - Batching) used, `requests` may contain multiple requests. Every - Python model, must create one pb_utils.InferenceResponse for every - pb_utils.InferenceRequest in `requests`. If there is an error, you can - set the error argument when creating a pb_utils.InferenceResponse - - Parameters - ---------- - requests : list - A list of pb_utils.InferenceRequest - - Returns - ------- - list - A list of pb_utils.InferenceResponse. The length of this list must - be the same as `requests` - """ - - output_dtype = self.output_dtype - - responses = [] - - # Every Python backend must iterate over everyone of the requests - # and create a pb_utils.InferenceResponse for each of them. - for request in requests: - # Get input numpy - sentence_input = pb_utils.get_input_tensor_by_name(request, "input") - sentences = list(sentence_input.as_numpy()) - sentences = np.squeeze(sentences, -1).tolist() - sentences = [s.decode('utf-8') for s in sentences] - - input_ids = self.tokenizer(sentences, - padding="longest", - max_length=512, - truncation=True, - return_tensors="pt").input_ids - output_ids = self.model.generate(input_ids, max_length=20) - outputs = np.array([self.tokenizer.decode(o, skip_special_tokens=True) for o in output_ids]) - - # Create output tensors. You need pb_utils.Tensor - # objects to create pb_utils.InferenceResponse. - output_tensor = pb_utils.Tensor("output", outputs.astype(output_dtype)) - - # Create InferenceResponse. You can set an error here in case - # there was a problem with handling this inference request. - # Below is an example of how you can set errors in inference - # response: - # - # pb_utils.InferenceResponse( - # output_tensors=..., TritonError("An error occured")) - inference_response = pb_utils.InferenceResponse(output_tensors=[output_tensor]) - responses.append(inference_response) - - # You should return a list of pb_utils.InferenceResponse. Length - # of this list must match the length of `requests` list. - return responses - - def finalize(self): - """`finalize` is called only once when the model is being unloaded. - Implementing `finalize` function is OPTIONAL. This function allows - the model to perform any necessary clean ups before exit. - """ - print('Cleaning up...') diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_torch/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_torch/config.pbtxt deleted file mode 100644 index 47db5468..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_generation_torch/config.pbtxt +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "hf_generation_torch" -backend: "python" -max_batch_size: 8192 - -input [ - { - name: "input" - data_type: TYPE_STRING - dims: [1] - } -] -output [ - { - name: "output" - data_type: TYPE_STRING - dims: [1] - } -] - -instance_group [{ kind: KIND_GPU }] - -parameters: { - key: "EXECUTION_ENV_PATH", - value: {string_value: "$$TRITON_MODEL_DIRECTORY/../huggingface-torch.tar.gz"} -} - diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_tf/1/model.py b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_tf/1/model.py deleted file mode 100644 index 2a1bfda6..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_tf/1/model.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import numpy as np -import json - -# triton_python_backend_utils is available in every Triton Python model. You -# need to use this module to create inference requests and responses. It also -# contains some utility functions for extracting information from model_config -# and converting Triton input/output types to numpy types. -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """Your Python model must use the same class name. Every Python model - that is created must have "TritonPythonModel" as the class name. - """ - - def initialize(self, args): - """`initialize` is called only once when the model is being loaded. - Implementing `initialize` function is optional. This function allows - the model to intialize any state associated with this model. - - Parameters - ---------- - args : dict - Both keys and values are strings. The dictionary keys and values are: - * model_config: A JSON string containing the model configuration - * model_instance_kind: A string containing model instance kind - * model_instance_device_id: A string containing model instance device ID - * model_repository: Model repository path - * model_version: Model version - * model_name: Model name - """ - import tensorflow as tf - print("tf: {}".format(tf.__version__)) - - # Enable GPU memory growth - gpus = tf.config.experimental.list_physical_devices('GPU') - if gpus: - try: - for gpu in gpus: - tf.config.experimental.set_memory_growth(gpu, True) - except RuntimeError as e: - print(e) - - from transformers import pipeline - self.pipe = pipeline("sentiment-analysis", device=0) - - # You must parse model_config. JSON string is not parsed here - self.model_config = model_config = json.loads(args['model_config']) - - # Get output configuration - label_config = pb_utils.get_output_config_by_name(model_config, "label") - score_config = pb_utils.get_output_config_by_name(model_config, "score") - - # Convert Triton types to numpy types - self.label_dtype = pb_utils.triton_string_to_numpy(label_config['data_type']) - self.score_dtype = pb_utils.triton_string_to_numpy(score_config['data_type']) - - def execute(self, requests): - """`execute` MUST be implemented in every Python model. `execute` - function receives a list of pb_utils.InferenceRequest as the only - argument. This function is called when an inference request is made - for this model. Depending on the batching configuration (e.g. Dynamic - Batching) used, `requests` may contain multiple requests. Every - Python model, must create one pb_utils.InferenceResponse for every - pb_utils.InferenceRequest in `requests`. If there is an error, you can - set the error argument when creating a pb_utils.InferenceResponse - - Parameters - ---------- - requests : list - A list of pb_utils.InferenceRequest - - Returns - ------- - list - A list of pb_utils.InferenceResponse. The length of this list must - be the same as `requests` - """ - - label_dtype = self.label_dtype - score_dtype = self.score_dtype - - responses = [] - - # Every Python backend must iterate over everyone of the requests - # and create a pb_utils.InferenceResponse for each of them. - for request in requests: - # Get input numpy - sentence_input = pb_utils.get_input_tensor_by_name(request, "sentence") - sentences = [s.decode('utf-8') for s in sentence_input.as_numpy().flatten()] - - results = self.pipe(sentences) - - label = np.array([res['label'] for res in results]) - score = np.array([res['score'] for res in results]) - - # Create output tensors. You need pb_utils.Tensor - # objects to create pb_utils.InferenceResponse. - label_tensor = pb_utils.Tensor("label", label.astype(label_dtype)) - score_tensor = pb_utils.Tensor("score", score.astype(score_dtype)) - - # Create InferenceResponse. You can set an error here in case - # there was a problem with handling this inference request. - # Below is an example of how you can set errors in inference - # response: - # - # pb_utils.InferenceResponse( - # output_tensors=..., TritonError("An error occured")) - inference_response = pb_utils.InferenceResponse(output_tensors=[label_tensor, score_tensor]) - responses.append(inference_response) - - # You should return a list of pb_utils.InferenceResponse. Length - # of this list must match the length of `requests` list. - return responses - - def finalize(self): - """`finalize` is called only once when the model is being unloaded. - Implementing `finalize` function is OPTIONAL. This function allows - the model to perform any necessary clean ups before exit. - """ - print('Cleaning up...') diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_tf/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_tf/config.pbtxt deleted file mode 100644 index df7082ca..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_tf/config.pbtxt +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "hf_pipeline_tf" -backend: "python" -max_batch_size: 8192 - -input [ - { - name: "sentence" - data_type: TYPE_STRING - dims: [1] - } -] -output [ - { - name: "label" - data_type: TYPE_STRING - dims: [1] - }, - { - name: "score" - data_type: TYPE_FP32 - dims: [1] - } -] - -instance_group [{ kind: KIND_GPU }] - -parameters: { - key: "EXECUTION_ENV_PATH", - value: {string_value: "$$TRITON_MODEL_DIRECTORY/../huggingface-tf.tar.gz"} -} - diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_torch/1/model.py b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_torch/1/model.py deleted file mode 100644 index f01886c9..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_torch/1/model.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import numpy as np -import json - -# triton_python_backend_utils is available in every Triton Python model. You -# need to use this module to create inference requests and responses. It also -# contains some utility functions for extracting information from model_config -# and converting Triton input/output types to numpy types. -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """Your Python model must use the same class name. Every Python model - that is created must have "TritonPythonModel" as the class name. - """ - - def initialize(self, args): - """`initialize` is called only once when the model is being loaded. - Implementing `initialize` function is optional. This function allows - the model to intialize any state associated with this model. - - Parameters - ---------- - args : dict - Both keys and values are strings. The dictionary keys and values are: - * model_config: A JSON string containing the model configuration - * model_instance_kind: A string containing model instance kind - * model_instance_device_id: A string containing model instance device ID - * model_repository: Model repository path - * model_version: Model version - * model_name: Model name - """ - import torch - print("torch: {}".format(torch.__version__)) - print("cuda: {}".format(torch.cuda.is_available())) - - import transformers - print("transformers: {}".format(transformers.__version__)) - - from transformers import pipeline - self.pipe = pipeline("sentiment-analysis", device=0) - - # You must parse model_config. JSON string is not parsed here - self.model_config = model_config = json.loads(args['model_config']) - - # Get output configuration - label_config = pb_utils.get_output_config_by_name(model_config, "label") - score_config = pb_utils.get_output_config_by_name(model_config, "score") - - # Convert Triton types to numpy types - self.label_dtype = pb_utils.triton_string_to_numpy(label_config['data_type']) - self.score_dtype = pb_utils.triton_string_to_numpy(score_config['data_type']) - - def execute(self, requests): - """`execute` MUST be implemented in every Python model. `execute` - function receives a list of pb_utils.InferenceRequest as the only - argument. This function is called when an inference request is made - for this model. Depending on the batching configuration (e.g. Dynamic - Batching) used, `requests` may contain multiple requests. Every - Python model, must create one pb_utils.InferenceResponse for every - pb_utils.InferenceRequest in `requests`. If there is an error, you can - set the error argument when creating a pb_utils.InferenceResponse - - Parameters - ---------- - requests : list - A list of pb_utils.InferenceRequest - - Returns - ------- - list - A list of pb_utils.InferenceResponse. The length of this list must - be the same as `requests` - """ - - label_dtype = self.label_dtype - score_dtype = self.score_dtype - - responses = [] - - # Every Python backend must iterate over everyone of the requests - # and create a pb_utils.InferenceResponse for each of them. - for request in requests: - # Get input numpy - sentence_input = pb_utils.get_input_tensor_by_name(request, "sentence") - sentences = [s.decode('utf-8') for s in sentence_input.as_numpy().flatten()] - - results = self.pipe(sentences) - - label = np.array([res['label'] for res in results]) - score = np.array([res['score'] for res in results]) - - # Create output tensors. You need pb_utils.Tensor - # objects to create pb_utils.InferenceResponse. - label_tensor = pb_utils.Tensor("label", label.astype(label_dtype)) - score_tensor = pb_utils.Tensor("score", score.astype(score_dtype)) - - # Create InferenceResponse. You can set an error here in case - # there was a problem with handling this inference request. - # Below is an example of how you can set errors in inference - # response: - # - # pb_utils.InferenceResponse( - # output_tensors=..., TritonError("An error occured")) - inference_response = pb_utils.InferenceResponse(output_tensors=[label_tensor, score_tensor]) - responses.append(inference_response) - - # You should return a list of pb_utils.InferenceResponse. Length - # of this list must match the length of `requests` list. - return responses - - def finalize(self): - """`finalize` is called only once when the model is being unloaded. - Implementing `finalize` function is OPTIONAL. This function allows - the model to perform any necessary clean ups before exit. - """ - print('Cleaning up...') diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_torch/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_torch/config.pbtxt deleted file mode 100644 index 4e54607d..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_pipeline_torch/config.pbtxt +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "hf_pipeline_torch" -backend: "python" -max_batch_size: 8192 - -input [ - { - name: "sentence" - data_type: TYPE_STRING - dims: [1] - } -] -output [ - { - name: "label" - data_type: TYPE_STRING - dims: [1] - }, - { - name: "score" - data_type: TYPE_FP32 - dims: [1] - } -] - -instance_group [{ kind: KIND_GPU }] - -parameters: { - key: "EXECUTION_ENV_PATH", - value: {string_value: "$$TRITON_MODEL_DIRECTORY/../huggingface-torch.tar.gz"} -} - diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_transformer_torch/1/model.py b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_transformer_torch/1/model.py deleted file mode 100644 index f49805de..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_transformer_torch/1/model.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import numpy as np -import json - -# triton_python_backend_utils is available in every Triton Python model. You -# need to use this module to create inference requests and responses. It also -# contains some utility functions for extracting information from model_config -# and converting Triton input/output types to numpy types. -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """Your Python model must use the same class name. Every Python model - that is created must have "TritonPythonModel" as the class name. - """ - - def initialize(self, args): - """`initialize` is called only once when the model is being loaded. - Implementing `initialize` function is optional. This function allows - the model to intialize any state associated with this model. - - Parameters - ---------- - args : dict - Both keys and values are strings. The dictionary keys and values are: - * model_config: A JSON string containing the model configuration - * model_instance_kind: A string containing model instance kind - * model_instance_device_id: A string containing model instance device ID - * model_repository: Model repository path - * model_version: Model version - * model_name: Model name - """ - import torch - print("torch: {}".format(torch.__version__)) - print("cuda: {}".format(torch.cuda.is_available())) - - import transformers - print("transformers: {}".format(transformers.__version__)) - - from sentence_transformers import SentenceTransformer - self.model = SentenceTransformer('paraphrase-MiniLM-L6-v2') - - # You must parse model_config. JSON string is not parsed here - self.model_config = model_config = json.loads(args['model_config']) - - # Get output configuration - embedding_config = pb_utils.get_output_config_by_name(model_config, "embedding") - - # Convert Triton types to numpy types - self.embedding_dtype = pb_utils.triton_string_to_numpy(embedding_config['data_type']) - - def execute(self, requests): - """`execute` MUST be implemented in every Python model. `execute` - function receives a list of pb_utils.InferenceRequest as the only - argument. This function is called when an inference request is made - for this model. Depending on the batching configuration (e.g. Dynamic - Batching) used, `requests` may contain multiple requests. Every - Python model, must create one pb_utils.InferenceResponse for every - pb_utils.InferenceRequest in `requests`. If there is an error, you can - set the error argument when creating a pb_utils.InferenceResponse - - Parameters - ---------- - requests : list - A list of pb_utils.InferenceRequest - - Returns - ------- - list - A list of pb_utils.InferenceResponse. The length of this list must - be the same as `requests` - """ - - embedding_dtype = self.embedding_dtype - - responses = [] - - # Every Python backend must iterate over everyone of the requests - # and create a pb_utils.InferenceResponse for each of them. - for request in requests: - # Get input numpy - sentence_input = pb_utils.get_input_tensor_by_name(request, "sentence") - sentences = list(sentence_input.as_numpy()) - sentences = np.squeeze(sentences, -1).tolist() - sentences = [s.decode('utf-8') for s in sentences] - - embedding = self.model.encode(sentences) - - # Create output tensors. You need pb_utils.Tensor - # objects to create pb_utils.InferenceResponse. - embedding_tensor = pb_utils.Tensor("embedding", embedding.astype(embedding_dtype)) - - # Create InferenceResponse. You can set an error here in case - # there was a problem with handling this inference request. - # Below is an example of how you can set errors in inference - # response: - # - # pb_utils.InferenceResponse( - # output_tensors=..., TritonError("An error occured")) - inference_response = pb_utils.InferenceResponse(output_tensors=[embedding_tensor]) - responses.append(inference_response) - - # You should return a list of pb_utils.InferenceResponse. Length - # of this list must match the length of `requests` list. - return responses - - def finalize(self): - """`finalize` is called only once when the model is being unloaded. - Implementing `finalize` function is OPTIONAL. This function allows - the model to perform any necessary clean ups before exit. - """ - print('Cleaning up...') diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_transformer_torch/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_transformer_torch/config.pbtxt deleted file mode 100644 index 798cf4fc..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/models_config/hf_transformer_torch/config.pbtxt +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "hf_transformer_torch" -backend: "python" -max_batch_size: 8192 - -input [ - { - name: "sentence" - data_type: TYPE_STRING - dims: [1] - } -] -output [ - { - name: "embedding" - data_type: TYPE_FP32 - dims: [384] - } -] - -instance_group [{ kind: KIND_GPU }] - -parameters: { - key: "EXECUTION_ENV_PATH", - value: {string_value: "$$TRITON_MODEL_DIRECTORY/../huggingface-torch.tar.gz"} -} - diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb index dcba0be8..749ba9d3 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb @@ -2,13 +2,16 @@ "cells": [ { "cell_type": "markdown", - "id": "60f7ac5d-4a95-4170-a0ac-a7faac9d9ef4", + "id": "9e9fe848", "metadata": {}, "source": [ + "\n", + "\n", "# PySpark Huggingface Inferencing\n", - "### Text Classification using Pipelines with Tensorflow\n", + "### Sentiment Analysis using Pipelines with Tensorflow\n", "\n", - "Based on: https://huggingface.co/docs/transformers/quicktour#pipeline-usage" + "In this notebook, we demonstrate distributed inference with Huggingface Pipelines to perform sentiment analysis. \n", + "From: https://huggingface.co/docs/transformers/quicktour#pipeline-usage" ] }, { @@ -16,14 +19,12 @@ "id": "1799fd4f", "metadata": {}, "source": [ - "### Using TensorFlow\n", - "Note that cuFFT/cuDNN/cuBLAS registration errors are expected with `tf=2.17.0` and will not affect behavior, as noted in [this issue.](https://github.com/tensorflow/tensorflow/issues/62075) \n", - "This notebook does not demonstrate inference with TensorRT, as [TF-TRT](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/index.html#tensorrt-10) does not yet support `tf=2.17.0`. See the `pytorch` notebooks for TensorRT demos." + "Note that cuFFT/cuDNN/cuBLAS registration errors are expected (as of `tf=2.17.0`) and will not affect behavior, as noted in [this issue.](https://github.com/tensorflow/tensorflow/issues/62075) " ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "0dd0f77b-ee1b-4477-a038-d25a4f1da0ea", "metadata": {}, "outputs": [ @@ -31,35 +32,50 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-03 16:47:48.209366: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2024-10-03 16:47:48.215921: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2024-10-03 16:47:48.223519: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2024-10-03 16:47:48.225906: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2024-10-03 16:47:48.231640: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "2025-01-27 12:06:26.417984: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2025-01-27 12:06:26.426005: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2025-01-27 12:06:26.434857: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2025-01-27 12:06:26.437414: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2025-01-27 12:06:26.444254: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2024-10-03 16:47:48.625790: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" + "2025-01-27 12:06:26.880520: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" ] } ], "source": [ "import tensorflow as tf\n", - "from transformers import pipeline" + "from transformers import pipeline\n", + "\n", + "# Manually enable Huggingface tokenizer parallelism to avoid disabling with PySpark parallelism.\n", + "# See (https://github.com/huggingface/transformers/issues/5486) for more info. \n", + "import os\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "d80fc3f8", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", + "I0000 00:00:1738008387.629698 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008387.653432 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008387.656121 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" + ] + } + ], "source": [ - "# set device if tensorflow gpu is available\n", "device = 0 if tf.config.list_physical_devices('GPU') else -1" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "e60a2877", "metadata": {}, "outputs": [ @@ -72,8 +88,6 @@ } ], "source": [ - "print(tf.__version__)\n", - "\n", "# Enable GPU memory growth\n", "gpus = tf.config.experimental.list_physical_devices('GPU')\n", "if gpus:\n", @@ -81,12 +95,14 @@ " for gpu in gpus:\n", " tf.config.experimental.set_memory_growth(gpu, True)\n", " except RuntimeError as e:\n", - " print(e)" + " print(e)\n", + "\n", + "print(tf.__version__)" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "553b28d2-a5d1-4d07-8a49-8f82b808e738", "metadata": {}, "outputs": [ @@ -95,8 +111,20 @@ "output_type": "stream", "text": [ "No model was supplied, defaulted to distilbert/distilbert-base-uncased-finetuned-sst-2-english and revision 714eb0f (https://huggingface.co/distilbert/distilbert-base-uncased-finetuned-sst-2-english).\n", - "Using a pipeline without specifying a model name and revision in production is not recommended.\n", - "2024-10-03 16:47:49.863791: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 46447 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n", + "Using a pipeline without specifying a model name and revision in production is not recommended.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "I0000 00:00:1738008387.957197 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008387.960022 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008387.962670 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008388.067491 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008388.068546 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008388.069484 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2025-01-27 12:06:28.070408: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 43043 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n", "All PyTorch model weights were used when initializing TFDistilBertForSequenceClassification.\n", "\n", "All the weights of TFDistilBertForSequenceClassification were initialized from the PyTorch model.\n", @@ -110,7 +138,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "3b91fe91-b725-4564-ae93-56e3fb51e47c", "metadata": {}, "outputs": [ @@ -120,7 +148,7 @@ "[{'label': 'POSITIVE', 'score': 0.9997794032096863}]" ] }, - "execution_count": 6, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -131,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "id": "0be39eb3-462c-42ff-b8f4-09f4e4fe3a3c", "metadata": {}, "outputs": [ @@ -152,15 +180,15 @@ }, { "cell_type": "markdown", - "id": "30c90100", + "id": "e29ee6d8", "metadata": {}, "source": [ - "#### Use another model and tokenizer in the pipeline" + "Let's try a different model and tokenizer in the pipeline." ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "id": "cd9d3349", "metadata": {}, "outputs": [], @@ -170,7 +198,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "id": "99e21b58", "metadata": {}, "outputs": [ @@ -178,10 +206,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "Some layers from the model checkpoint at nlptown/bert-base-multilingual-uncased-sentiment were not used when initializing TFBertForSequenceClassification: ['dropout_37']\n", - "- This IS expected if you are initializing TFBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", - "- This IS NOT expected if you are initializing TFBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", - "All the layers of TFBertForSequenceClassification were initialized from the model checkpoint at nlptown/bert-base-multilingual-uncased-sentiment.\n", + "All PyTorch model weights were used when initializing TFBertForSequenceClassification.\n", + "\n", + "All the weights of TFBertForSequenceClassification were initialized from the PyTorch model.\n", "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertForSequenceClassification for predictions without further training.\n" ] } @@ -195,179 +222,357 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "id": "31079133", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[{'label': '5 stars', 'score': 0.7272655963897705}]" + "[{'label': '5 stars', 'score': 0.7272477746009827}]" ] }, - "execution_count": 10, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "classifier = pipeline(\"sentiment-analysis\", model=model, tokenizer=tokenizer)\n", + "classifier = pipeline(\"sentiment-analysis\", model=model, tokenizer=tokenizer, device=device)\n", "classifier(\"Nous sommes très heureux de vous présenter la bibliothèque 🤗 Transformers.\")" ] }, { "cell_type": "markdown", - "id": "ae92b15e-0da0-46c3-81a3-fabaedbfc42c", + "id": "e6357234", "metadata": {}, "source": [ - "## Inference using Spark DL API" + "## PySpark" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "id": "69dd6a1a-f450-47f0-9dbf-ad250585a011", "metadata": {}, "outputs": [], "source": [ - "import os\n", - "import pandas as pd\n", "from pyspark.sql.functions import col, struct, pandas_udf\n", "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.types import FloatType, StringType, StructField, StructType\n", + "from pyspark.sql.types import *\n", "from pyspark.sql import SparkSession\n", "from pyspark import SparkConf" ] }, { "cell_type": "code", - "execution_count": null, - "id": "6e0e0dd7", + "execution_count": 11, + "id": "287b1e96", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "import pandas as pd\n", + "import datasets\n", + "from datasets import load_dataset\n", + "datasets.disable_progress_bars()" + ] + }, + { + "cell_type": "markdown", + "id": "50e124cd", + "metadata": {}, + "source": [ + "Check the cluster environment to handle any platform-specific Spark configurations." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "36001f55", "metadata": {}, "outputs": [], "source": [ - "conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n", + "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n", + "on_standalone = not (on_databricks or on_dataproc)" + ] + }, + { + "cell_type": "markdown", + "id": "48c7271a", + "metadata": {}, + "source": [ + "#### Create Spark Session\n", "\n", + "For local standalone clusters, we'll connect to the cluster and create the Spark Session. \n", + "For CSP environments, Spark will either be preconfigured (Databricks) or we'll need to create the Spark Session (Dataproc)." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6e0e0dd7", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/27 20:06:30 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/01/27 20:06:30 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", + "25/01/27 20:06:30 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "25/01/27 20:06:31 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + ] + } + ], + "source": [ "conf = SparkConf()\n", + "\n", "if 'spark' not in globals():\n", - " # If Spark is not already started with Jupyter, attach to Spark Standalone\n", - " import socket\n", - " hostname = socket.gethostname()\n", - " conf.setMaster(f\"spark://{hostname}:7077\") # assuming Master is on default port 7077\n", - "conf.set(\"spark.task.maxFailures\", \"1\")\n", - "conf.set(\"spark.driver.memory\", \"8g\")\n", - "conf.set(\"spark.executor.memory\", \"8g\")\n", - "conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled\", \"false\")\n", - "conf.set(\"spark.sql.pyspark.jvmStacktrace.enabled\", \"true\")\n", - "conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", - "conf.set(\"spark.python.worker.reuse\", \"true\")\n", - "# Create Spark Session\n", + " if on_standalone:\n", + " import socket\n", + " \n", + " conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + " hostname = socket.gethostname()\n", + " conf.setMaster(f\"spark://{hostname}:7077\")\n", + " conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", + " conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_env}/lib:{conda_env}/lib/python3.11/site-packages/nvidia_pytriton.libs:$LD_LIBRARY_PATH\")\n", + " source = \"/usr/lib/x86_64-linux-gnu/libstdc++.so.6\"\n", + " target = f\"{conda_env}/lib/libstdc++.so.6\"\n", + " try:\n", + " if os.path.islink(target) or os.path.exists(target):\n", + " os.remove(target)\n", + " os.symlink(source, target)\n", + " except OSError as e:\n", + " print(f\"Error creating symlink: {e}\")\n", + " elif on_dataproc:\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conda_lib_path=\"/opt/conda/miniconda3/lib\"\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_lib_path}:$LD_LIBRARY_PATH\")\n", + " conf.set(\"spark.executorEnv.TF_GPU_ALLOCATOR\", \"cuda_malloc_async\")\n", + " conf.set(\"spark.executor.instances\", \"4\") # dataproc defaults to 2\n", + "\n", + " conf.set(\"spark.executor.cores\", \"8\")\n", + " conf.set(\"spark.task.resource.gpu.amount\", \"0.125\")\n", + " conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n", + " conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", + " conf.set(\"spark.python.worker.reuse\", \"true\")\n", + "\n", + "conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"1000\")\n", "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", "sc = spark.sparkContext" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "id": "42d70208", "metadata": {}, "outputs": [], "source": [ - "from datasets import load_dataset\n", - "\n", - "# Load the IMDB dataset\n", - "data = load_dataset(\"imdb\", split=\"test\")\n", - "\n", - "lines = []\n", - "for example in data:\n", - " # first sentence only\n", - " lines.append([example[\"text\"]])\n", - "\n", - "len(lines)\n", - "\n", - "df = spark.createDataFrame(lines, ['lines']).repartition(8).cache()" + "dataset = load_dataset(\"imdb\", split=\"test\")\n", + "dataset = dataset.to_pandas().drop(columns=\"label\")" + ] + }, + { + "cell_type": "markdown", + "id": "95ded4b2", + "metadata": {}, + "source": [ + "#### Create PySpark DataFrame" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "id": "ac24f3c2", "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "StructType([StructField('text', StringType(), True)])" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = spark.createDataFrame(dataset).repartition(8)\n", + "df.schema" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "1db4db3a", + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "24/10/03 16:47:58 WARN TaskSetManager: Stage 0 contains a task of very large size (3860 KiB). The maximum recommended task size is 1000 KiB.\n", " \r" ] + }, + { + "data": { + "text/plain": [ + "25000" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "df.write.mode(\"overwrite\").parquet(\"imdb_test\")" + "df.count()" ] }, { "cell_type": "code", - "execution_count": 15, - "id": "9665b7b6-d7e9-4bd4-b29d-7a449ac5b574", + "execution_count": 17, + "id": "517fe2e9", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "25/01/27 20:06:38 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] }, + { + "data": { + "text/plain": [ + "[Row(text=\"Anyone remember the first CKY, CKY2K etc..? Back when it was about making crazy cool stuff, rather than watching Bam Margera act like a douchebag, spoiled 5 year old, super/rock-star wannabe.

The show used to be awesome, however, Bam's fame and wealth has led him to believe, that we now enjoy him acting childish and idiotic, more than actual cool stuff, that used to be in ex. CKY2K.

The acts are so repetitive, there's like nothing new, except annoying stupidity and rehearsed comments... The only things we see is Bam Margera, so busy showing us how much he doesn't care, how much money he got or whatsoever.

I really got nothing much left to say except, give us back CKY2K, cause Bam suck..

I enjoy watching Steve-o, Knoxville etc. a thousand times more.\")]" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.take(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "e176d28b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/27 20:06:38 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + ] + } + ], + "source": [ + "data_path = \"spark-dl-datasets/imdb_test\"\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " data_path = \"dbfs:/FileStore/\" + data_path\n", + "\n", + "df.write.mode(\"overwrite\").parquet(data_path)" + ] + }, + { + "cell_type": "markdown", + "id": "395e0374", + "metadata": {}, + "source": [ + "#### Load and preprocess DataFrame\n", + "\n", + "Define our preprocess function. We'll take the first sentence from each sample as our input for sentiment analysis." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "9665b7b6-d7e9-4bd4-b29d-7a449ac5b574", + "metadata": {}, + "outputs": [], + "source": [ + "@pandas_udf(\"string\")\n", + "def preprocess(text: pd.Series) -> pd.Series:\n", + " return pd.Series([s.split(\".\")[0] for s in text])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "26693020", + "metadata": {}, + "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+--------------------------------------------------------------------------------+\n", - "| sentence|\n", - "+--------------------------------------------------------------------------------+\n", - "| |\n", - "|Hard up, No proper jobs going down at the pit, why not rent your kids! DIY pi...|\n", - "|I watched this movie to see the direction one of the most promising young tal...|\n", - "| This movie makes you wish imdb would let you vote a zero|\n", - "|I never want to see this movie again!

Not only is it dreadfully ba...|\n", - "|(As a note, I'd like to say that I saw this movie at my annual church camp, w...|\n", - "| Don't get me wrong, I love the TV series of League Of Gentlemen|\n", - "|Did you ever think, like after watching a horror movie with a group of friend...|\n", - "| Awful, awful, awful|\n", - "|This movie seems a little clunky around the edges, like not quite enough zani...|\n", - "|I rented this movie hoping that it would provide some good entertainment and ...|\n", - "|Well, where to start describing this celluloid debacle? You already know the ...|\n", - "| I hoped for this show to be somewhat realistic|\n", - "| All I have to say is one word|\n", - "|Honestly awful film, bad editing, awful lighting, dire dialog and scrappy scr...|\n", - "|This critique tells the story of 4 little friends who went to watch Angels an...|\n", - "| This review contains a partial spoiler|\n", - "| I'm rather surprised that anybody found this film touching or moving|\n", - "| If you like bad movies (and you must to watch this one) here's a good one|\n", - "|This is really bad, the characters were bland, the story was boring, and ther...|\n", - "+--------------------------------------------------------------------------------+\n", + "+----------------------------------------------------------------------------------------------------+\n", + "| input|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "|Doesn't anyone bother to check where this kind of sludge comes from before blathering on about it...|\n", + "| There were two things I hated about WASTED : The directing and the script |\n", + "| I'm rather surprised that anybody found this film touching or moving|\n", + "|Cultural Vandalism Is the new Hallmark production of Gulliver's Travels an act of cultural vandal...|\n", + "|I was at Wrestlemania VI in Toronto as a 10 year old, and the event I saw then was pretty differe...|\n", + "| This movie has been done before|\n", + "|[ as a new resolution for this year 2005, i decide to write a comment for each movie I saw in the...|\n", + "|This movie is over hyped!! I am sad to say that I manage to watch the first 15 minutes of this mo...|\n", + "|This show had a promising start as sort of the opposite of 'Oceans 11' but has developed into a s...|\n", + "|MINOR PLOT SPOILERS AHEAD!!!

How did such talented actors get involved in such mindles...|\n", + "| There is not one character on this sitcom with any redeeming qualities|\n", + "| Tommy Lee Jones was the best Woodroe and no one can play Woodroe F|\n", + "| My wife rented this movie and then conveniently never got to see it|\n", + "|This is one of those star-filled over-the-top comedies that could a) be hysterical, or b) wish th...|\n", + "|This excruciatingly boring and unfunny movie made me think that Chaplin was the real Hitler, as o...|\n", + "| you will likely be sorely disappointed by this sequel that's not a sequel|\n", + "| If I was British, I would be embarrassed by this portrayal of incompetence|\n", + "|One of those movies in which there are no big twists whatsoever and you can predict pretty much w...|\n", + "| This show is like watching someone who is in training to someday host a show|\n", + "| Sigh|\n", + "+----------------------------------------------------------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] } ], "source": [ - "# only use first sentence of IMDB reviews\n", - "@pandas_udf(\"string\")\n", - "def first_sentence(text: pd.Series) -> pd.Series:\n", - " return pd.Series([s.split(\".\")[0] for s in text])\n", + "# Limit to N rows, since this can be slow\n", + "df = spark.read.parquet(data_path).limit(256).repartition(8)\n", + "df = df.select(preprocess(col(\"text\")).alias(\"input\")).cache()\n", + "df.show(truncate=100)" + ] + }, + { + "cell_type": "markdown", + "id": "76dc525c", + "metadata": {}, + "source": [ + "## Inference using Spark DL API\n", "\n", - "df = spark.read.parquet(\"imdb_test\").withColumn(\"sentence\", first_sentence(col(\"lines\"))).select(\"sentence\").limit(100).cache()\n", - "df.show(truncate=80)" + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "\n", + "- predict_batch_fn uses Tensorflow APIs to load the model and return a predict function which operates on numpy arrays \n", + "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 21, "id": "0da9d25c-5ebe-4503-bb19-154fcc047cbf", "metadata": {}, "outputs": [], @@ -394,7 +599,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 22, "id": "78afef29-ee30-4267-9fb6-be2dcb86cbba", "metadata": {}, "outputs": [], @@ -404,12 +609,12 @@ " StructField(\"label\", StringType(), True),\n", " StructField(\"score\", FloatType(), True)\n", " ]),\n", - " batch_size=10)" + " batch_size=32)" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 23, "id": "a5bc327e-89cf-4731-82e6-e66cb93deef1", "metadata": {}, "outputs": [ @@ -417,15 +622,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 11:> (0 + 1) / 1]\r" + "[Stage 18:=======> (1 + 7) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 9.15 ms, sys: 6.76 ms, total: 15.9 ms\n", - "Wall time: 5 s\n" + "CPU times: user 6.91 ms, sys: 3.39 ms, total: 10.3 ms\n", + "Wall time: 4.75 s\n" ] }, { @@ -438,14 +643,15 @@ ], "source": [ "%%time\n", + "# first pass caches model/fn\n", "# note: expanding the \"struct\" return_type to top-level columns\n", - "preds = df.withColumn(\"preds\", classify(struct(\"sentence\"))).select(\"sentence\", \"preds.*\")\n", + "preds = df.withColumn(\"preds\", classify(struct(\"input\"))).select(\"input\", \"preds.*\")\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 24, "id": "ac642895-cfd6-47ee-9b21-02e7835424e4", "metadata": {}, "outputs": [ @@ -453,15 +659,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 13:> (0 + 1) / 1]\r" + "[Stage 21:> (0 + 8) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 4.86 ms, sys: 2.19 ms, total: 7.05 ms\n", - "Wall time: 2.81 s\n" + "CPU times: user 2.93 ms, sys: 3.4 ms, total: 6.33 ms\n", + "Wall time: 1.24 s\n" ] }, { @@ -474,14 +680,13 @@ ], "source": [ "%%time\n", - "# note: expanding the \"struct\" return_type to top-level columns\n", - "preds = df.withColumn(\"preds\", classify(\"sentence\")).select(\"sentence\", \"preds.*\")\n", + "preds = df.withColumn(\"preds\", classify(\"input\")).select(\"input\", \"preds.*\")\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 25, "id": "76a44d80-d5db-405f-989c-7246379cfb95", "metadata": {}, "outputs": [ @@ -489,15 +694,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 15:> (0 + 1) / 1]\r" + "[Stage 24:> (0 + 8) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.91 ms, sys: 1.96 ms, total: 5.87 ms\n", - "Wall time: 2.76 s\n" + "CPU times: user 3.63 ms, sys: 2.03 ms, total: 5.66 ms\n", + "Wall time: 1.31 s\n" ] }, { @@ -510,14 +715,13 @@ ], "source": [ "%%time\n", - "# note: expanding the \"struct\" return_type to top-level columns\n", - "preds = df.withColumn(\"preds\", classify(col(\"sentence\"))).select(\"sentence\", \"preds.*\")\n", + "preds = df.withColumn(\"preds\", classify(col(\"input\"))).select(\"input\", \"preds.*\")\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 26, "id": "c01761b3-c766-46b0-ae0b-fcf968ffb3a1", "metadata": {}, "outputs": [ @@ -526,28 +730,28 @@ "output_type": "stream", "text": [ "+--------------------------------------------------------------------------------+--------+----------+\n", - "| sentence| label| score|\n", + "| input| label| score|\n", "+--------------------------------------------------------------------------------+--------+----------+\n", - "| |POSITIVE|0.74807304|\n", - "|Hard up, No proper jobs going down at the pit, why not rent your kids! DIY pi...|NEGATIVE| 0.9996724|\n", - "|I watched this movie to see the direction one of the most promising young tal...|POSITIVE| 0.9994948|\n", - "| This movie makes you wish imdb would let you vote a zero|NEGATIVE| 0.9981299|\n", - "|I never want to see this movie again!

Not only is it dreadfully ba...|NEGATIVE|0.99883264|\n", - "|(As a note, I'd like to say that I saw this movie at my annual church camp, w...|POSITIVE| 0.9901753|\n", - "| Don't get me wrong, I love the TV series of League Of Gentlemen|POSITIVE|0.99983096|\n", - "|Did you ever think, like after watching a horror movie with a group of friend...|POSITIVE| 0.9992768|\n", - "| Awful, awful, awful|NEGATIVE| 0.9997433|\n", - "|This movie seems a little clunky around the edges, like not quite enough zani...|NEGATIVE| 0.9996525|\n", - "|I rented this movie hoping that it would provide some good entertainment and ...|NEGATIVE|0.99643254|\n", - "|Well, where to start describing this celluloid debacle? You already know the ...|NEGATIVE|0.99973005|\n", - "| I hoped for this show to be somewhat realistic|POSITIVE| 0.8417903|\n", - "| All I have to say is one word|NEGATIVE|0.97844803|\n", - "|Honestly awful film, bad editing, awful lighting, dire dialog and scrappy scr...|NEGATIVE| 0.9997701|\n", - "|This critique tells the story of 4 little friends who went to watch Angels an...|POSITIVE| 0.9942386|\n", - "| This review contains a partial spoiler|NEGATIVE|0.99620205|\n", + "|Doesn't anyone bother to check where this kind of sludge comes from before bl...|NEGATIVE| 0.9984061|\n", + "| There were two things I hated about WASTED : The directing and the script |NEGATIVE| 0.9979007|\n", "| I'm rather surprised that anybody found this film touching or moving|POSITIVE|0.83874947|\n", - "| If you like bad movies (and you must to watch this one) here's a good one|POSITIVE| 0.9936475|\n", - "|This is really bad, the characters were bland, the story was boring, and ther...|NEGATIVE|0.99953806|\n", + "|Cultural Vandalism Is the new Hallmark production of Gulliver's Travels an ac...|NEGATIVE|0.99727434|\n", + "|I was at Wrestlemania VI in Toronto as a 10 year old, and the event I saw the...|POSITIVE| 0.982114|\n", + "| This movie has been done before|NEGATIVE|0.94210696|\n", + "|[ as a new resolution for this year 2005, i decide to write a comment for eac...|NEGATIVE| 0.9967818|\n", + "|This movie is over hyped!! I am sad to say that I manage to watch the first 1...|NEGATIVE| 0.9985843|\n", + "|This show had a promising start as sort of the opposite of 'Oceans 11' but ha...|NEGATIVE|0.99926835|\n", + "|MINOR PLOT SPOILERS AHEAD!!!

How did such talented actors get invo...|NEGATIVE|0.99956733|\n", + "| There is not one character on this sitcom with any redeeming qualities|NEGATIVE| 0.9985662|\n", + "| Tommy Lee Jones was the best Woodroe and no one can play Woodroe F|POSITIVE| 0.994562|\n", + "| My wife rented this movie and then conveniently never got to see it|NEGATIVE|0.99841607|\n", + "|This is one of those star-filled over-the-top comedies that could a) be hyste...|NEGATIVE| 0.9953243|\n", + "|This excruciatingly boring and unfunny movie made me think that Chaplin was t...|NEGATIVE| 0.9997607|\n", + "| you will likely be sorely disappointed by this sequel that's not a sequel|NEGATIVE| 0.9997198|\n", + "| If I was British, I would be embarrassed by this portrayal of incompetence|NEGATIVE| 0.9965172|\n", + "|One of those movies in which there are no big twists whatsoever and you can p...|NEGATIVE| 0.9986059|\n", + "| This show is like watching someone who is in training to someday host a show|NEGATIVE|0.97015846|\n", + "| Sigh|NEGATIVE| 0.9923151|\n", "+--------------------------------------------------------------------------------+--------+----------+\n", "only showing top 20 rows\n", "\n" @@ -560,289 +764,386 @@ }, { "cell_type": "markdown", - "id": "eb826fde-99d9-43fe-8ddc-f5acbe76b4e9", + "id": "fc8127d9", "metadata": {}, "source": [ - "### Using Triton Inference Server\n", + "## Using Triton Inference Server\n", + "In this section, we demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. \n", + "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server. \n", "\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment. " + "The process looks like this:\n", + "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n", + "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n", + "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n", + "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n", + "\n", + "\"drawing\"" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "4d4be844-4b8c-47df-bd09-0c280c7ff16b", + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial" ] }, { "cell_type": "markdown", - "id": "10368010-f94d-4167-91a1-2cf9ed91a2c9", + "id": "4f15dfcb", "metadata": {}, "source": [ - "This notebook uses the [Python backend with a custom execution environment](https://github.com/triton-inference-server/python_backend#creating-custom-execution-environments) with the compatible versions of Python/Numpy for Triton 24.08, using a conda-pack environment created as follows:\n", - "```\n", - "conda create -n huggingface-tf -c conda-forge python=3.10.0\n", - "conda activate huggingface-tf\n", - "\n", - "export PYTHONNOUSERSITE=True\n", - "pip install numpy==1.26.4 tensorflow[and-cuda] tf-keras transformers conda-pack\n", - "\n", - "conda-pack # huggingface-tf.tar.gz\n", - "```" + "Import the utility functions from pytriton_utils.py:" ] }, { "cell_type": "code", - "execution_count": 22, - "id": "4d4be844-4b8c-47df-bd09-0c280c7ff16b", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 28, + "id": "bfa7ec9d", + "metadata": {}, "outputs": [], "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "import os\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import col, struct, pandas_udf\n", - "from pyspark.sql.types import FloatType, StringType, StructField, StructType" + "sc.addPyFile(\"pytriton_utils.py\")\n", + "\n", + "from pytriton_utils import (\n", + " use_stage_level_scheduling,\n", + " find_ports,\n", + " start_triton,\n", + " stop_triton\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "1bf04546", + "metadata": {}, + "source": [ + "Define the Triton Server function:" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 29, "id": "7e53df9f-43cb-4c38-b8ac-dc2cbad99815", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ - "%%bash\n", - "# copy custom model to expected layout for Triton\n", - "rm -rf models\n", - "mkdir -p models\n", - "cp -r models_config/hf_pipeline_tf models\n", + "def triton_server(ports):\n", + " import time\n", + " import signal\n", + " import numpy as np\n", + " import tensorflow as tf\n", + " from transformers import pipeline\n", + " from pytriton.decorators import batch\n", + " from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n", + " from pytriton.triton import Triton, TritonConfig\n", + " from pyspark import TaskContext\n", + "\n", + " print(f\"SERVER: Initializing pipeline on worker {TaskContext.get().partitionId()}.\")\n", + " # Enable GPU memory growth\n", + " gpus = tf.config.experimental.list_physical_devices('GPU')\n", + " if gpus:\n", + " try:\n", + " for gpu in gpus:\n", + " tf.config.experimental.set_memory_growth(gpu, True)\n", + " except RuntimeError as e:\n", + " print(e)\n", + " \n", + " device = 0 if tf.config.list_physical_devices('GPU') else -1\n", + " \n", + " pipe = pipeline(\"sentiment-analysis\", device=device)\n", + " print(f\"SERVER: Using {device} device.\")\n", + "\n", + " @batch\n", + " def _infer_fn(**inputs):\n", + " sentences = np.squeeze(inputs[\"text\"]).tolist()\n", + " print(f\"SERVER: Received batch of size {len(sentences)}\")\n", + " decoded_sentences = [s.decode(\"utf-8\") for s in sentences]\n", + " return {\n", + " \"outputs\": np.array([[json.dumps(o)] for o in pipe(decoded_sentences)])\n", + " }\n", + "\n", + " workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n", + " triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n", + " with Triton(config=triton_conf, workspace=workspace_path) as triton:\n", + " triton.bind(\n", + " model_name=\"SentimentAnalysis\",\n", + " infer_func=_infer_fn,\n", + " inputs=[\n", + " Tensor(name=\"text\", dtype=object, shape=(-1,)),\n", + " ],\n", + " outputs=[\n", + " Tensor(name=\"outputs\", dtype=object, shape=(-1,)),\n", + " ],\n", + " config=ModelConfig(\n", + " max_batch_size=64,\n", + " batcher=DynamicBatcher(max_queue_delay_microseconds=5000), # 5ms\n", + " ),\n", + " strict=True,\n", + " )\n", "\n", - "# add custom execution environment\n", - "cp huggingface-tf.tar.gz models" + " def _stop_triton(signum, frame):\n", + " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", + " triton.stop()\n", + "\n", + " signal.signal(signal.SIGTERM, _stop_triton)\n", + "\n", + " print(\"SERVER: Serving inference\")\n", + " triton.serve()" ] }, { "cell_type": "markdown", - "id": "db4a5b06-126a-4bc4-baae-a45ea30832a7", - "metadata": { - "tags": [] - }, + "id": "19d9028d", + "metadata": {}, + "source": [ + "#### Start Triton servers" + ] + }, + { + "cell_type": "markdown", + "id": "2f85dc27", + "metadata": {}, "source": [ - "#### Start Triton Server on each executor" + "**Specify the number of nodes in the cluster.** \n", + "Following the README, the example standalone cluster uses 1 node. The example Databricks/Dataproc cluster scripts use 4 nodes by default. " ] }, { "cell_type": "code", - "execution_count": 24, - "id": "144acb8e-4c08-40fc-a9ed-f721c409ee68", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 30, + "id": "714e6ef9", + "metadata": {}, + "outputs": [], + "source": [ + "# Change based on cluster setup\n", + "num_nodes = 1 if on_standalone else 4" + ] + }, + { + "cell_type": "markdown", + "id": "c10905de", + "metadata": {}, + "source": [ + "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "156de815", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + } + ], + "source": [ + "sc = spark.sparkContext\n", + "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" + ] + }, + { + "cell_type": "markdown", + "id": "736ac5f4", + "metadata": {}, + "source": [ + "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "4b6044f9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using ports [7000, 7001, 7002]\n" + ] + } + ], + "source": [ + "model_name = \"SentimentAnalysis\"\n", + "ports = find_ports()\n", + "assert len(ports) == 3\n", + "print(f\"Using ports {ports}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "f75c30c5", + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "[Stage 28:> (0 + 1) / 1]\r" ] }, { - "data": { - "text/plain": [ - "[True]" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "Triton Server PIDs:\n", + " {\n", + " \"cb4ae00-lcedt\": 3019330\n", + "}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] } ], "source": [ - "num_executors = 1\n", - "triton_models_dir = \"{}/models\".format(os.getcwd())\n", - "huggingface_cache_dir = \"{}/.cache/huggingface\".format(os.path.expanduser('~'))\n", - "nodeRDD = sc.parallelize(list(range(num_executors)), num_executors)\n", - "\n", - "def start_triton(it):\n", - " import docker\n", - " import time\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " if containers:\n", - " print(\">>>> containers: {}\".format([c.short_id for c in containers]))\n", - " else:\n", - " container=client.containers.run(\n", - " \"nvcr.io/nvidia/tritonserver:24.08-py3\", \"tritonserver --model-repository=/models\",\n", - " detach=True,\n", - " device_requests=[docker.types.DeviceRequest(device_ids=[\"0\"], capabilities=[['gpu']])],\n", - " environment=[\n", - " \"TRANSFORMERS_CACHE=/cache\"\n", - " ],\n", - " name=\"spark-triton\",\n", - " network_mode=\"host\",\n", - " remove=True,\n", - " shm_size=\"256M\",\n", - " volumes={\n", - " triton_models_dir: {\"bind\": \"/models\", \"mode\": \"ro\"},\n", - " huggingface_cache_dir: {\"bind\": \"/cache\", \"mode\": \"rw\"}\n", - " }\n", - " )\n", - " print(\">>>> starting triton: {}\".format(container.short_id))\n", - " # wait for triton to be running\n", - " time.sleep(15)\n", - " \n", - " client = grpcclient.InferenceServerClient(\"localhost:8001\")\n", - " \n", - " elapsed = 0\n", - " timeout = 120\n", - " ready = False\n", - " while not ready and elapsed < timeout:\n", - " try:\n", - " time.sleep(5)\n", - " elapsed += 5\n", - " ready = client.is_server_ready()\n", - " except Exception as e:\n", - " pass\n", + "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", + " ports=ports,\n", + " model_name=model_name)).collectAsMap()\n", + "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + ] + }, + { + "cell_type": "markdown", + "id": "e4c4017c", + "metadata": {}, + "source": [ + "#### Define client function" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "35bf6939", + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{ports[0]}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "431b864c", + "metadata": {}, + "outputs": [], + "source": [ + "def triton_fn(url, model_name):\n", + " import numpy as np\n", + " from pytriton.client import ModelClient\n", "\n", - " return [True]\n", + " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", "\n", - "nodeRDD.barrier().mapPartitions(start_triton).collect()" + " def infer_batch(inputs):\n", + " with ModelClient(url, model_name, inference_timeout_s=240) as client:\n", + " flattened = np.squeeze(inputs).tolist()\n", + " # Encode batch\n", + " encoded_batch = [[text.encode(\"utf-8\")] for text in flattened]\n", + " encoded_batch_np = np.array(encoded_batch, dtype=np.bytes_)\n", + " # Run inference\n", + " result_data = client.infer_batch(encoded_batch_np)\n", + " result_data = np.squeeze(result_data[\"outputs\"], -1)\n", + " return [json.loads(o) for o in result_data]\n", + " \n", + " return infer_batch" ] }, { "cell_type": "markdown", - "id": "c24d77ab-60d3-45eb-a9c2-dc811eca0af4", + "id": "5a8ec7be", "metadata": {}, "source": [ - "#### Run inference" + "#### Load and preprocess DataFrame" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 36, "id": "d53fb283-bf9e-4571-8c68-b75a41f1f067", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ - "# only use first sentence of IMDB reviews\n", "@pandas_udf(\"string\")\n", - "def first_sentence(text: pd.Series) -> pd.Series:\n", - " return pd.Series([s.split(\".\")[0] for s in text])\n", - "\n", - "df = spark.read.parquet(\"imdb_test\").withColumn(\"sentence\", first_sentence(col(\"lines\"))).select(\"sentence\").limit(1000)" + "def preprocess(text: pd.Series) -> pd.Series:\n", + " return pd.Series([s.split(\".\")[0] for s in text])" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 37, "id": "29b0cc0d-c480-4e4a-bd41-207dc314cba5", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/27 20:06:54 WARN CacheManager: Asked to cache already cached data.\n" + ] + } + ], "source": [ - "def triton_fn(triton_uri, model_name):\n", - " import numpy as np\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " np_types = {\n", - " \"BOOL\": np.dtype(np.bool_),\n", - " \"INT8\": np.dtype(np.int8),\n", - " \"INT16\": np.dtype(np.int16),\n", - " \"INT32\": np.dtype(np.int32),\n", - " \"INT64\": np.dtype(np.int64),\n", - " \"FP16\": np.dtype(np.float16),\n", - " \"FP32\": np.dtype(np.float32),\n", - " \"FP64\": np.dtype(np.float64),\n", - " \"FP64\": np.dtype(np.double),\n", - " \"BYTES\": np.dtype(object)\n", - " }\n", - "\n", - " client = grpcclient.InferenceServerClient(triton_uri)\n", - " model_meta = client.get_model_metadata(model_name)\n", - " \n", - " def predict(inputs):\n", - " if isinstance(inputs, np.ndarray):\n", - " # single ndarray input\n", - " request = [grpcclient.InferInput(model_meta.inputs[0].name, inputs.shape, model_meta.inputs[0].datatype)]\n", - " request[0].set_data_from_numpy(inputs.astype(np_types[model_meta.inputs[0].datatype]))\n", - " else:\n", - " # dict of multiple ndarray inputs\n", - " request = [grpcclient.InferInput(i.name, inputs[i.name].shape, i.datatype) for i in model_meta.inputs]\n", - " for i in request:\n", - " i.set_data_from_numpy(inputs[i.name()].astype(np_types[i.datatype()]))\n", - " \n", - " response = client.infer(model_name, inputs=request)\n", - " \n", - " if len(model_meta.outputs) > 1:\n", - " # return dictionary of numpy arrays\n", - " return {o.name: response.as_numpy(o.name) for o in model_meta.outputs}\n", - " else:\n", - " # return single numpy array\n", - " return response.as_numpy(model_meta.outputs[0].name)\n", - " \n", - " return predict" + "df = spark.read.parquet(data_path).limit(256).repartition(8)\n", + "df = df.select(preprocess(col(\"text\")).alias(\"input\")).cache()" + ] + }, + { + "cell_type": "markdown", + "id": "da39990f", + "metadata": {}, + "source": [ + "#### Run Inference" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 38, "id": "3930cfcd-3284-4c6a-a9b5-36b8053fe899", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ - "from functools import partial\n", - "\n", - "classify = predict_batch_udf(partial(triton_fn, triton_uri=\"localhost:8001\", model_name=\"hf_pipeline_tf\"),\n", + "classify = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", " return_type=StructType([\n", " StructField(\"label\", StringType(), True),\n", " StructField(\"score\", FloatType(), True)\n", " ]),\n", " input_tensor_shapes=[[1]],\n", - " batch_size=100)" + " batch_size=32)" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 39, "id": "8eecbf23-4e9e-4d4c-8645-98209b25db2c", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 20:> (0 + 1) / 1]\r" + "[Stage 32:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 22.5 ms, sys: 5.9 ms, total: 28.4 ms\n", - "Wall time: 24.6 s\n" + "CPU times: user 7.31 ms, sys: 4 ms, total: 11.3 ms\n", + "Wall time: 7.44 s\n" ] }, { @@ -857,33 +1158,29 @@ "%%time\n", "# first pass caches model/fn\n", "# note: expanding the \"struct\" return_type to top-level columns\n", - "preds = df.withColumn(\"preds\", classify(struct(\"sentence\"))).select(\"sentence\", \"preds.*\")\n", + "preds = df.withColumn(\"preds\", classify(struct(\"input\"))).select(\"input\", \"preds.*\")\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 40, "id": "566ba28c-0ca4-4479-a24a-c8a362228b89", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 21:> (0 + 1) / 1]\r" + "[Stage 35:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 12.2 ms, sys: 10.1 ms, total: 22.3 ms\n", - "Wall time: 23.8 s\n" + "CPU times: user 6.68 ms, sys: 1.86 ms, total: 8.54 ms\n", + "Wall time: 7.33 s\n" ] }, { @@ -896,34 +1193,29 @@ ], "source": [ "%%time\n", - "# note: expanding the \"struct\" return_type to top-level columns\n", - "preds = df.withColumn(\"preds\", classify(\"sentence\")).select(\"sentence\", \"preds.*\")\n", + "preds = df.withColumn(\"preds\", classify(\"input\")).select(\"input\", \"preds.*\")\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 41, "id": "44c7e776-08da-484a-ba07-9d6add1a0f15", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 22:> (0 + 1) / 1]\r" + "[Stage 38:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 8.74 ms, sys: 8.23 ms, total: 17 ms\n", - "Wall time: 23.8 s\n" + "CPU times: user 7.34 ms, sys: 1.15 ms, total: 8.49 ms\n", + "Wall time: 6.88 s\n" ] }, { @@ -936,56 +1228,44 @@ ], "source": [ "%%time\n", - "# note: expanding the \"struct\" return_type to top-level columns\n", - "preds = df.withColumn(\"preds\", classify(col(\"sentence\"))).select(\"sentence\", \"preds.*\")\n", + "preds = df.withColumn(\"preds\", classify(col(\"input\"))).select(\"input\", \"preds.*\")\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 42, "id": "f61d79f8-661e-4d9e-a3aa-c0754b854603", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 23:> (0 + 1) / 1]\r" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+----------+\n", - "|sentence |label |score |\n", - "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+----------+\n", - "| |POSITIVE|0.74807304|\n", - "|Hard up, No proper jobs going down at the pit, why not rent your kids! DIY pimp story without the gratuitous sex scenes, either hard core or soft core, therefore reads like a public information film from the fifties, give this a wide miss, use a barge pole if you can|NEGATIVE|0.9996724 |\n", - "|I watched this movie to see the direction one of the most promising young talents in movies was going |POSITIVE|0.9994948 |\n", - "|This movie makes you wish imdb would let you vote a zero |NEGATIVE|0.9981299 |\n", - "|I never want to see this movie again!

Not only is it dreadfully bad, but I can't stand seeing my hero Stan Laurel looking so old and sick |NEGATIVE|0.99883264|\n", - "|(As a note, I'd like to say that I saw this movie at my annual church camp, where the entire youth group laughed at it |POSITIVE|0.9901753 |\n", - "|Don't get me wrong, I love the TV series of League Of Gentlemen |POSITIVE|0.99983096|\n", - "|Did you ever think, like after watching a horror movie with a group of friends: \"Wow, this is so cool! We have got to make a splatter horror movie ourselves some day soon |POSITIVE|0.9992768 |\n", - "|Awful, awful, awful |NEGATIVE|0.9997433 |\n", - "|This movie seems a little clunky around the edges, like not quite enough zaniness was thrown it when it should have been |NEGATIVE|0.9996525 |\n", - "|I rented this movie hoping that it would provide some good entertainment and some cool poker knowledge or stories |NEGATIVE|0.99643254|\n", - "|Well, where to start describing this celluloid debacle? You already know the big fat NADA passing as a plot, so let's jut point out that this is so PC it's offensive |NEGATIVE|0.99973005|\n", - "|I hoped for this show to be somewhat realistic |POSITIVE|0.8417903 |\n", - "|All I have to say is one word |NEGATIVE|0.97844803|\n", - "|Honestly awful film, bad editing, awful lighting, dire dialog and scrappy screenplay |NEGATIVE|0.9997701 |\n", - "|This critique tells the story of 4 little friends who went to watch Angels and Demons the movie on the first night it came out, even though it was a school night, because \"Angels and Demons is worth it |POSITIVE|0.9942386 |\n", - "|This review contains a partial spoiler |NEGATIVE|0.99620205|\n", - "|I'm rather surprised that anybody found this film touching or moving |POSITIVE|0.83874947|\n", - "|If you like bad movies (and you must to watch this one) here's a good one |POSITIVE|0.9936475 |\n", - "|This is really bad, the characters were bland, the story was boring, and there is no sex scene |NEGATIVE|0.99953806|\n", - "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+----------+\n", + "+--------------------------------------------------------------------------------+--------+----------+\n", + "| input| label| score|\n", + "+--------------------------------------------------------------------------------+--------+----------+\n", + "|Doesn't anyone bother to check where this kind of sludge comes from before bl...|NEGATIVE| 0.9984061|\n", + "| There were two things I hated about WASTED : The directing and the script |NEGATIVE| 0.9979007|\n", + "| I'm rather surprised that anybody found this film touching or moving|POSITIVE|0.83874947|\n", + "|Cultural Vandalism Is the new Hallmark production of Gulliver's Travels an ac...|NEGATIVE|0.99727434|\n", + "|I was at Wrestlemania VI in Toronto as a 10 year old, and the event I saw the...|POSITIVE| 0.982114|\n", + "| This movie has been done before|NEGATIVE|0.94210696|\n", + "|[ as a new resolution for this year 2005, i decide to write a comment for eac...|NEGATIVE| 0.9967818|\n", + "|This movie is over hyped!! I am sad to say that I manage to watch the first 1...|NEGATIVE| 0.9985843|\n", + "|This show had a promising start as sort of the opposite of 'Oceans 11' but ha...|NEGATIVE|0.99926835|\n", + "|MINOR PLOT SPOILERS AHEAD!!!

How did such talented actors get invo...|NEGATIVE|0.99956733|\n", + "| There is not one character on this sitcom with any redeeming qualities|NEGATIVE| 0.9985662|\n", + "| Tommy Lee Jones was the best Woodroe and no one can play Woodroe F|POSITIVE| 0.994562|\n", + "| My wife rented this movie and then conveniently never got to see it|NEGATIVE|0.99841607|\n", + "|This is one of those star-filled over-the-top comedies that could a) be hyste...|NEGATIVE| 0.9953243|\n", + "|This excruciatingly boring and unfunny movie made me think that Chaplin was t...|NEGATIVE| 0.9997607|\n", + "| you will likely be sorely disappointed by this sequel that's not a sequel|NEGATIVE| 0.9997198|\n", + "| If I was British, I would be embarrassed by this portrayal of incompetence|NEGATIVE| 0.9965172|\n", + "|One of those movies in which there are no big twists whatsoever and you can p...|NEGATIVE| 0.9986059|\n", + "| This show is like watching someone who is in training to someday host a show|NEGATIVE|0.97015846|\n", + "| Sigh|NEGATIVE| 0.9923151|\n", + "+--------------------------------------------------------------------------------+--------+----------+\n", "only showing top 20 rows\n", "\n" ] @@ -999,29 +1279,30 @@ } ], "source": [ - "preds.show(truncate=False)" + "preds.show(truncate=80)" ] }, { "cell_type": "markdown", - "id": "e197c146-1794-47f0-bcd9-7e8d8ab8625f", - "metadata": { - "tags": [] - }, + "id": "fac2ae57", + "metadata": {}, "source": [ - "#### Stop Triton Server on each executor" + "#### Shut down server on each executor" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 43, "id": "425d3b28-7705-45ba-8a18-ad34fc895219", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + }, { "name": "stderr", "output_type": "stream", @@ -1035,36 +1316,26 @@ "[True]" ] }, - "execution_count": 32, + "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "def stop_triton(it):\n", - " import docker\n", - " import time\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " print(\">>>> stopping containers: {}\".format([c.short_id for c in containers]))\n", - " if containers:\n", - " container=containers[0]\n", - " container.stop(timeout=120)\n", - "\n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(stop_triton).collect()" + "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", + "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 44, "id": "9f19643c-4ee4-44f2-b762-2078c0c8eba9", "metadata": {}, "outputs": [], "source": [ - "spark.stop()" + "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n", + " spark.stop()" ] }, { diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb index 1e99ed36..0a4c8340 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb @@ -5,15 +5,18 @@ "id": "60f7ac5d-4a95-4170-a0ac-a7faac9d9ef4", "metadata": {}, "source": [ + "\n", + "\n", "# PySpark Huggingface Inferencing\n", - "### Text Classification using Pipelines with PyTorch\n", + "### Sentiment Analysis using Pipelines with PyTorch\n", "\n", - "Based on: https://huggingface.co/docs/transformers/quicktour#pipeline-usage" + "In this notebook, we demonstrate distributed inference with Huggingface Pipelines to perform sentiment analysis. \n", + "From: https://huggingface.co/docs/transformers/quicktour#pipeline-usage" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "0dd0f77b-ee1b-4477-a038-d25a4f1da0ea", "metadata": {}, "outputs": [], @@ -24,18 +27,17 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "e1f756c6", "metadata": {}, "outputs": [], "source": [ - "# set device if gpu is available\n", - "device = 0 if torch.cuda.is_available() else -1" + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "553b28d2-a5d1-4d07-8a49-8f82b808e738", "metadata": {}, "outputs": [ @@ -54,7 +56,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "3b91fe91-b725-4564-ae93-56e3fb51e47c", "metadata": {}, "outputs": [ @@ -64,7 +66,7 @@ "[{'label': 'POSITIVE', 'score': 0.9997795224189758}]" ] }, - "execution_count": 5, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -75,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "0be39eb3-462c-42ff-b8f4-09f4e4fe3a3c", "metadata": {}, "outputs": [ @@ -99,12 +101,12 @@ "id": "f752f929", "metadata": {}, "source": [ - "#### Use another model and tokenizer in the pipeline" + "Let's try a different model and tokenizer in the pipeline." ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "id": "9861865f", "metadata": {}, "outputs": [], @@ -114,7 +116,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "id": "506e7834", "metadata": {}, "outputs": [], @@ -127,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "id": "312017fc", "metadata": {}, "outputs": [ @@ -137,7 +139,7 @@ "[{'label': '5 stars', 'score': 0.7272652983665466}]" ] }, - "execution_count": 9, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -152,154 +154,322 @@ "id": "ae92b15e-0da0-46c3-81a3-fabaedbfc42c", "metadata": {}, "source": [ - "## Inference using Spark DL API" + "## PySpark" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "id": "69dd6a1a-f450-47f0-9dbf-ad250585a011", "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", "from pyspark.sql.functions import col, struct, pandas_udf\n", "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.types import FloatType, StringType, StructField, StructType\n", + "from pyspark.sql.types import *\n", "from pyspark.sql import SparkSession\n", - "from pyspark import SparkConf\n", - "import os" + "from pyspark import SparkConf" ] }, { "cell_type": "code", - "execution_count": null, - "id": "6e0e0dd7", + "execution_count": 10, + "id": "42c19ad8", "metadata": {}, "outputs": [], "source": [ - "conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + "import os\n", + "import json\n", + "import pandas as pd\n", + "import datasets\n", + "from datasets import load_dataset\n", + "datasets.disable_progress_bars()" + ] + }, + { + "cell_type": "markdown", + "id": "3f1a0210", + "metadata": {}, + "source": [ + "Check the cluster environment to handle any platform-specific Spark configurations." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "79aaf5ec", + "metadata": {}, + "outputs": [], + "source": [ + "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n", + "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n", + "on_standalone = not (on_databricks or on_dataproc)" + ] + }, + { + "cell_type": "markdown", + "id": "b99f9c38", + "metadata": {}, + "source": [ + "#### Create Spark Session\n", "\n", + "For local standalone clusters, we'll connect to the cluster and create the Spark Session. \n", + "For CSP environments, Spark will either be preconfigured (Databricks) or we'll need to create the Spark Session (Dataproc)." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "6e0e0dd7", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/27 19:43:21 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/01/27 19:43:21 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", + "25/01/27 19:43:21 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "25/01/27 19:43:22 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + ] + } + ], + "source": [ "conf = SparkConf()\n", + "\n", "if 'spark' not in globals():\n", - " # If Spark is not already started with Jupyter, attach to Spark Standalone\n", - " import socket\n", - " hostname = socket.gethostname()\n", - " conf.setMaster(f\"spark://{hostname}:7077\") # assuming Master is on default port 7077\n", - "conf.set(\"spark.task.maxFailures\", \"1\")\n", - "conf.set(\"spark.driver.memory\", \"8g\")\n", - "conf.set(\"spark.executor.memory\", \"8g\")\n", - "conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled\", \"false\")\n", - "conf.set(\"spark.sql.pyspark.jvmStacktrace.enabled\", \"true\")\n", - "conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", - "conf.set(\"spark.python.worker.reuse\", \"true\")\n", - "# Create Spark Session\n", + " if on_standalone:\n", + " import socket\n", + " conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + " hostname = socket.gethostname()\n", + " conf.setMaster(f\"spark://{hostname}:7077\")\n", + " conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", + " conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_env}/lib:{conda_env}/lib/python3.11/site-packages/nvidia_pytriton.libs:$LD_LIBRARY_PATH\")\n", + " elif on_dataproc:\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conda_lib_path=\"/opt/conda/miniconda3/lib\"\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_lib_path}:$LD_LIBRARY_PATH\")\n", + " conf.set(\"spark.executor.instances\", \"4\") # dataproc defaults to 2\n", + "\n", + " conf.set(\"spark.executor.cores\", \"8\")\n", + " conf.set(\"spark.task.resource.gpu.amount\", \"0.125\")\n", + " conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n", + " conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", + " conf.set(\"spark.python.worker.reuse\", \"true\")\n", + "\n", + "conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"1000\")\n", "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", "sc = spark.sparkContext" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "id": "42d70208", "metadata": {}, "outputs": [], "source": [ - "from datasets import load_dataset\n", - "\n", - "# Load the IMDB dataset\n", - "data = load_dataset(\"imdb\", split=\"test\")\n", - "\n", - "lines = []\n", - "for example in data:\n", - " # first sentence only\n", - " lines.append([example[\"text\"]])\n", - "\n", - "len(lines)\n", - "\n", - "df = spark.createDataFrame(lines, ['lines']).repartition(8).cache()" + "dataset = load_dataset(\"imdb\", split=\"test\")\n", + "dataset = dataset.to_pandas().drop(columns=\"label\")" + ] + }, + { + "cell_type": "markdown", + "id": "de0f421d", + "metadata": {}, + "source": [ + "#### Create PySpark DataFrame" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "id": "ac24f3c2", "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "StructType([StructField('text', StringType(), True)])" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = spark.createDataFrame(dataset).repartition(8)\n", + "df.schema" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "b0d1876b", + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "24/10/03 16:44:02 WARN TaskSetManager: Stage 0 contains a task of very large size (3860 KiB). The maximum recommended task size is 1000 KiB.\n", " \r" ] + }, + { + "data": { + "text/plain": [ + "25000" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "df.write.mode(\"overwrite\").parquet(\"imdb_test\")" + "df.count()" ] }, { "cell_type": "code", - "execution_count": 14, - "id": "9665b7b6-d7e9-4bd4-b29d-7a449ac5b574", + "execution_count": 16, + "id": "06ec6bb6", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "25/01/27 19:43:28 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] }, + { + "data": { + "text/plain": [ + "[Row(text=\"Anyone remember the first CKY, CKY2K etc..? Back when it was about making crazy cool stuff, rather than watching Bam Margera act like a douchebag, spoiled 5 year old, super/rock-star wannabe.

The show used to be awesome, however, Bam's fame and wealth has led him to believe, that we now enjoy him acting childish and idiotic, more than actual cool stuff, that used to be in ex. CKY2K.

The acts are so repetitive, there's like nothing new, except annoying stupidity and rehearsed comments... The only things we see is Bam Margera, so busy showing us how much he doesn't care, how much money he got or whatsoever.

I really got nothing much left to say except, give us back CKY2K, cause Bam suck..

I enjoy watching Steve-o, Knoxville etc. a thousand times more.\")]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.take(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "eeadf4e2", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/27 19:43:29 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + ] + } + ], + "source": [ + "data_path = \"spark-dl-datasets/imdb_test\"\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " data_path = \"dbfs:/FileStore/\" + data_path\n", + "\n", + "df.write.mode(\"overwrite\").parquet(data_path)" + ] + }, + { + "cell_type": "markdown", + "id": "09cddc95", + "metadata": {}, + "source": [ + "#### Load and preprocess DataFrame\n", + "\n", + "Define our preprocess function. We'll take the first sentence from each sample as our input for sentiment analysis." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "9665b7b6-d7e9-4bd4-b29d-7a449ac5b574", + "metadata": {}, + "outputs": [], + "source": [ + "@pandas_udf(\"string\")\n", + "def preprocess(text: pd.Series) -> pd.Series:\n", + " return pd.Series([s.split(\".\")[0] for s in text])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "74cfa3ff", + "metadata": {}, + "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+--------------------------------------------------------------------------------+\n", - "| sentence|\n", - "+--------------------------------------------------------------------------------+\n", - "| |\n", - "|Hard up, No proper jobs going down at the pit, why not rent your kids! DIY pi...|\n", - "|I watched this movie to see the direction one of the most promising young tal...|\n", - "| This movie makes you wish imdb would let you vote a zero|\n", - "|I never want to see this movie again!

Not only is it dreadfully ba...|\n", - "|(As a note, I'd like to say that I saw this movie at my annual church camp, w...|\n", - "| Don't get me wrong, I love the TV series of League Of Gentlemen|\n", - "|Did you ever think, like after watching a horror movie with a group of friend...|\n", - "| Awful, awful, awful|\n", - "|This movie seems a little clunky around the edges, like not quite enough zani...|\n", - "|I rented this movie hoping that it would provide some good entertainment and ...|\n", - "|Well, where to start describing this celluloid debacle? You already know the ...|\n", - "| I hoped for this show to be somewhat realistic|\n", - "| All I have to say is one word|\n", - "|Honestly awful film, bad editing, awful lighting, dire dialog and scrappy scr...|\n", - "|This critique tells the story of 4 little friends who went to watch Angels an...|\n", - "| This review contains a partial spoiler|\n", - "| I'm rather surprised that anybody found this film touching or moving|\n", - "| If you like bad movies (and you must to watch this one) here's a good one|\n", - "|This is really bad, the characters were bland, the story was boring, and ther...|\n", - "+--------------------------------------------------------------------------------+\n", + "+----------------------------------------------------------------------------------------------------+\n", + "| input|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "|Doesn't anyone bother to check where this kind of sludge comes from before blathering on about it...|\n", + "| There were two things I hated about WASTED : The directing and the script |\n", + "| I'm rather surprised that anybody found this film touching or moving|\n", + "|Cultural Vandalism Is the new Hallmark production of Gulliver's Travels an act of cultural vandal...|\n", + "|I was at Wrestlemania VI in Toronto as a 10 year old, and the event I saw then was pretty differe...|\n", + "| This movie has been done before|\n", + "|[ as a new resolution for this year 2005, i decide to write a comment for each movie I saw in the...|\n", + "|This movie is over hyped!! I am sad to say that I manage to watch the first 15 minutes of this mo...|\n", + "|This show had a promising start as sort of the opposite of 'Oceans 11' but has developed into a s...|\n", + "|MINOR PLOT SPOILERS AHEAD!!!

How did such talented actors get involved in such mindles...|\n", + "| There is not one character on this sitcom with any redeeming qualities|\n", + "| Tommy Lee Jones was the best Woodroe and no one can play Woodroe F|\n", + "| My wife rented this movie and then conveniently never got to see it|\n", + "|This is one of those star-filled over-the-top comedies that could a) be hysterical, or b) wish th...|\n", + "|This excruciatingly boring and unfunny movie made me think that Chaplin was the real Hitler, as o...|\n", + "| you will likely be sorely disappointed by this sequel that's not a sequel|\n", + "| If I was British, I would be embarrassed by this portrayal of incompetence|\n", + "|One of those movies in which there are no big twists whatsoever and you can predict pretty much w...|\n", + "| This show is like watching someone who is in training to someday host a show|\n", + "| Sigh|\n", + "+----------------------------------------------------------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] } ], "source": [ - "# only use first sentence of IMDB reviews\n", - "@pandas_udf(\"string\")\n", - "def first_sentence(text: pd.Series) -> pd.Series:\n", - " return pd.Series([s.split(\".\")[0] for s in text])\n", + "# Limit to N rows, since this can be slow\n", + "df = spark.read.parquet(data_path).limit(256).repartition(8)\n", + "df = df.select(preprocess(col(\"text\")).alias(\"input\")).cache()\n", + "df.show(truncate=100)" + ] + }, + { + "cell_type": "markdown", + "id": "1ad92750", + "metadata": {}, + "source": [ + "## Inference using Spark DL API\n", + "\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", "\n", - "df = spark.read.parquet(\"imdb_test\").withColumn(\"sentence\", first_sentence(col(\"lines\"))).select(\"sentence\").limit(100).cache()\n", - "df.show(truncate=80)" + "- predict_batch_fn uses PyTorch APIs to load the model and return a predict function which operates on numpy arrays \n", + "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 20, "id": "0da9d25c-5ebe-4503-bb19-154fcc047cbf", "metadata": {}, "outputs": [], @@ -308,7 +478,7 @@ " import torch\n", " from transformers import pipeline\n", " \n", - " device = 0 if torch.cuda.is_available() else -1\n", + " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", " pipe = pipeline(\"sentiment-analysis\", device=device)\n", " def predict(inputs):\n", " return pipe(inputs.tolist())\n", @@ -317,7 +487,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 21, "id": "78afef29-ee30-4267-9fb6-be2dcb86cbba", "metadata": {}, "outputs": [], @@ -327,12 +497,12 @@ " StructField(\"label\", StringType(), True),\n", " StructField(\"score\", FloatType(), True)\n", " ]),\n", - " batch_size=10)" + " batch_size=32)" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 22, "id": "a5bc327e-89cf-4731-82e6-e66cb93deef1", "metadata": {}, "outputs": [ @@ -340,15 +510,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 11:> (0 + 1) / 1]\r" + "[Stage 18:=====================> (3 + 5) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 12.6 ms, sys: 2.39 ms, total: 15 ms\n", - "Wall time: 2.02 s\n" + "CPU times: user 8.71 ms, sys: 5.88 ms, total: 14.6 ms\n", + "Wall time: 3.03 s\n" ] }, { @@ -361,14 +531,15 @@ ], "source": [ "%%time\n", + "# first pass caches model/fn\n", "# note: expanding the \"struct\" return_type to top-level columns\n", - "preds = df.withColumn(\"preds\", classify(struct(\"sentence\"))).select(\"sentence\", \"preds.*\")\n", + "preds = df.withColumn(\"preds\", classify(struct(\"input\"))).select(\"input\", \"preds.*\")\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 23, "id": "ac642895-cfd6-47ee-9b21-02e7835424e4", "metadata": {}, "outputs": [ @@ -376,21 +547,20 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.13 ms, sys: 1.06 ms, total: 3.19 ms\n", - "Wall time: 237 ms\n" + "CPU times: user 3.41 ms, sys: 1.2 ms, total: 4.61 ms\n", + "Wall time: 391 ms\n" ] } ], "source": [ "%%time\n", - "# note: expanding the \"struct\" return_type to top-level columns\n", - "preds = df.withColumn(\"preds\", classify(\"sentence\")).select(\"sentence\", \"preds.*\")\n", + "preds = df.withColumn(\"preds\", classify(\"input\")).select(\"input\", \"preds.*\")\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 24, "id": "76a44d80-d5db-405f-989c-7246379cfb95", "metadata": {}, "outputs": [ @@ -398,21 +568,20 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.28 ms, sys: 790 μs, total: 3.07 ms\n", - "Wall time: 230 ms\n" + "CPU times: user 4.57 ms, sys: 291 μs, total: 4.87 ms\n", + "Wall time: 409 ms\n" ] } ], "source": [ "%%time\n", - "# note: expanding the \"struct\" return_type to top-level columns\n", - "preds = df.withColumn(\"preds\", classify(col(\"sentence\"))).select(\"sentence\", \"preds.*\")\n", + "preds = df.withColumn(\"preds\", classify(col(\"input\"))).select(\"input\", \"preds.*\")\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 25, "id": "c01761b3-c766-46b0-ae0b-fcf968ffb3a1", "metadata": {}, "outputs": [ @@ -421,28 +590,28 @@ "output_type": "stream", "text": [ "+--------------------------------------------------------------------------------+--------+----------+\n", - "| sentence| label| score|\n", + "| input| label| score|\n", "+--------------------------------------------------------------------------------+--------+----------+\n", - "| |POSITIVE| 0.7481212|\n", - "|Hard up, No proper jobs going down at the pit, why not rent your kids! DIY pi...|NEGATIVE|0.99967253|\n", - "|I watched this movie to see the direction one of the most promising young tal...|POSITIVE| 0.9994943|\n", - "| This movie makes you wish imdb would let you vote a zero|NEGATIVE| 0.9981305|\n", - "|I never want to see this movie again!

Not only is it dreadfully ba...|NEGATIVE| 0.9988337|\n", - "|(As a note, I'd like to say that I saw this movie at my annual church camp, w...|POSITIVE| 0.9901974|\n", - "| Don't get me wrong, I love the TV series of League Of Gentlemen|POSITIVE| 0.9998311|\n", - "|Did you ever think, like after watching a horror movie with a group of friend...|POSITIVE| 0.9992779|\n", - "| Awful, awful, awful|NEGATIVE| 0.9997433|\n", - "|This movie seems a little clunky around the edges, like not quite enough zani...|NEGATIVE|0.99965274|\n", - "|I rented this movie hoping that it would provide some good entertainment and ...|NEGATIVE|0.99642426|\n", - "|Well, where to start describing this celluloid debacle? You already know the ...|NEGATIVE|0.99973005|\n", - "| I hoped for this show to be somewhat realistic|POSITIVE| 0.8426496|\n", - "| All I have to say is one word|NEGATIVE| 0.9784491|\n", - "|Honestly awful film, bad editing, awful lighting, dire dialog and scrappy scr...|NEGATIVE| 0.99977|\n", - "|This critique tells the story of 4 little friends who went to watch Angels an...|POSITIVE| 0.9942334|\n", - "| This review contains a partial spoiler|NEGATIVE| 0.996191|\n", + "|Doesn't anyone bother to check where this kind of sludge comes from before bl...|NEGATIVE| 0.9984042|\n", + "| There were two things I hated about WASTED : The directing and the script |NEGATIVE| 0.9979019|\n", "| I'm rather surprised that anybody found this film touching or moving|POSITIVE| 0.8392794|\n", - "| If you like bad movies (and you must to watch this one) here's a good one|POSITIVE|0.99366415|\n", - "|This is really bad, the characters were bland, the story was boring, and ther...|NEGATIVE|0.99953806|\n", + "|Cultural Vandalism Is the new Hallmark production of Gulliver's Travels an ac...|NEGATIVE|0.99726933|\n", + "|I was at Wrestlemania VI in Toronto as a 10 year old, and the event I saw the...|POSITIVE|0.98212516|\n", + "| This movie has been done before|NEGATIVE|0.94194806|\n", + "|[ as a new resolution for this year 2005, i decide to write a comment for eac...|NEGATIVE|0.99678314|\n", + "|This movie is over hyped!! I am sad to say that I manage to watch the first 1...|NEGATIVE| 0.9985846|\n", + "|This show had a promising start as sort of the opposite of 'Oceans 11' but ha...|NEGATIVE|0.99926823|\n", + "|MINOR PLOT SPOILERS AHEAD!!!

How did such talented actors get invo...|NEGATIVE| 0.9995671|\n", + "| There is not one character on this sitcom with any redeeming qualities|NEGATIVE|0.99856514|\n", + "| Tommy Lee Jones was the best Woodroe and no one can play Woodroe F|POSITIVE| 0.9945687|\n", + "| My wife rented this movie and then conveniently never got to see it|NEGATIVE| 0.9984137|\n", + "|This is one of those star-filled over-the-top comedies that could a) be hyste...|NEGATIVE| 0.9953224|\n", + "|This excruciatingly boring and unfunny movie made me think that Chaplin was t...|NEGATIVE| 0.9997607|\n", + "| you will likely be sorely disappointed by this sequel that's not a sequel|NEGATIVE|0.99971956|\n", + "| If I was British, I would be embarrassed by this portrayal of incompetence|NEGATIVE|0.99651587|\n", + "|One of those movies in which there are no big twists whatsoever and you can p...|NEGATIVE|0.99860746|\n", + "| This show is like watching someone who is in training to someday host a show|NEGATIVE| 0.970153|\n", + "| Sigh|NEGATIVE|0.99231356|\n", "+--------------------------------------------------------------------------------+--------+----------+\n", "only showing top 20 rows\n", "\n" @@ -455,296 +624,370 @@ }, { "cell_type": "markdown", - "id": "eb826fde-99d9-43fe-8ddc-f5acbe76b4e9", + "id": "8ba1a6ce", "metadata": {}, "source": [ - "### Using Triton Inference Server\n", + "## Using Triton Inference Server\n", + "In this section, we demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. \n", + "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server. \n", + "\n", + "The process looks like this:\n", + "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n", + "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n", + "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n", + "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n", "\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment." + "\"drawing\"" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "4d4be844-4b8c-47df-bd09-0c280c7ff16b", + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial" ] }, { "cell_type": "markdown", - "id": "10368010-f94d-4167-91a1-2cf9ed91a2c9", + "id": "42867a0b", "metadata": {}, "source": [ - "This notebook uses the [Python backend with a custom execution environment](https://github.com/triton-inference-server/python_backend#creating-custom-execution-environments) with the compatible versions of Python/Numpy for Triton 24.08, using a conda-pack environment created as follows:\n", - "```\n", - "conda create -n huggingface-torch -c conda-forge python=3.10.0\n", - "conda activate huggingface-torch\n", - "\n", - "export PYTHONNOUSERSITE=True\n", - "pip install numpy==1.26.4 conda-pack sentencepiece sentence_transformers transformers\n", - "\n", - "conda-pack # huggingface-torch.tar.gz\n", - "```" + "Import the utility functions from pytriton_utils.py:" ] }, { "cell_type": "code", - "execution_count": 21, - "id": "4d4be844-4b8c-47df-bd09-0c280c7ff16b", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 27, + "id": "4e6764c4", + "metadata": {}, "outputs": [], "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "import os\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import col, struct, pandas_udf\n", - "from pyspark.sql.types import FloatType, StringType, StructField, StructType" + "sc.addPyFile(\"pytriton_utils.py\")\n", + "\n", + "from pytriton_utils import (\n", + " use_stage_level_scheduling,\n", + " find_ports,\n", + " start_triton,\n", + " stop_triton\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "bab70481", + "metadata": {}, + "source": [ + "Define the Triton Server function:" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 28, "id": "7e53df9f-43cb-4c38-b8ac-dc2cbad99815", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ - "%%bash\n", - "# copy custom model to expected layout for Triton\n", - "rm -rf models\n", - "mkdir -p models\n", - "cp -r models_config/hf_pipeline_torch models\n", + "def triton_server(ports):\n", + " import time\n", + " import signal\n", + " import numpy as np\n", + " import torch\n", + " from transformers import pipeline\n", + " from pytriton.decorators import batch\n", + " from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n", + " from pytriton.triton import Triton, TritonConfig\n", + " from pyspark import TaskContext\n", + "\n", + " print(f\"SERVER: Initializing pipeline on worker {TaskContext.get().partitionId()}.\")\n", + " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + " pipe = pipeline(\"sentiment-analysis\", device=device)\n", + " print(f\"SERVER: Using {device} device.\")\n", + "\n", + " @batch\n", + " def _infer_fn(**inputs):\n", + " sentences = np.squeeze(inputs[\"text\"]).tolist()\n", + " print(f\"SERVER: Received batch of size {len(sentences)}\")\n", + " decoded_sentences = [s.decode(\"utf-8\") for s in sentences]\n", + " return {\n", + " \"outputs\": np.array([[json.dumps(o)] for o in pipe(decoded_sentences)])\n", + " }\n", "\n", - "# add custom execution environment\n", - "cp huggingface-torch.tar.gz models" + " workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n", + " triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n", + " with Triton(config=triton_conf, workspace=workspace_path) as triton:\n", + " triton.bind(\n", + " model_name=\"SentimentAnalysis\",\n", + " infer_func=_infer_fn,\n", + " inputs=[\n", + " Tensor(name=\"text\", dtype=object, shape=(-1,)),\n", + " ],\n", + " outputs=[\n", + " Tensor(name=\"outputs\", dtype=object, shape=(-1,)),\n", + " ],\n", + " config=ModelConfig(\n", + " max_batch_size=64,\n", + " batcher=DynamicBatcher(max_queue_delay_microseconds=5000), # 5ms\n", + " ),\n", + " strict=True,\n", + " )\n", + "\n", + " def _stop_triton(signum, frame):\n", + " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", + " triton.stop()\n", + "\n", + " signal.signal(signal.SIGTERM, _stop_triton)\n", + "\n", + " print(\"SERVER: Serving inference\")\n", + " triton.serve()" ] }, { "cell_type": "markdown", - "id": "db4a5b06-126a-4bc4-baae-a45ea30832a7", - "metadata": { - "tags": [] - }, + "id": "7c5f4f2d", + "metadata": {}, + "source": [ + "#### Start Triton servers" + ] + }, + { + "cell_type": "markdown", + "id": "5463c517", + "metadata": {}, "source": [ - "#### Start Triton Server on each executor" + "**Specify the number of nodes in the cluster.** \n", + "Following the README, the example standalone cluster uses 1 node. The example Databricks/Dataproc cluster scripts use 4 nodes by default. " ] }, { "cell_type": "code", - "execution_count": 23, - "id": "144acb8e-4c08-40fc-a9ed-f721c409ee68", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 29, + "id": "a4757163", + "metadata": {}, + "outputs": [], + "source": [ + "# Change based on cluster setup\n", + "num_nodes = 1 if on_standalone else 4" + ] + }, + { + "cell_type": "markdown", + "id": "767c40e6", + "metadata": {}, + "source": [ + "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "ad13db78", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + } + ], + "source": [ + "sc = spark.sparkContext\n", + "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" + ] + }, + { + "cell_type": "markdown", + "id": "5febf6e8", + "metadata": {}, + "source": [ + "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "79a4e9d7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using ports [7000, 7001, 7002]\n" + ] + } + ], + "source": [ + "model_name = \"SentimentAnalysis\"\n", + "\n", + "ports = find_ports()\n", + "assert len(ports) == 3\n", + "print(f\"Using ports {ports}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a1a4c4c", + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "[Stage 28:> (0 + 1) / 1]\r" ] }, { - "data": { - "text/plain": [ - "[True]" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "Triton Server PIDs:\n", + " {\n", + " \"cb4ae00-lcedt\": 2956583\n", + "}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] } ], "source": [ - "num_executors = 1\n", - "triton_models_dir = \"{}/models\".format(os.getcwd())\n", - "huggingface_cache_dir = \"{}/.cache/huggingface\".format(os.path.expanduser('~'))\n", - "nodeRDD = sc.parallelize(list(range(num_executors)), num_executors)\n", - "\n", - "def start_triton(it):\n", - " import docker\n", - " import time\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " if containers:\n", - " print(\">>>> containers: {}\".format([c.short_id for c in containers]))\n", - " else:\n", - " container=client.containers.run(\n", - " \"nvcr.io/nvidia/tritonserver:24.08-py3\", \"tritonserver --model-repository=/models\",\n", - " detach=True,\n", - " device_requests=[docker.types.DeviceRequest(device_ids=[\"0\"], capabilities=[['gpu']])],\n", - " environment=[\n", - " \"TRANSFORMERS_CACHE=/cache\"\n", - " ],\n", - " name=\"spark-triton\",\n", - " network_mode=\"host\",\n", - " remove=True,\n", - " shm_size=\"256M\",\n", - " volumes={\n", - " triton_models_dir: {\"bind\": \"/models\", \"mode\": \"ro\"},\n", - " huggingface_cache_dir: {\"bind\": \"/cache\", \"mode\": \"rw\"}\n", - " }\n", - " )\n", - " print(\">>>> starting triton: {}\".format(container.short_id))\n", - " # wait for triton to be running\n", - " time.sleep(15)\n", - " \n", - " client = grpcclient.InferenceServerClient(\"localhost:8001\")\n", - " \n", - " elapsed = 0\n", - " timeout = 120\n", - " ready = False\n", - " while not ready and elapsed < timeout:\n", - " try:\n", - " time.sleep(5)\n", - " elapsed += 5\n", - " ready = client.is_server_ready()\n", - " except Exception as e:\n", - " pass\n", - "\n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(start_triton).collect()" + "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", + " ports=ports,\n", + " model_name=model_name)).collectAsMap()\n", + "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" ] }, { "cell_type": "markdown", - "id": "c24d77ab-60d3-45eb-a9c2-dc811eca0af4", + "id": "f5ae0b8e", "metadata": {}, "source": [ - "#### Run inference" + "#### Define client function" ] }, { "cell_type": "code", - "execution_count": 24, - "id": "d53fb283-bf9e-4571-8c68-b75a41f1f067", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 33, + "id": "f6899d96", + "metadata": {}, "outputs": [], "source": [ - "# only use first sentence of IMDB reviews\n", - "@pandas_udf(\"string\")\n", - "def first_sentence(text: pd.Series) -> pd.Series:\n", - " return pd.Series([s.split(\".\")[0] for s in text])\n", - "\n", - "df = spark.read.parquet(\"imdb_test\").withColumn(\"sentence\", first_sentence(col(\"lines\"))).select(\"sentence\").limit(1000)" + "url = f\"http://localhost:{ports[0]}\"" ] }, { "cell_type": "code", - "execution_count": 25, - "id": "29b0cc0d-c480-4e4a-bd41-207dc314cba5", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 34, + "id": "14760940", + "metadata": {}, "outputs": [], "source": [ - "def triton_fn(triton_uri, model_name):\n", + "def triton_fn(url, model_name):\n", " import numpy as np\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " np_types = {\n", - " \"BOOL\": np.dtype(np.bool_),\n", - " \"INT8\": np.dtype(np.int8),\n", - " \"INT16\": np.dtype(np.int16),\n", - " \"INT32\": np.dtype(np.int32),\n", - " \"INT64\": np.dtype(np.int64),\n", - " \"FP16\": np.dtype(np.float16),\n", - " \"FP32\": np.dtype(np.float32),\n", - " \"FP64\": np.dtype(np.float64),\n", - " \"FP64\": np.dtype(np.double),\n", - " \"BYTES\": np.dtype(object)\n", - " }\n", + " from pytriton.client import ModelClient\n", "\n", - " client = grpcclient.InferenceServerClient(triton_uri)\n", - " model_meta = client.get_model_metadata(model_name)\n", - " \n", - " def predict(inputs):\n", - " if isinstance(inputs, np.ndarray):\n", - " # single ndarray input\n", - " request = [grpcclient.InferInput(model_meta.inputs[0].name, inputs.shape, model_meta.inputs[0].datatype)]\n", - " request[0].set_data_from_numpy(inputs.astype(np_types[model_meta.inputs[0].datatype]))\n", - " else:\n", - " # dict of multiple ndarray inputs\n", - " request = [grpcclient.InferInput(i.name, inputs[i.name].shape, i.datatype) for i in model_meta.inputs]\n", - " for i in request:\n", - " i.set_data_from_numpy(inputs[i.name()].astype(np_types[i.datatype()]))\n", - " \n", - " response = client.infer(model_name, inputs=request)\n", - " \n", - " if len(model_meta.outputs) > 1:\n", - " # return dictionary of numpy arrays\n", - " return {o.name: response.as_numpy(o.name) for o in model_meta.outputs}\n", - " else:\n", - " # return single numpy array\n", - " return response.as_numpy(model_meta.outputs[0].name)\n", + " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", + "\n", + " def infer_batch(inputs):\n", + " with ModelClient(url, model_name, inference_timeout_s=240) as client:\n", + " flattened = np.squeeze(inputs).tolist()\n", + " # Encode batch\n", + " encoded_batch = [[text.encode(\"utf-8\")] for text in flattened]\n", + " encoded_batch_np = np.array(encoded_batch, dtype=np.bytes_)\n", + " # Run inference\n", + " result_data = client.infer_batch(encoded_batch_np)\n", + " result_data = np.squeeze(result_data[\"outputs\"], -1)\n", + " return [json.loads(o) for o in result_data]\n", " \n", - " return predict" + " return infer_batch" + ] + }, + { + "cell_type": "markdown", + "id": "a741e23a", + "metadata": {}, + "source": [ + "#### Load and preprocess DataFrame" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 35, + "id": "ccc884a4", + "metadata": {}, + "outputs": [], + "source": [ + "@pandas_udf(\"string\")\n", + "def preprocess(text: pd.Series) -> pd.Series:\n", + " return pd.Series([s.split(\".\")[0] for s in text])" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "c426fdbe", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/27 19:43:39 WARN CacheManager: Asked to cache already cached data.\n" + ] + } + ], + "source": [ + "df = spark.read.parquet(data_path).limit(256).repartition(8)\n", + "df = df.select(preprocess(col(\"text\")).alias(\"input\")).cache()" + ] + }, + { + "cell_type": "markdown", + "id": "7da06df4", + "metadata": {}, + "source": [ + "#### Run Inference" + ] + }, + { + "cell_type": "code", + "execution_count": 37, "id": "3930cfcd-3284-4c6a-a9b5-36b8053fe899", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ - "from functools import partial\n", - "\n", - "classify = predict_batch_udf(partial(triton_fn, triton_uri=\"localhost:8001\", model_name=\"hf_pipeline_torch\"),\n", + "classify = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", " return_type=StructType([\n", " StructField(\"label\", StringType(), True),\n", " StructField(\"score\", FloatType(), True)\n", " ]),\n", " input_tensor_shapes=[[1]],\n", - " batch_size=100)" + " batch_size=32)" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 38, "id": "8eecbf23-4e9e-4d4c-8645-98209b25db2c", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 20:> (0 + 1) / 1]\r" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 5.89 ms, sys: 5.41 ms, total: 11.3 ms\n", - "Wall time: 1.98 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" + "CPU times: user 3.67 ms, sys: 1.81 ms, total: 5.48 ms\n", + "Wall time: 647 ms\n" ] } ], @@ -752,157 +995,116 @@ "%%time\n", "# first pass caches model/fn\n", "# note: expanding the \"struct\" return_type to top-level columns\n", - "preds = df.withColumn(\"preds\", classify(struct(\"sentence\"))).select(\"sentence\", \"preds.*\")\n", + "preds = df.withColumn(\"preds\", classify(struct(\"input\"))).select(\"input\", \"preds.*\")\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 39, "id": "566ba28c-0ca4-4479-a24a-c8a362228b89", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 21:> (0 + 1) / 1]\r" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 5.87 ms, sys: 2.39 ms, total: 8.26 ms\n", - "Wall time: 1.87 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" + "CPU times: user 436 μs, sys: 3.23 ms, total: 3.66 ms\n", + "Wall time: 477 ms\n" ] } ], "source": [ "%%time\n", - "# note: expanding the \"struct\" return_type to top-level columns\n", - "preds = df.withColumn(\"preds\", classify(\"sentence\")).select(\"sentence\", \"preds.*\")\n", + "preds = df.withColumn(\"preds\", classify(\"input\")).select(\"input\", \"preds.*\")\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 40, "id": "44c7e776-08da-484a-ba07-9d6add1a0f15", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 22:> (0 + 1) / 1]\r" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 5.24 ms, sys: 1.13 ms, total: 6.37 ms\n", - "Wall time: 1.86 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" + "CPU times: user 3.62 ms, sys: 792 μs, total: 4.41 ms\n", + "Wall time: 459 ms\n" ] } ], "source": [ "%%time\n", - "# note: expanding the \"struct\" return_type to top-level columns\n", - "preds = df.withColumn(\"preds\", classify(col(\"sentence\"))).select(\"sentence\", \"preds.*\")\n", + "preds = df.withColumn(\"preds\", classify(col(\"input\"))).select(\"input\", \"preds.*\")\n", "results = preds.collect()" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 41, "id": "f61d79f8-661e-4d9e-a3aa-c0754b854603", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+----------+\n", - "|sentence |label |score |\n", - "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+----------+\n", - "| |POSITIVE|0.7481212 |\n", - "|Hard up, No proper jobs going down at the pit, why not rent your kids! DIY pimp story without the gratuitous sex scenes, either hard core or soft core, therefore reads like a public information film from the fifties, give this a wide miss, use a barge pole if you can|NEGATIVE|0.99967253|\n", - "|I watched this movie to see the direction one of the most promising young talents in movies was going |POSITIVE|0.9994943 |\n", - "|This movie makes you wish imdb would let you vote a zero |NEGATIVE|0.9981305 |\n", - "|I never want to see this movie again!

Not only is it dreadfully bad, but I can't stand seeing my hero Stan Laurel looking so old and sick |NEGATIVE|0.9988337 |\n", - "|(As a note, I'd like to say that I saw this movie at my annual church camp, where the entire youth group laughed at it |POSITIVE|0.9901974 |\n", - "|Don't get me wrong, I love the TV series of League Of Gentlemen |POSITIVE|0.9998311 |\n", - "|Did you ever think, like after watching a horror movie with a group of friends: \"Wow, this is so cool! We have got to make a splatter horror movie ourselves some day soon |POSITIVE|0.9992779 |\n", - "|Awful, awful, awful |NEGATIVE|0.9997433 |\n", - "|This movie seems a little clunky around the edges, like not quite enough zaniness was thrown it when it should have been |NEGATIVE|0.99965274|\n", - "|I rented this movie hoping that it would provide some good entertainment and some cool poker knowledge or stories |NEGATIVE|0.99642426|\n", - "|Well, where to start describing this celluloid debacle? You already know the big fat NADA passing as a plot, so let's jut point out that this is so PC it's offensive |NEGATIVE|0.99973005|\n", - "|I hoped for this show to be somewhat realistic |POSITIVE|0.8426496 |\n", - "|All I have to say is one word |NEGATIVE|0.9784491 |\n", - "|Honestly awful film, bad editing, awful lighting, dire dialog and scrappy screenplay |NEGATIVE|0.99977 |\n", - "|This critique tells the story of 4 little friends who went to watch Angels and Demons the movie on the first night it came out, even though it was a school night, because \"Angels and Demons is worth it |POSITIVE|0.9942334 |\n", - "|This review contains a partial spoiler |NEGATIVE|0.996191 |\n", - "|I'm rather surprised that anybody found this film touching or moving |POSITIVE|0.8392794 |\n", - "|If you like bad movies (and you must to watch this one) here's a good one |POSITIVE|0.99366415|\n", - "|This is really bad, the characters were bland, the story was boring, and there is no sex scene |NEGATIVE|0.99953806|\n", - "+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------+----------+\n", + "+----------------------------------------------------------------------+--------+----------+\n", + "| input| label| score|\n", + "+----------------------------------------------------------------------+--------+----------+\n", + "|Doesn't anyone bother to check where this kind of sludge comes from...|NEGATIVE| 0.9984042|\n", + "|There were two things I hated about WASTED : The directing and the ...|NEGATIVE| 0.9979019|\n", + "| I'm rather surprised that anybody found this film touching or moving|POSITIVE| 0.8392794|\n", + "|Cultural Vandalism Is the new Hallmark production of Gulliver's Tra...|NEGATIVE|0.99726933|\n", + "|I was at Wrestlemania VI in Toronto as a 10 year old, and the event...|POSITIVE|0.98212516|\n", + "| This movie has been done before|NEGATIVE|0.94194806|\n", + "|[ as a new resolution for this year 2005, i decide to write a comme...|NEGATIVE|0.99678314|\n", + "|This movie is over hyped!! I am sad to say that I manage to watch t...|NEGATIVE| 0.9985846|\n", + "|This show had a promising start as sort of the opposite of 'Oceans ...|NEGATIVE|0.99926823|\n", + "|MINOR PLOT SPOILERS AHEAD!!!

How did such talented actor...|NEGATIVE| 0.9995671|\n", + "|There is not one character on this sitcom with any redeeming qualities|NEGATIVE|0.99856514|\n", + "| Tommy Lee Jones was the best Woodroe and no one can play Woodroe F|POSITIVE| 0.9945687|\n", + "| My wife rented this movie and then conveniently never got to see it|NEGATIVE| 0.9984137|\n", + "|This is one of those star-filled over-the-top comedies that could a...|NEGATIVE| 0.9953224|\n", + "|This excruciatingly boring and unfunny movie made me think that Cha...|NEGATIVE| 0.9997607|\n", + "|you will likely be sorely disappointed by this sequel that's not a ...|NEGATIVE|0.99971956|\n", + "|If I was British, I would be embarrassed by this portrayal of incom...|NEGATIVE|0.99651587|\n", + "|One of those movies in which there are no big twists whatsoever and...|NEGATIVE|0.99860746|\n", + "|This show is like watching someone who is in training to someday ho...|NEGATIVE| 0.970153|\n", + "| Sigh|NEGATIVE|0.99231356|\n", + "+----------------------------------------------------------------------+--------+----------+\n", "only showing top 20 rows\n", "\n" ] } ], "source": [ - "preds.show(truncate=False)" + "preds.show(truncate=70)" ] }, { "cell_type": "markdown", - "id": "e197c146-1794-47f0-bcd9-7e8d8ab8625f", - "metadata": { - "tags": [] - }, + "id": "2248858c", + "metadata": {}, "source": [ - "#### Stop Triton Server on each executor" + "#### Shut down server on each executor" ] }, { "cell_type": "code", - "execution_count": 31, - "id": "425d3b28-7705-45ba-8a18-ad34fc895219", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 42, + "id": "e3a4e51f", + "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + }, { "name": "stderr", "output_type": "stream", @@ -916,36 +1118,26 @@ "[True]" ] }, - "execution_count": 31, + "execution_count": 42, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "def stop_triton(it):\n", - " import docker\n", - " import time\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " print(\">>>> stopping containers: {}\".format([c.short_id for c in containers]))\n", - " if containers:\n", - " container=containers[0]\n", - " container.stop(timeout=120)\n", - "\n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(stop_triton).collect()" + "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", + "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 43, "id": "9f19643c-4ee4-44f2-b762-2078c0c8eba9", "metadata": {}, "outputs": [], "source": [ - "spark.stop()" + "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n", + " spark.stop()" ] }, { diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pytriton_utils.py b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pytriton_utils.py new file mode 120000 index 00000000..330ea4a5 --- /dev/null +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pytriton_utils.py @@ -0,0 +1 @@ +../pytriton_utils.py \ No newline at end of file diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/sentence_transformers_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/sentence_transformers_torch.ipynb index deac314d..46c961de 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/sentence_transformers_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/sentence_transformers_torch.ipynb @@ -5,16 +5,19 @@ "id": "777fc40d", "metadata": {}, "source": [ + "\n", + "\n", "# PySpark Huggingface Inferencing\n", - "## Sentence Transformers with PyTorch\n", + "### Sentence Transformers with PyTorch\n", "\n", + "In this notebook, we demonstrate distributed inference with the Huggingface SentenceTransformer library for sentence embedding. \n", "From: https://huggingface.co/sentence-transformers" ] }, { "cell_type": "code", "execution_count": 1, - "id": "731faab7-a700-46f8-bba5-1c8764e5eacb", + "id": "c5f0d0a8", "metadata": {}, "outputs": [ { @@ -22,27 +25,46 @@ "output_type": "stream", "text": [ "/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:13: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", - " from tqdm.autonotebook import tqdm, trange\n", + " from tqdm.autonotebook import tqdm, trange\n" + ] + } + ], + "source": [ + "import torch\n", + "from sentence_transformers import SentenceTransformer\n", + "\n", + "# Manually enable Huggingface tokenizer parallelism to avoid disabling with PySpark parallelism.\n", + "# See (https://github.com/huggingface/transformers/issues/5486) for more info. \n", + "import os\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "731faab7-a700-46f8-bba5-1c8764e5eacb", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ "/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/transformers/tokenization_utils_base.py:1617: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be deprecated in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n", " warnings.warn(\n" ] } ], "source": [ - "from sentence_transformers import SentenceTransformer\n", - "model = SentenceTransformer('paraphrase-MiniLM-L6-v2')\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "model = SentenceTransformer(\"paraphrase-MiniLM-L6-v2\", device=device)\n", "\n", - "#Sentences we want to encode. Example:\n", "sentence = ['This framework generates embeddings for each input sentence']\n", - "\n", - "\n", - "#Sentences are encoded by calling model.encode()\n", "embedding = model.encode(sentence)" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "96eea5ca-3cf7-46e3-b40c-598538112d24", "metadata": {}, "outputs": [ @@ -69,33 +91,68 @@ "## PySpark" ] }, + { + "cell_type": "code", + "execution_count": 4, + "id": "dbda3e66-005a-4ad0-8017-c1cc7cbf0058", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql.types import *\n", + "from pyspark import SparkConf\n", + "from pyspark.sql import SparkSession\n", + "from pyspark.sql.functions import pandas_udf, col, struct\n", + "from pyspark.ml.functions import predict_batch_udf" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "b525c5c4", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import pandas as pd\n", + "import datasets\n", + "from datasets import load_dataset\n", + "datasets.disable_progress_bars()" + ] + }, { "cell_type": "markdown", - "id": "e8938317-e31e-4e8d-b2d8-f92c1b5a300c", + "id": "58e7c1bc", "metadata": {}, "source": [ - "## Inference using Spark DL API\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment." + "Check the cluster environment to handle any platform-specific Spark configurations." ] }, { "cell_type": "code", - "execution_count": 3, - "id": "dbda3e66-005a-4ad0-8017-c1cc7cbf0058", + "execution_count": 6, + "id": "5a013217", "metadata": {}, "outputs": [], "source": [ - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import col, struct\n", - "from pyspark.sql.types import ArrayType, FloatType\n", - "from pyspark.sql import SparkSession\n", - "from pyspark import SparkConf\n", - "from datasets import load_dataset" + "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n", + "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n", + "on_standalone = not (on_databricks or on_dataproc)" + ] + }, + { + "cell_type": "markdown", + "id": "ad3c003d", + "metadata": {}, + "source": [ + "#### Create Spark Session\n", + "\n", + "For local standalone clusters, we'll connect to the cluster and create the Spark Session. \n", + "For CSP environments, Spark will either be preconfigured (Databricks) or we'll need to create the Spark Session (Dataproc)." ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 7, "id": "23ec67ba", "metadata": {}, "outputs": [ @@ -103,136 +160,260 @@ "name": "stderr", "output_type": "stream", "text": [ - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "24/10/08 00:19:28 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "24/10/08 00:19:28 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "25/01/27 19:47:12 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/01/27 19:47:12 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "24/10/08 00:19:28 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" + "25/01/27 19:47:12 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "25/01/27 19:47:12 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" ] } ], "source": [ - "import os\n", - "conda_env = os.environ.get(\"CONDA_PREFIX\")\n", - "\n", "conf = SparkConf()\n", + "\n", "if 'spark' not in globals():\n", - " # If Spark is not already started with Jupyter, attach to Spark Standalone\n", - " import socket\n", - " hostname = socket.gethostname()\n", - " conf.setMaster(f\"spark://{hostname}:7077\") # assuming Master is on default port 7077\n", - "conf.set(\"spark.task.maxFailures\", \"1\")\n", - "conf.set(\"spark.driver.memory\", \"8g\")\n", - "conf.set(\"spark.executor.memory\", \"8g\")\n", - "conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled\", \"false\")\n", - "conf.set(\"spark.sql.pyspark.jvmStacktrace.enabled\", \"true\")\n", - "conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", - "conf.set(\"spark.python.worker.reuse\", \"true\")\n", - "# Create Spark Session\n", + " if on_standalone:\n", + " import socket\n", + " conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + " hostname = socket.gethostname()\n", + " conf.setMaster(f\"spark://{hostname}:7077\")\n", + " conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", + " conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_env}/lib:{conda_env}/lib/python3.11/site-packages/nvidia_pytriton.libs:$LD_LIBRARY_PATH\")\n", + " elif on_dataproc:\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conda_lib_path=\"/opt/conda/miniconda3/lib\"\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_lib_path}:$LD_LIBRARY_PATH\")\n", + " conf.set(\"spark.executor.instances\", \"4\") # dataproc defaults to 2\n", + "\n", + " conf.set(\"spark.executor.cores\", \"8\")\n", + " conf.set(\"spark.task.resource.gpu.amount\", \"0.125\")\n", + " conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n", + " conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", + " conf.set(\"spark.python.worker.reuse\", \"true\")\n", + "\n", + "conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"1000\")\n", "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", "sc = spark.sparkContext" ] }, + { + "cell_type": "markdown", + "id": "4cfd1394", + "metadata": {}, + "source": [ + "Load the IMBD Movie Reviews dataset from Huggingface." + ] + }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "id": "9bc1edb5", "metadata": {}, + "outputs": [], + "source": [ + "dataset = load_dataset(\"imdb\", split=\"test\")\n", + "dataset = dataset.to_pandas().drop(columns=\"label\")" + ] + }, + { + "cell_type": "markdown", + "id": "59c71bff", + "metadata": {}, + "source": [ + "#### Create PySpark DataFrame" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "836e5f84-12c6-4c95-838e-53de7e46a20b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "StructType([StructField('text', StringType(), True)])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = spark.createDataFrame(dataset).repartition(8)\n", + "df.schema" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "36703d23-37a3-40df-b09a-c68206d285b6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "25000" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.count()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "1f122ae3", + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "25/01/27 19:47:19 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] + }, + { + "data": { + "text/plain": [ + "[Row(text=\"Anyone remember the first CKY, CKY2K etc..? Back when it was about making crazy cool stuff, rather than watching Bam Margera act like a douchebag, spoiled 5 year old, super/rock-star wannabe.

The show used to be awesome, however, Bam's fame and wealth has led him to believe, that we now enjoy him acting childish and idiotic, more than actual cool stuff, that used to be in ex. CKY2K.

The acts are so repetitive, there's like nothing new, except annoying stupidity and rehearsed comments... The only things we see is Bam Margera, so busy showing us how much he doesn't care, how much money he got or whatsoever.

I really got nothing much left to say except, give us back CKY2K, cause Bam suck..

I enjoy watching Steve-o, Knoxville etc. a thousand times more.\")]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "# load IMDB reviews (test) dataset and write to parquet\n", - "data = load_dataset(\"imdb\", split=\"test\")\n", - "\n", - "lines = []\n", - "for example in data:\n", - " lines.append([example[\"text\"].split(\".\")[0]])\n", - "\n", - "len(lines)\n", + "df.take(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "14fd59fb", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/27 19:47:19 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + ] + } + ], + "source": [ + "data_path = \"spark-dl-datasets/imdb_test\"\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " data_path = \"dbfs:/FileStore/\" + data_path\n", "\n", - "df = spark.createDataFrame(lines, ['lines']).repartition(10)\n", - "df.schema\n", + "df.write.mode(\"overwrite\").parquet(data_path)" + ] + }, + { + "cell_type": "markdown", + "id": "6bb083ec", + "metadata": {}, + "source": [ + "#### Load and preprocess DataFrame\n", "\n", - "df.write.mode(\"overwrite\").parquet(\"imdb_test\")" + "Define our preprocess function. We'll take the first sentence from each sample as our input for translation." ] }, { "cell_type": "code", - "execution_count": 6, - "id": "836e5f84-12c6-4c95-838e-53de7e46a20b", + "execution_count": 13, + "id": "2510bdd1", "metadata": {}, "outputs": [], "source": [ - "# only use first N examples, since this is slow\n", - "df = spark.read.parquet(\"imdb_test\").limit(100).cache()" + "@pandas_udf(\"string\")\n", + "def preprocess(text: pd.Series) -> pd.Series:\n", + " return pd.Series([s.split(\".\")[0] for s in text])" ] }, { "cell_type": "code", - "execution_count": 7, - "id": "36703d23-37a3-40df-b09a-c68206d285b6", + "execution_count": 14, + "id": "5bb28548", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| lines|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| This is so overly clichéd you'll want to switch it off after the first 45 minutes|\n", - "| I was very disappointed by this movie|\n", - "| I think vampire movies (usually) are wicked|\n", - "| Though not a complete waste of time, 'Eighteen' really wasn't all sweet as it pretended to be|\n", - "|This film did well at the box office, and the producers of this mess thought the stars had such good chemistry in thi...|\n", - "| Peter Crawford discovers a comet on a collision course with the moon|\n", - "|This tale of the upper-classes getting their come-uppance and wallowing in their high-class misery is like a contempo...|\n", - "|Words almost fail me to describe how terrible this Irish vanity project (funded by Canadian taxpayers - both federal ...|\n", - "| This was the most uninteresting horror flick I have seen to date|\n", - "| Heart of Darkness was terrible|\n", - "| I saw this movie when it was first released in Pittsburgh Pa|\n", - "|It was funny because the whole thing was so unrealistic, I mean, come on, like a pop star would just show up at a pub...|\n", - "|Watching this movie, you just have to ask: What were they thinking? There are so many noticeably bad parts of this mo...|\n", - "| In a sense, this movie did not even compare to the novel|\n", - "| Poor Jane Austen ought to be glad she's not around to see this dreadful wreck of an adaptation|\n", - "| I gave this movie a four-star rating for a few reasons|\n", - "| It seems that Dee Snyder ran out of ideas halfway through the script|\n", - "| Now, let me see if I have this correct, a lunatic serial killer is going around murdering estate agents|\n", - "| Tommy Lee Jones was the best Woodroe and no one can play Woodroe F|\n", - "|First of all, I would like to say that I am a fan of all of the actors that appear in this film and at the time that ...|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", + "+----------------------------------------------------------------------------------------------------+\n", + "| input|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "|Doesn't anyone bother to check where this kind of sludge comes from before blathering on about it...|\n", + "| There were two things I hated about WASTED : The directing and the script |\n", + "| I'm rather surprised that anybody found this film touching or moving|\n", + "|Cultural Vandalism Is the new Hallmark production of Gulliver's Travels an act of cultural vandal...|\n", + "|I was at Wrestlemania VI in Toronto as a 10 year old, and the event I saw then was pretty differe...|\n", + "| This movie has been done before|\n", + "|[ as a new resolution for this year 2005, i decide to write a comment for each movie I saw in the...|\n", + "|This movie is over hyped!! I am sad to say that I manage to watch the first 15 minutes of this mo...|\n", + "|This show had a promising start as sort of the opposite of 'Oceans 11' but has developed into a s...|\n", + "|MINOR PLOT SPOILERS AHEAD!!!

How did such talented actors get involved in such mindles...|\n", + "| There is not one character on this sitcom with any redeeming qualities|\n", + "| Tommy Lee Jones was the best Woodroe and no one can play Woodroe F|\n", + "| My wife rented this movie and then conveniently never got to see it|\n", + "|This is one of those star-filled over-the-top comedies that could a) be hysterical, or b) wish th...|\n", + "|This excruciatingly boring and unfunny movie made me think that Chaplin was the real Hitler, as o...|\n", + "| you will likely be sorely disappointed by this sequel that's not a sequel|\n", + "| If I was British, I would be embarrassed by this portrayal of incompetence|\n", + "|One of those movies in which there are no big twists whatsoever and you can predict pretty much w...|\n", + "| This show is like watching someone who is in training to someday host a show|\n", + "| Sigh|\n", + "+----------------------------------------------------------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] } ], "source": [ - "df.show(truncate=120)" + "# Limit to N rows, since this can be slow\n", + "df = spark.read.parquet(data_path).limit(256).repartition(8)\n", + "df = df.select(preprocess(col(\"text\")).alias(\"input\")).cache()\n", + "df.show(truncate=100)" + ] + }, + { + "cell_type": "markdown", + "id": "014eae88", + "metadata": {}, + "source": [ + "## Inference using Spark DL API\n", + "\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "\n", + "- predict_batch_fn uses PyTorch APIs to load the model and return a predict function which operates on numpy arrays \n", + "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 15, "id": "f780c026-0f3f-4aea-8b61-5b3dbae83fb7", "metadata": {}, "outputs": [], "source": [ "def predict_batch_fn():\n", + " import torch\n", " from sentence_transformers import SentenceTransformer\n", - " model = SentenceTransformer(\"paraphrase-MiniLM-L6-v2\")\n", + "\n", + " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + " model = SentenceTransformer(\"paraphrase-MiniLM-L6-v2\", device=device)\n", " def predict(inputs):\n", " return model.encode(inputs.tolist())\n", " return predict" @@ -240,19 +421,19 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 16, "id": "f5c88ddc-ca19-4430-8b0e-b9fae143b237", "metadata": {}, "outputs": [], "source": [ "encode = predict_batch_udf(predict_batch_fn,\n", " return_type=ArrayType(FloatType()),\n", - " batch_size=10)" + " batch_size=32)" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 17, "id": "85344c22-4a4d-4cb0-8771-5836ae2794db", "metadata": {}, "outputs": [ @@ -260,552 +441,565 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 9:> (0 + 1) / 1]\r" + " \r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 4.34 ms, sys: 4.15 ms, total: 8.48 ms\n", - "Wall time: 2.58 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" + "CPU times: user 8.98 ms, sys: 4.45 ms, total: 13.4 ms\n", + "Wall time: 3.67 s\n" ] } ], "source": [ "%%time\n", "# first pass caches model/fn\n", - "embeddings = df.withColumn(\"encoding\", encode(struct(\"lines\")))\n", + "embeddings = df.withColumn(\"embedding\", encode(struct(\"input\")))\n", "results = embeddings.collect()" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 18, "id": "c23bb885-6ab0-4471-943d-4c10414100fa", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 11:> (0 + 1) / 1]\r" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 1.76 ms, sys: 4.89 ms, total: 6.65 ms\n", - "Wall time: 2.47 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" + "CPU times: user 8.72 ms, sys: 873 μs, total: 9.59 ms\n", + "Wall time: 154 ms\n" ] } ], "source": [ "%%time\n", - "embeddings = df.withColumn(\"encoding\", encode(\"lines\"))\n", + "embeddings = df.withColumn(\"embedding\", encode(\"input\"))\n", "results = embeddings.collect()" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 19, "id": "93bc6da3-d853-4233-b805-cb4a46f4f9b9", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 13:> (0 + 1) / 1]\r" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 1.55 ms, sys: 6.05 ms, total: 7.6 ms\n", - "Wall time: 2.46 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" + "CPU times: user 3.61 ms, sys: 3.82 ms, total: 7.43 ms\n", + "Wall time: 180 ms\n" ] } ], "source": [ "%%time\n", - "embeddings = df.withColumn(\"encoding\", encode(col(\"lines\")))\n", + "embeddings = df.withColumn(\"embedding\", encode(col(\"input\")))\n", "results = embeddings.collect()" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 20, "id": "2073616f-7151-4760-92f2-441dd0bfe9fe", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 15:> (0 + 1) / 1]\r" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "| lines| encoding|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "|This is so overly clichéd you'll want to switch it off af...|[-0.06755405, -0.13365394, 0.36675274, -0.2772311, -0.085...|\n", - "| I was very disappointed by this movie|[-0.05903806, 0.16684641, 0.16768408, 0.10940918, 0.18100...|\n", - "| I think vampire movies (usually) are wicked|[0.025601083, -0.5308639, -0.319133, -0.013351389, -0.338...|\n", - "|Though not a complete waste of time, 'Eighteen' really wa...|[0.20991832, 0.5228605, 0.44517252, -0.031682555, -0.4117...|\n", - "|This film did well at the box office, and the producers o...|[0.18097948, -0.03622232, -0.34149718, 0.061557338, -0.06...|\n", - "|Peter Crawford discovers a comet on a collision course wi...|[-0.27548054, 0.196654, -0.24626413, -0.39380816, -0.5501...|\n", - "|This tale of the upper-classes getting their come-uppance...|[0.24201547, 0.011018356, -0.080340266, 0.31388673, -0.28...|\n", - "|Words almost fail me to describe how terrible this Irish ...|[0.055901285, -0.14539501, -0.14005454, -0.038912475, 0.4...|\n", - "|This was the most uninteresting horror flick I have seen ...|[0.27159664, -0.012541974, -0.31898177, 0.058205508, 0.56...|\n", - "| Heart of Darkness was terrible|[0.1593065, 0.36501122, 0.10715093, 0.76344764, 0.2555183...|\n", - "|I saw this movie when it was first released in Pittsburgh Pa|[-0.34647614, 0.115615666, -0.18874267, 0.36590436, -0.06...|\n", - "|It was funny because the whole thing was so unrealistic, ...|[0.09473594, -0.43785918, 0.14436111, 0.0045353747, -0.08...|\n", - "|Watching this movie, you just have to ask: What were they...|[0.43020695, -0.09714467, 0.1356213, 0.23126744, -0.03908...|\n", - "| In a sense, this movie did not even compare to the novel|[0.2838324, -0.018966805, -0.37275136, 0.27034461, 0.2017...|\n", - "|Poor Jane Austen ought to be glad she's not around to see...|[0.27462235, -0.32494685, 0.48243234, 0.07208571, 0.22470...|\n", - "| I gave this movie a four-star rating for a few reasons|[0.31143323, -0.09470663, -0.10863629, 0.077851094, -0.15...|\n", - "|It seems that Dee Snyder ran out of ideas halfway through...|[0.44354546, -0.08122106, -0.15206784, -0.29244298, 0.559...|\n", - "|Now, let me see if I have this correct, a lunatic serial ...|[0.39831734, 0.15871558, -0.35366735, -0.11643518, -0.137...|\n", - "|Tommy Lee Jones was the best Woodroe and no one can play ...|[-0.20960264, -0.15760101, -0.30596393, -0.51817703, -0.0...|\n", - "|First of all, I would like to say that I am a fan of all ...|[0.25831866, -0.26871824, 0.026099348, -0.3459879, -0.180...|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "| input| embedding|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "|Doesn't anyone bother to check where this kind ...|[0.118947476, -0.053823642, -0.29726124, 0.0720...|\n", + "|There were two things I hated about WASTED : Th...|[0.18953452, 0.11079162, 0.07503566, 0.01050696...|\n", + "|I'm rather surprised that anybody found this fi...|[-0.0010759671, -0.14203517, -0.06649738, 0.129...|\n", + "|Cultural Vandalism Is the new Hallmark producti...|[0.34815887, -0.2966917, -0.10905265, 0.1051652...|\n", + "|I was at Wrestlemania VI in Toronto as a 10 yea...|[0.45902696, 0.019472413, 0.28720972, -0.070724...|\n", + "| This movie has been done before|[-0.062292397, -0.025909504, -0.031942524, 0.01...|\n", + "|[ as a new resolution for this year 2005, i dec...|[0.3469342, -0.14378615, 0.30223376, -0.1102267...|\n", + "|This movie is over hyped!! I am sad to say that...|[0.13230576, -0.06588756, 0.0472389, 0.08353163...|\n", + "|This show had a promising start as sort of the ...|[-0.19361982, -0.14412567, 0.15149693, -0.17715...|\n", + "|MINOR PLOT SPOILERS AHEAD!!!

How did...|[-0.048036292, 0.050720096, -0.04668727, -0.316...|\n", + "|There is not one character on this sitcom with ...|[0.13720773, -0.5963504, 0.30331734, -0.3830607...|\n", + "|Tommy Lee Jones was the best Woodroe and no one...|[-0.20960267, -0.15760122, -0.30596405, -0.5181...|\n", + "|My wife rented this movie and then conveniently...|[0.46534792, -0.40655977, 0.054217298, -0.03414...|\n", + "|This is one of those star-filled over-the-top c...|[0.14433198, -0.016140658, 0.3775344, 0.0659043...|\n", + "|This excruciatingly boring and unfunny movie ma...|[0.056464806, 0.01144963, -0.51797307, 0.089813...|\n", + "|you will likely be sorely disappointed by this ...|[-0.44146675, -0.17866582, 0.49889183, -0.26819...|\n", + "|If I was British, I would be embarrassed by thi...|[0.1191261, -0.15379854, 0.17487673, -0.5123498...|\n", + "|One of those movies in which there are no big t...|[-0.016174048, -0.5558219, -0.024818476, 0.1543...|\n", + "|This show is like watching someone who is in tr...|[0.033776704, -0.6682203, 0.30547586, -0.581407...|\n", + "| Sigh|[-0.119870394, 0.40893683, 0.4174831, -0.010004...|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] } ], "source": [ - "embeddings.show(truncate=60)" + "embeddings.show(truncate=50)" ] }, { "cell_type": "markdown", - "id": "b730f5a3-f7eb-42aa-8869-881ecd0f5542", + "id": "0c9c6535", "metadata": {}, "source": [ - "### Using Triton Inference Server\n", + "## Using Triton Inference Server\n", + "In this section, we demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. \n", + "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server. \n", + "\n", + "The process looks like this:\n", + "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n", + "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n", + "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n", + "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n", "\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment." + "\"drawing\"" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "772e337e-1098-4c7b-ba81-8cb221a518e2", + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial" ] }, { "cell_type": "markdown", - "id": "5f502a20", + "id": "759385ac", "metadata": {}, "source": [ - "This notebook uses the [Python backend with a custom execution environment](https://github.com/triton-inference-server/python_backend#creating-custom-execution-environments) with the compatible versions of Python/Numpy for Triton 24.08, using a conda-pack environment created as follows:\n", - "```\n", - "conda create -n huggingface-torch -c conda-forge python=3.10.0\n", - "conda activate huggingface-torch\n", - "\n", - "export PYTHONNOUSERSITE=True\n", - "pip install numpy==1.26.4 conda-pack sentencepiece sentence_transformers transformers\n", - "\n", - "conda-pack # huggingface-torch.tar.gz\n", - "```" + "Import the utility functions from pytriton_utils.py:" ] }, { "cell_type": "code", - "execution_count": 14, - "id": "772e337e-1098-4c7b-ba81-8cb221a518e2", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 22, + "id": "485fb0de", + "metadata": {}, "outputs": [], "source": [ - "import numpy as np\n", - "import os\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import col, struct\n", - "from pyspark.sql.types import ArrayType, FloatType" + "sc.addPyFile(\"pytriton_utils.py\")\n", + "\n", + "from pytriton_utils import (\n", + " use_stage_level_scheduling,\n", + " find_ports,\n", + " start_triton,\n", + " stop_triton\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "ece5c38a", + "metadata": {}, + "source": [ + "Define the Triton Server function:" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 23, "id": "69d0c93a-bb0b-46c5-9d28-7b08a2e70964", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ - "%%bash\n", - "# copy custom model to expected layout for Triton\n", - "rm -rf models\n", - "mkdir -p models\n", - "cp -r models_config/hf_transformer_torch models\n", + "def triton_server(ports):\n", + " import time\n", + " import signal\n", + " import numpy as np\n", + " import torch\n", + " from sentence_transformers import SentenceTransformer\n", + " from pytriton.decorators import batch\n", + " from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n", + " from pytriton.triton import Triton, TritonConfig\n", + " from pyspark import TaskContext\n", + "\n", + " print(f\"SERVER: Initializing sentence transformer on worker {TaskContext.get().partitionId()}.\")\n", + " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + " model = SentenceTransformer(\"paraphrase-MiniLM-L6-v2\", device=device)\n", + " print(f\"SERVER: Using {device} device.\")\n", + "\n", + " @batch\n", + " def _infer_fn(**inputs):\n", + " sentences = np.squeeze(inputs[\"text\"])\n", + " print(f\"SERVER: Received batch of size {len(sentences)}\")\n", + " decoded_sentences = [s.decode(\"utf-8\") for s in sentences]\n", + " embeddings = model.encode(decoded_sentences)\n", + " return {\n", + " \"embeddings\": embeddings,\n", + " }\n", "\n", - "# add custom execution environment\n", - "cp huggingface-torch.tar.gz models" + " workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n", + " triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n", + " with Triton(config=triton_conf, workspace=workspace_path) as triton:\n", + " triton.bind(\n", + " model_name=\"SentenceTransformer\",\n", + " infer_func=_infer_fn,\n", + " inputs=[\n", + " Tensor(name=\"text\", dtype=object, shape=(-1,)),\n", + " ],\n", + " outputs=[\n", + " Tensor(name=\"embeddings\", dtype=np.float32, shape=(-1,)),\n", + " ],\n", + " config=ModelConfig(\n", + " max_batch_size=64,\n", + " batcher=DynamicBatcher(max_queue_delay_microseconds=5000), # 5ms\n", + " ),\n", + " strict=True,\n", + " )\n", + "\n", + " def _stop_triton(signum, frame):\n", + " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", + " triton.stop()\n", + "\n", + " signal.signal(signal.SIGTERM, _stop_triton)\n", + "\n", + " print(\"SERVER: Serving inference\")\n", + " triton.serve()" ] }, { "cell_type": "markdown", - "id": "dd4d7d4b-1a0b-4c5f-bc93-be2a039b6ea0", - "metadata": { - "tags": [] - }, + "id": "79532110", + "metadata": {}, + "source": [ + "#### Start Triton servers" + ] + }, + { + "cell_type": "markdown", + "id": "bef23176", + "metadata": {}, "source": [ - "#### Start Triton Server on each executor" + "**Specify the number of nodes in the cluster.** \n", + "Following the README, the example standalone cluster uses 1 node. The example Databricks/Dataproc cluster scripts use 4 nodes by default. " ] }, { "cell_type": "code", - "execution_count": 16, - "id": "1654cdc1-4f9a-4fd5-b7ac-6ca4215bde5d", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 24, + "id": "b992802e", + "metadata": {}, + "outputs": [], + "source": [ + "# Change based on cluster setup\n", + "num_nodes = 1 if on_standalone else 4" + ] + }, + { + "cell_type": "markdown", + "id": "642d9f8b", + "metadata": {}, + "source": [ + "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "69015ae1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reqesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + } + ], + "source": [ + "sc = spark.sparkContext\n", + "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" + ] + }, + { + "cell_type": "markdown", + "id": "32d5e8e9", + "metadata": {}, + "source": [ + "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "012b2d60", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using ports [7000, 7001, 7002]\n" + ] + } + ], + "source": [ + "model_name = \"SentenceTransformer\"\n", + "ports = find_ports()\n", + "assert len(ports) == 3\n", + "print(f\"Using ports {ports}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea38ac6b", + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "[Stage 28:> (0 + 1) / 1]\r" ] }, { - "data": { - "text/plain": [ - "[True]" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "Triton Server PIDs:\n", + " {\n", + " \"cb4ae00-lcedt\": 2960340\n", + "}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] } ], "source": [ - "num_executors = 1\n", - "triton_models_dir = \"{}/models\".format(os.getcwd())\n", - "huggingface_cache_dir = \"{}/.cache/huggingface\".format(os.path.expanduser('~'))\n", - "nodeRDD = sc.parallelize(list(range(num_executors)), num_executors)\n", - "\n", - "def start_triton(it):\n", - " import docker\n", - " import time\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " if containers:\n", - " print(\">>>> containers: {}\".format([c.short_id for c in containers]))\n", - " else:\n", - " container=client.containers.run(\n", - " \"nvcr.io/nvidia/tritonserver:24.08-py3\", \"tritonserver --model-repository=/models\",\n", - " detach=True,\n", - " device_requests=[docker.types.DeviceRequest(device_ids=[\"0\"], capabilities=[['gpu']])],\n", - " environment=[\n", - " \"TRANSFORMERS_CACHE=/cache\"\n", - " ],\n", - " name=\"spark-triton\",\n", - " network_mode=\"host\",\n", - " remove=True,\n", - " shm_size=\"512M\",\n", - " volumes={\n", - " triton_models_dir: {\"bind\": \"/models\", \"mode\": \"ro\"},\n", - " huggingface_cache_dir: {\"bind\": \"/cache\", \"mode\": \"rw\"}\n", - " }\n", - " )\n", - " print(\">>>> starting triton: {}\".format(container.short_id))\n", - "\n", - " # wait for triton to be running\n", - " time.sleep(15)\n", - " client = grpcclient.InferenceServerClient(\"localhost:8001\")\n", - " ready = False\n", - " while not ready:\n", - " try:\n", - " ready = client.is_server_ready()\n", - " except Exception as e:\n", - " time.sleep(5)\n", + "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", + " ports=ports,\n", + " model_name=model_name)).collectAsMap()\n", + "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + ] + }, + { + "cell_type": "markdown", + "id": "1fd19fae", + "metadata": {}, + "source": [ + "#### Define client function" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "00d82bfe", + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{ports[0]}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "807dbc45", + "metadata": {}, + "outputs": [], + "source": [ + "def triton_fn(url, model_name):\n", + " import numpy as np\n", + " from pytriton.client import ModelClient\n", "\n", - " return [True]\n", + " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", "\n", - "nodeRDD.barrier().mapPartitions(start_triton).collect()" + " def infer_batch(inputs):\n", + " with ModelClient(url, model_name, inference_timeout_s=240) as client:\n", + " flattened = np.squeeze(inputs).tolist()\n", + " # Encode batch\n", + " encoded_batch = [[text.encode(\"utf-8\")] for text in flattened]\n", + " encoded_batch_np = np.array(encoded_batch, dtype=np.bytes_)\n", + " # Run inference\n", + " result_data = client.infer_batch(encoded_batch_np)\n", + " return result_data[\"embeddings\"]\n", + " \n", + " return infer_batch" ] }, { "cell_type": "markdown", - "id": "ee34de5f-89f8-455e-b45e-a557a4ab0f05", + "id": "af174106", "metadata": {}, "source": [ - "#### Run inference" + "#### Load and preprocess DataFrame" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 30, "id": "2969d502-e97b-49d6-bf80-7d177ae867cf", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ - "from functools import partial\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import col, struct\n", - "from pyspark.sql.types import ArrayType, FloatType" + "@pandas_udf(\"string\")\n", + "def preprocess(text: pd.Series) -> pd.Series:\n", + " return pd.Series([s.split(\".\")[0] for s in text])" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 31, "id": "c8f1e6d6-6519-49e7-8465-4419547633b8", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "24/10/08 00:20:24 WARN CacheManager: Asked to cache already cached data.\n" + "25/01/27 19:47:31 WARN CacheManager: Asked to cache already cached data.\n" ] } ], "source": [ - "# only use first N examples, since this is slow\n", - "df = spark.read.parquet(\"imdb_test\").limit(100).cache()" + "df = spark.read.parquet(data_path).limit(256).repartition(8)\n", + "df = df.select(preprocess(col(\"text\")).alias(\"input\")).cache()" ] }, { - "cell_type": "code", - "execution_count": 19, - "id": "29b0cc0d-c480-4e4a-bd41-207dc314cba5", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], + "cell_type": "markdown", + "id": "cf0ee731", + "metadata": {}, "source": [ - "def triton_fn(triton_uri, model_name):\n", - " import numpy as np\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " np_types = {\n", - " \"BOOL\": np.dtype(np.bool_),\n", - " \"INT8\": np.dtype(np.int8),\n", - " \"INT16\": np.dtype(np.int16),\n", - " \"INT32\": np.dtype(np.int32),\n", - " \"INT64\": np.dtype(np.int64),\n", - " \"FP16\": np.dtype(np.float16),\n", - " \"FP32\": np.dtype(np.float32),\n", - " \"FP64\": np.dtype(np.float64),\n", - " \"FP64\": np.dtype(np.double),\n", - " \"BYTES\": np.dtype(object)\n", - " }\n", - "\n", - " client = grpcclient.InferenceServerClient(triton_uri)\n", - " model_meta = client.get_model_metadata(model_name)\n", - " \n", - " def predict(inputs):\n", - " if isinstance(inputs, np.ndarray):\n", - " # single ndarray input\n", - " request = [grpcclient.InferInput(model_meta.inputs[0].name, inputs.shape, model_meta.inputs[0].datatype)]\n", - " request[0].set_data_from_numpy(inputs.astype(np_types[model_meta.inputs[0].datatype]))\n", - " else:\n", - " # dict of multiple ndarray inputs\n", - " request = [grpcclient.InferInput(i.name, inputs[i.name].shape, i.datatype) for i in model_meta.inputs]\n", - " for i in request:\n", - " i.set_data_from_numpy(inputs[i.name()].astype(np_types[i.datatype()]))\n", - " \n", - " response = client.infer(model_name, inputs=request)\n", - " \n", - " if len(model_meta.outputs) > 1:\n", - " # return dictionary of numpy arrays\n", - " return {o.name: response.as_numpy(o.name) for o in model_meta.outputs}\n", - " else:\n", - " # return single numpy array\n", - " return response.as_numpy(model_meta.outputs[0].name)\n", - " \n", - " return predict" + "#### Run Inference" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 32, "id": "9c712b8f-6eb4-4fb8-9f0a-04feef847fea", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ - "encode = predict_batch_udf(partial(triton_fn, triton_uri=\"localhost:8001\", model_name=\"hf_transformer_torch\"),\n", + "encode = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", " return_type=ArrayType(FloatType()),\n", " input_tensor_shapes=[[1]],\n", - " batch_size=100)" + " batch_size=32)" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 33, "id": "934c1a1f-b126-45b0-9c15-265236820ad3", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 4.65 ms, sys: 2.85 ms, total: 7.49 ms\n", - "Wall time: 480 ms\n" + "CPU times: user 11.6 ms, sys: 2.74 ms, total: 14.3 ms\n", + "Wall time: 577 ms\n" ] } ], "source": [ "%%time\n", "# first pass caches model/fn\n", - "embeddings = df.withColumn(\"encoding\", encode(struct(\"lines\")))\n", + "embeddings = df.withColumn(\"embedding\", encode(struct(\"input\")))\n", "results = embeddings.collect()" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 34, "id": "f84cd3f6-b6a8-4142-859a-91f3c183457b", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 1.45 ms, sys: 1.1 ms, total: 2.56 ms\n", - "Wall time: 384 ms\n" + "CPU times: user 5.36 ms, sys: 3.16 ms, total: 8.52 ms\n", + "Wall time: 205 ms\n" ] } ], "source": [ "%%time\n", - "embeddings = df.withColumn(\"encoding\", encode(\"lines\"))\n", + "embeddings = df.withColumn(\"embedding\", encode(\"input\"))\n", "results = embeddings.collect()" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 35, "id": "921a4c01-e296-4406-be90-86f20c8c582d", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 1.63 ms, sys: 1.28 ms, total: 2.91 ms\n", - "Wall time: 416 ms\n" + "CPU times: user 5.75 ms, sys: 430 μs, total: 6.18 ms\n", + "Wall time: 180 ms\n" ] } ], "source": [ "%%time\n", - "embeddings = df.withColumn(\"encoding\", encode(col(\"lines\")))\n", + "embeddings = df.withColumn(\"embedding\", encode(col(\"input\")))\n", "results = embeddings.collect()" ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 36, "id": "9f67584e-9c4e-474f-b6ea-7811b14d116e", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "| lines| encoding|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", - "|This is so overly clichéd you'll want to switch it off af...|[-0.06755393, -0.1336537, 0.366753, -0.2772312, -0.085145...|\n", - "| I was very disappointed by this movie|[-0.059038587, 0.1668467, 0.16768396, 0.10940957, 0.18100...|\n", - "| I think vampire movies (usually) are wicked|[0.025601566, -0.5308643, -0.31913283, -0.013350786, -0.3...|\n", - "|Though not a complete waste of time, 'Eighteen' really wa...|[0.2099183, 0.5228606, 0.4451728, -0.031682458, -0.411756...|\n", - "|This film did well at the box office, and the producers o...|[0.1809797, -0.036222238, -0.34149715, 0.06155738, -0.066...|\n", - "|Peter Crawford discovers a comet on a collision course wi...|[-0.27548066, 0.196654, -0.24626443, -0.3938084, -0.55015...|\n", - "|This tale of the upper-classes getting their come-uppance...|[0.24201535, 0.011018419, -0.080340445, 0.31388694, -0.28...|\n", - "|Words almost fail me to describe how terrible this Irish ...|[0.05590127, -0.14539507, -0.14005487, -0.03891221, 0.444...|\n", - "|This was the most uninteresting horror flick I have seen ...|[0.2715968, -0.012542339, -0.3189819, 0.05820581, 0.56001...|\n", - "| Heart of Darkness was terrible|[0.15930629, 0.36501077, 0.10715161, 0.7634482, 0.2555183...|\n", - "|I saw this movie when it was first released in Pittsburgh Pa|[-0.34647676, 0.11561544, -0.18874292, 0.36590466, -0.068...|\n", - "|It was funny because the whole thing was so unrealistic, ...|[0.09473588, -0.4378593, 0.14436121, 0.0045354995, -0.085...|\n", - "|Watching this movie, you just have to ask: What were they...|[0.43020678, -0.09714476, 0.13562134, 0.23126753, -0.0390...|\n", - "| In a sense, this movie did not even compare to the novel|[0.28383228, -0.01896684, -0.37275153, 0.27034503, 0.2017...|\n", - "|Poor Jane Austen ought to be glad she's not around to see...|[0.27462238, -0.32494652, 0.48243237, 0.07208576, 0.22470...|\n", - "| I gave this movie a four-star rating for a few reasons|[0.311433, -0.09470633, -0.10863638, 0.07785072, -0.15611...|\n", - "|It seems that Dee Snyder ran out of ideas halfway through...|[0.44354525, -0.08122053, -0.15206799, -0.29244322, 0.559...|\n", - "|Now, let me see if I have this correct, a lunatic serial ...|[0.39831725, 0.15871589, -0.35366756, -0.11643555, -0.137...|\n", - "|Tommy Lee Jones was the best Woodroe and no one can play ...|[-0.20960276, -0.157601, -0.30596414, -0.5181772, -0.0852...|\n", - "|First of all, I would like to say that I am a fan of all ...|[0.25831848, -0.26871827, 0.026099432, -0.34598774, -0.18...|\n", - "+------------------------------------------------------------+------------------------------------------------------------+\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "| input| embedding|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", + "|Doesn't anyone bother to check where this kind ...|[0.118947476, -0.053823642, -0.29726124, 0.0720...|\n", + "|There were two things I hated about WASTED : Th...|[0.18953452, 0.11079162, 0.07503566, 0.01050696...|\n", + "|I'm rather surprised that anybody found this fi...|[-0.0010759671, -0.14203517, -0.06649738, 0.129...|\n", + "|Cultural Vandalism Is the new Hallmark producti...|[0.34815887, -0.2966917, -0.10905265, 0.1051652...|\n", + "|I was at Wrestlemania VI in Toronto as a 10 yea...|[0.45902696, 0.019472413, 0.28720972, -0.070724...|\n", + "| This movie has been done before|[-0.062292397, -0.025909504, -0.031942524, 0.01...|\n", + "|[ as a new resolution for this year 2005, i dec...|[0.3469342, -0.14378615, 0.30223376, -0.1102267...|\n", + "|This movie is over hyped!! I am sad to say that...|[0.13230576, -0.06588756, 0.0472389, 0.08353163...|\n", + "|This show had a promising start as sort of the ...|[-0.19361982, -0.14412567, 0.15149693, -0.17715...|\n", + "|MINOR PLOT SPOILERS AHEAD!!!

How did...|[-0.048036292, 0.050720096, -0.04668727, -0.316...|\n", + "|There is not one character on this sitcom with ...|[0.13720773, -0.5963504, 0.30331734, -0.3830607...|\n", + "|Tommy Lee Jones was the best Woodroe and no one...|[-0.20960267, -0.15760122, -0.30596405, -0.5181...|\n", + "|My wife rented this movie and then conveniently...|[0.46534792, -0.40655977, 0.054217298, -0.03414...|\n", + "|This is one of those star-filled over-the-top c...|[0.14433198, -0.016140658, 0.3775344, 0.0659043...|\n", + "|This excruciatingly boring and unfunny movie ma...|[0.056464806, 0.01144963, -0.51797307, 0.089813...|\n", + "|you will likely be sorely disappointed by this ...|[-0.44146675, -0.17866582, 0.49889183, -0.26819...|\n", + "|If I was British, I would be embarrassed by thi...|[0.1191261, -0.15379854, 0.17487673, -0.5123498...|\n", + "|One of those movies in which there are no big t...|[-0.016174048, -0.5558219, -0.024818476, 0.1543...|\n", + "|This show is like watching someone who is in tr...|[0.033776704, -0.6682203, 0.30547586, -0.581407...|\n", + "| Sigh|[-0.119870394, 0.40893683, 0.4174831, -0.010004...|\n", + "+--------------------------------------------------+--------------------------------------------------+\n", "only showing top 20 rows\n", "\n" ] } ], "source": [ - "embeddings.show(truncate=60)" + "embeddings.show(truncate=50)" ] }, { @@ -820,14 +1014,17 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 37, "id": "d8e5466b-b5dc-4fe1-9012-0c87cdd72962", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reqesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + }, { "name": "stderr", "output_type": "stream", @@ -841,36 +1038,26 @@ "[True]" ] }, - "execution_count": 25, + "execution_count": 37, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "def stop_triton(it):\n", - " import docker\n", - " import time\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " print(\">>>> stopping containers: {}\".format([c.short_id for c in containers]))\n", - " if containers:\n", - " container=containers[0]\n", - " container.stop(timeout=120)\n", - "\n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(stop_triton).collect()" + "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", + "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 38, "id": "e82b9518-da7b-4ebc-8990-c8ab909bec18", "metadata": {}, "outputs": [], "source": [ - "spark.stop()" + "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n", + " spark.stop()" ] }, { diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/images/spark-pytriton.png b/examples/ML+DL-Examples/Spark-DL/dl_inference/images/spark-pytriton.png new file mode 100644 index 0000000000000000000000000000000000000000..fed2a16783e7cc52a0b8a02e26d936c4d8e81b8e GIT binary patch literal 112801 zcmeFYbySqw_Xmu0h^Poimx#2IGBij_m(+lC_t1@s(o#crclQ8FcjwUE=+HI1!@Xbc z_g=rhyVm>fyVlEEz%$SDoOAZxXYYMJJLZd`yd*9*2{sZE60Wq=8)YOUv;rg~=JV^F-2)HF)Bp|TN86DVzPBD@V#t0U80n6mOVRsA zKh)ZS%wS6UDjMAX@SNMo9&O71Gv*7N3aX4HEhJ5*jA=qNaitM7%_rlaa{V?Ks~u+J z)u6{=Q~jpUk@3Mb$I;@^5HCJbH)ohsG`0$o6u!x4G06P($g6Uvxu>ltT*BxuB)ht8 z8WX!RAz>r=Tiu2Ed&5=D#@yt)i;K4!UpIe_)guXi!|Xa5tvAP{l$=D>cs7WHq!u_0 zs$_pN=1KfM;VHwz2>iz>sO%B|rRd@|3vNTX_a&F=^s_9vJmveWtP>b3rVERc3wd zIyH6GM^>xAh&AwDg_N{Gs^S}3kO&F`(@hq02;Th!!+W-!KA)nV2|jY{mox1cd= zUgpbHc2B(940d6)d3MWDA+;jH=LragQTu&+;)hcuxt^cv|wKxP_l z67uFZIy>As)SIPyPmwd1c;k=`Tc4ncy#9P&iCRqxXEg9FHPh#N^};C;LQk+egzXh+ zo_%?W?e-+|Nm(KKFcoV$l2YBo03=IL^EO#RfTHu-bPLgZ*;B>8Dog&Roto z&TKVk9uJ_w|U!SF$@wDeOO&$xBFNpP$yPbStmI6;s6DRUiz{2yT`KOPf<_7 z=KDSmeb5&JeS%VH6frMRB}E_lNp@3P(T+a?VDeyQeqhMpABos{d!PO}KF`CxF9Yqt z?IJ6r#Kgqp#2E1piT(QB;~nB@*}=x2j0Gc~NIic>XwD!lIxAyAl}tNN3!#qcK{w)Y zVCD^IikXvY7WRGvr;?4~0vq?ijHY_jduw`6dzzzZySMvZt;hDQ_r~;)M>SHP`HQAK zRebm4fiX*Vq(h_wn4Li(rZyTcQXy)Q#zzqzbRCB*|4sOtieIi`TAALm|94CR@rmq; zJe?g03-S{hQ1qv4;?e0UBdu5aFN*`Y)AQ2IGN0zh6~D`O)v}ku$~H=CSC>(n&2duQ z4jZL6#x>4i*K@0MsB}n$dBmVG+Aw;?(8LhNc*iIdIH@U2JeeRbur9!x7`AwBL2aRF zaW&3b0C^{@y;SAfm}d5E1nq3%JqZ6q@nmA>YNE7I!NSbK)$%jvshQ5OYkGdxc20be zW`Uc8qfT(adIsGv;{>4aB&0;jbW~+~{h0OgLrp_Xc+HzeiAG5enJz_oQF^^x@y&Er zC2)FTnsxy+pD#T(nL7xT`q`;tsg>bw6uvl_}PhIWOyS zL=ulNj}`S;P3!c`E%Glm4P39&K*Y;~otbxl#}nTtX(loTSNyX5vrCPejRV&$nedrP z)Uqe6Ei05TC=yhzCb>p$Z*RZddBAz~)cw@{RPEG-m{a1V zApQ1S zJ3YkoMe#&&M!%1~gMp0#^%s7~XJ$Iwrr)-t_O0cEzL>ulqgcuZtxp>todcEH)IX5l z_aHi;XlA(<`JnXSSq6tPtAn|b;o8-U@Vv;p^eyob%Z$>Fyber*Sws8sujMD@QL_Bt zJuUWzacA@8QNF(WU}Lt%3LR^=9oQJtn!24jS#Vpscn5jNRX^6sxWn_+nV)Ju@pTAY zc+Pl-LN$6xH~Efa&lT{K@MRu%K2FAWc^*bz|2&2c@~{k*z+a6CPfD*&RziHs0PR<}`ZU!1^lE zlb(3csi&y_I)We`#H=l-9p@#`^~}_9a7?FkE>%5K!%@R!!+g`an-tilqf$wzMcPv1 zn0(OS56awyOi@k?&qY)$Pk9DkM_$*GW^y|}F?&+P_Vi-#sxe1&M$~?JUQRx-pheP$ z?Y!Bv@?=9Gv4!wwPzj;-6EK0#!uNTH6O2B}UZ3+CuJ@Kh+>O>(HHNh~wTm;$4XeUm zJf&@Vx(9uei?feMGe&D0)%O~dO;pq&iFMi)9(i*uO)q=EYojEYR@z)cRX#T_fEyg^*N|-HDyx-4;$v z(oYXQP;6OL*;?`EdE05@&pk6I5vn>K9%!QogeDFoSZYOU1%><4M$!&>hh1F^Dz#)L zcRlS&2;ZeF1&9bJjbm4DRLNJ-RpVB7Rh^|A3(U+ckK3;h9qEzCo2M4=7rVMciC5zN zITi;uQj+=U0cR_oD;Hk}36k+x-{{rN6O2wcjvTJ6%L2iyP8^LxK<8%9ouJj2C=Fr_ zXvvJOCB(+#roFVLC1tC8*BtWRC-Ww!Z6Iu+17doqb2YdKs{?IwU!7LNTj2|Qk6h3X zSRs6zt)x#I!S06>H~m*Cc$0Vr=)n}q6tM!zKB`xVKW$mWoW*hj1wNGuJ@B~J|G)>k zK096%3Mb^J7Gm%yIH|Jlx7Zu6n>rsbl}!m1Boat8rc`4K1ybCO?GNqZkKkvP zwwE?T7GM%5a&6j)waHQ6qQV!lA&KsK$08?fe&HbqX=exuB==s1TJ zN1MIy?c0;$m5@F%zox$DJB5oBVThErfa+UkuUlPPG5)w+}CnOBS z|Hp{mH)+Vfo}v|`q5Qf=F1~wFL`6(m8u4Gn(81W)#?j2yDXF!;5HZ!1x#~NocXDq5 zhPKwM?~QB?j9Fc+?e2yk@w);Lm)6Ek@2Om^t!x|tt^zcFya7O5-`!@Tq59($CrbgE zcXEnUVzv&(RNSmDSzpo!VpCC3@jDoq0F>WI{AW1glK_pGlan2Qjm^cyh1G?F)z-n3 z?G-OCFWXCYHghE7K+)XQ*h=$_xivyOh&2Sc zUvu&Q@&5lj`R|H<8~N_PBVTcHz5M&ozdibWsG6g(gP5%~VofK(|8C8H2LJuxe+Kfi z-Ld{RQT!?AKkgy~Er`v}_SdEfVy|F!`XM0+BT2szQFXnyH;>`^M4YtsCy^*7HT4HV z_d|%w!#8h&sHBu0J$XXKZZu+R*YiIR`P)rFW>zZ07kDLj=6kd0{|P~NuCLI0D| zLn1Sk^x&>Ush5h$)MDu2^on2pphDQ4C=Zy~a9z98ay;02L;D-w5f;c;=mjTyFU1)q zlIkDcG3JYhRRnCF4LtayE%KWLmL_qYk>r&qH~Ks6k6lcjMN<4J#|R13%QJnYEKb;q zE%bj12Qiz`ak|H{`tQvDCEX@PU*T#ZLC%Jd|7%MS_s%1c{4Dg`6XtOKIrU$!FMUKt zEr9$;A2I&7O#`E1s$PwbtpWdy3brRyiu)h&LRo(o&@Xv4KNC)y1*bee`#0@zO-E44 z=4s|c{4c8WgYF*kMDsCcJ<-2WiGzlz>ai@A+4uX7|3_PW*{~kwoo46uK>ly_MM4I? zKv0Q8Nun$9+s^&^=qZg5Hk*Ll0ouP&DT<&Hcs{(v{%^vTMF`t#+uVWe->4+QMo{@Z zPy+nuuRZ>!*!3PGge|KO!D=HDu1Gf3}e{%7Ixy3W%l{~`@FjlS&93LjH;c;g=HgHH;RP-AyROLqlTZEU=CPTqj=n9|6;@T}rl&6B zyU7Gc>OH0?Emb>WqGDN^jS`dafrWSMru8AWi_dngwO!1YIsA4)3v0UT9<_AO8j}`X zaqt{hZV#{8)E|EFQ2POBE!rTsSNf3ZBv6C@Ip9x6^p~~XSya~BG|^C47><>0ZCknR zq9(*B*=uTAkkUh>rU(>P!A)untelwR1Tu}kU?OHWvKudta`5AAVTlAc;vGNDn$&2q zgZp2@2A*t>!NtFX0is|$YuK1`Z|+H`F21o^U^GTfoU)4gED%^6{NV$-?EdqKVy~;t zc$HT>Dl0LkjE0TUyO)tYi$}FLbBjbhF-$*mRSVRBFv5OvJ$w9Gc``MX9Qc=1_L%A8 zg2d@!(o^+YI7El$v{L|Dc3q!kUx?1L2dR7c4-1I^qA^Y|Zq}V_<(ZwhoKiC=a;9bq z!j@jC`3x(W9$39Amu8$Q(Lc~C)t=%-nI^Dos4XX{{L1X7o7iUD9#J|!I1uqFK&Rdc z$1IO9@RU4uIiNXmN^o~`G6frTw7_@sv#!=Y%ZLR#lsOR;L-IN6uQ$*h?-gv+j&3Yz$@RZh@JZG?l%Wubv?~LzKFYnDa)Fa>e`7E&7 zR+83X%X$3I{L%yWbpQdDs>cjRR^#7|dWjr4n~{)bR3s_*OEsX${Diw!_=E&6Gk*Yo zl6CO~-tmFa0ZFNQ+4O?$6*(wI*MB7}VDFi%{z`Yg1ANYHch2+UkxZ^6RC+ZQ*JyDN z1=cm>gNTKRw7S6kz~|7Z!?!Q->L(&_QAa-`#npFU=Xo@0&8q%%@>VH{Ww`#KksnfxNarsroFi^ z$ncroMyl4V$#bIW=&i$u)?fn-WoHj*<|0qv#nD57^SLi#=)tiAn-W$#R3}$Af>8uG zw~dd$n+boHH^MR~P#GtgH!znSWE%zZ%9=0$bMoSRcy{?G5PA(}1g)re zSlJIK=t7E&j*s}cm2*quc)ZHVFvBD8B{QDY%G|yydi$P1sU~jRMoWd=7@^+92VVzqw(9(Vn>jt z$RyZW6#mf`$^;;gifu*8G(Vq&JO=Ui0*XaX92lu|MzB?PQApLToKOpDOF(dIjh=OZ zV_wMzl;Ch6G3&0{&O?up%yLi=wysPx({r|_G3}s2jj0N~^?r(@Z3T4@xC`(Zc(`>& zd0Vy{%dD30{+K?exXxB7-Tsrl{wJ@~I#XA>DK#vof$h>qXao*R%vxF_tUbm8tk75>92O z*2?AE)G_uX+eaWgXA=BU=%PBHvoi@iuYEdQn5Aru0F@D{vm+OIEp9GV{9`}q5Jq6s zYy6MH)-MY)x=^|Im4)Vz^L#V4pg_9?)Xc(b-r|H&cEUY|%r@PW2^tJk=>;C=S$m9p z`_X*XyhoT-Q#%56uYx)10%Ka94>$^})wCEaTS!LCTG?IAhxQMk>TSwvGaCv9*$e7z}rPQ49X?s#+MwPF^1-`z9sEjcOWOBbE?ny5&@L~svDv^X|lAAL4s z%*(=}skqAW3X2g7OFi2s8nhD(n6AM{w0|mzSen0V%FazA?V8guF)d}3bk(JvDJu)) zb?Obi_LPEmt?eAcB}OIU5H$I!jI}m8dON1af25~b>lr}C>#EAC*T5_;OYqfTJxFSz zAtqb#=k{opGH&zORDq89V&^Q#8olhQSzYPUsE zF}6ijklyLZU~yKxGG%8QY%oeMvu*lI!yR?aiAh1-KJSd?+AoVYCTi>0 zeRKoR#he4LC}zd6Dw>HI>?6$L_l4DJ1N8QUPl4PL0-!R zzC~%e7_BVfuT6I%ak0g%-4PWHP@|NWU4D#K8c*u6Ul9tM+0}b3LBV6`lF0umEf*x< zxC%X!Af4hmb6g)_ux_l0OnT3&*1UD19)v3pbRl_icCc@46Sc=1FtA}$+OuB?{rEWV z+`fItg446fF~zg~aft0RJFJ37b=e3vV9Ow6o1*Gh5i>#G!nheZD3Bcte_y6ntZyk_ zKH`=+nNJBY84>b5^eKpDZ>7DR#lMIl;^Ou-S8l|>Lid= zr*17yJzKZJXu#^)KRsZ&VJlx=77l!#D(5-my#un|J!T4T9@nif9(vvegzq&Bra$@z zLhu#tMRSUnrpQ}x2h~)-wY6`pEn*{vAJYCvq&7s)S8j~F<}7P>7o5DDQsOKj{c-wZtcn&ZM#wc~_6#BPaRjKL+^Vz$F16KyEEnF0sJH(1*E(L9Xm-D zc3leX#a@n&_>!ZJ#tCn6JJAyTOMqKs3Frp({X zQVhLemivuact4|MEzi?z*L=hz8qo3wEVndn@(=57<)VoWNYfl)#d z{qu?)*%hJTa7x~EK@fcOsklu_pgcwxlYBC2q*L|8SZ2 zP?govA}{%7fKJ1jMC4AB7X|FL`ia8^fAv0g>z>n>n5<*o^4;lh zInuLSPcEluh>Qhi5hJelxZ`+vfUp)3m16Q_>cbmHd^@?@_j>8)LSP*&wOzGW-b-;R zamO9~3f==aP$BV2P)U82Q*T&2{w;SRX=ysISeA66#OQ`!V@L52a=WHRr zO`?kTarcTZvwg6-4ik8?y!!?5`8nSX#xC)0LPzhDuhe>YcBGGrb#uZ~jjP8<8-^~U zb|v-P-#PW-R=(L2corpjV{_o{If0{^B0z3PHm1^i%*NzSunE=b^1(j%`9|A#K8I_q z&pfqzsq|92MK&U zk<>cxWB=8t@h`MK;1SAOGw!dNJNT=e)+90@FdBUkx(3f|KIJgtz|Hk6J ziw}d<#eGgESXNhohp}Y!qW#Y)^sXR_8VOY;Y)W17JsVuwbLwwST}LN+JBvx4Ima3P z23C1QgooBxlnctXRG*s6G?^q`CIW1tt29mV(Z*F%#vA@5{D^5a^b>8-`}e&b%-0; zT+a8=g!%rL;Z#fP=u-901ote`T{L`}Q!;Kshw0-*lilm#q;y0Sp=7AX2G?iXpl{Kh zvZ~f7RIruh6O_|s_xNbIj4A-T=s3Fs_>W&ACwo+LjA~A^ao=bVBaFmkofvjz?Ji)mF9k< z!{wTJSF;zr_No4rO+^ie%4-L2Y<_U!riSg|_9WF;>|}#y>5(K}+@%8BjpkwZ92=VT z%dcVZa9iSh)qdwDnk6~pki65JTyZ_zd*n_m>J52u`5aE`8xz`e8gLy6Exf3F8A#Bi zYEH&ZlmYqE$+38+k<{xQf}q->;9KgVUEL$6`mkXeJ5$}B7#_jfXu&RKYYMCFcV^Uc zr#)3Oop7zg(^(1MSs-}$^qW_pxX(=R;89U7g&X5lTe3q$b1_^qxrTS;aZ)08ui11v+35zu`4b;3PO6OlvN2fwN@G#uF6c2 z--EhDk{nM2EG$r5Bs^i$P*rEuqP-o|w9%0tVx_q02_HQaFqJpGIV%-V)J=CM9toII zR*sxCRXD8PNd>`fBn`;|WeUIU*%ne&H%nq8FE9)5;Msb+Z`U0XSia7EVXahnzf`wj zf!Y93^r9TbVVdnL@R{r$lxr+z6cQ3*(5McnF6G%1dVAuX*2now*EF_BWKiJ)$a{+; zcPJrn9w;WN!Y#Kwnk@_4_ZYmM*+`6|?Hah*^%BS-Y>+B4!o!u`d)f8d;otOzidj#7 zWLtor0D`VM(J+`+Ws~bDO2o9v>zafBY7BbCMXL^g0b>>}(rKXgxl>aeddJC6J;~bx z%=^2X@d(gembx=Nm%Wu^*nAkjx zNJ^aU2Ss#g3o+TAUI?Q#cJZ(h>CBcC2@}2m-w@efNO) zdL7AsJQkC5fw}dw#ed{w@j4fOo0IhPBMMHQD$#pZ98OgYmFT_oQfWu=@FH&6{2;nH zW&5@M)wf5>0NfsajTVY9yPOS7WtDhX9{(s-%v0 zaVQ@ow`OxwC3)h(pw;ksZf4opBc^LVP2aR^zFJB7YR_Hp_Y#G=%fD+#Phr2Z5_RYY8U5+&3tI##Fp(S z8RXz*&+VZ5o(6fDgUw@#p~x+VUmlz?);yA`M1(41dztBmU$D0LM=sQM?sM24t;>SR z2L#qfM>n6aDk*xc5QAyjK|zu5d;3~20*8&*+2ch*#8C|Ij2Zk$92^U~W~g)%-SFML z9a8W~xY^T&s4Wyh*L3#WF4S+3xca)!x}*jqCZ0w4T%W)Q9dceMEk=<)5;%Ua<5+xjw`&|Y3BKj(ljcmg2dgx+4yUFi&PGo_= zgp5zgQ-|CGWqyv_E`Kw@ubt`zG0OA}4$SYURIb2t0ewk}gn3CbrXhVcy0yBR%&~k2 z=2m;V-)vn#O>HB#0(NgPYC&2Lz3@)IE?*v=2BvznVmsxU=$%yGVHY$GxP4=hyn=y( z(#mXF6-l(DNUJs~ySuF9Ha&6)atd;vNumWVV|-$m#tjNrx^jJuV|0zpRGqPxq+{}~ z+qV$9FaQLf93=5PENLaLRJrTM{S@8X;=}PttbYg2QxS`KxXnqv?LBS5z0Z(9qh37w zR76U5EY%2#GUefN07fKD2ihigfA}pvO*Qg?cTTZ``Y9=(??NX&={+a-ne$t4ef~i> zTPx+(rg))MU_a2UEoPjGE!kzZ`q`7b$$bjSA7xcLp1HdW6laN5=Fd?|BqBX{JUEZ7zJp|pK&=7C6AY^D8(<$})nk1OE0zs02~+%Zc`;OiUD zb8>WYu>gWCsUu=hON5kTW8LNS9x5VZp%s9DMZF;YTSd98z+%B%QCvhqBUC5D^A*Et zBzR|4Ban2=3z6A3C{k}S_-5f&(#Q1Ji^lsh6u4e{WXNM;5eomf^p%Lx$rJDT`oxIW zo)dlpS}5B%va*(3L^tpkej0Xv)Qqt&iBYAOXRu89kVn?F{S zv&EIMC!X8DtW{8kt=K}vhPTTE`0U9c(`e&DTv~$syJqQWY~PMJ zW)Ly=lPwAM)iua`gbkhrWEq1=`H(&PX06p*riEKI&{iDL&UeTi^aPnAhI6ak%Rheb z&QmK;8#(vex@0ga9~H;fab3#ByOdd!FO@>%h1-sWPE0_vwc`>o`{93vq z_b5_jYB&2JEMKwGCy}0yd#{2u29e6V+{#U7bW!(Au_iZKtaq!`j%cWpA%J!M&?s`8 z`7Nm`yd;5z8i*>b$>pJm_nlEKznv>!oTq?N8d5@mRU+vC`~1sMF8@ras-eu zKR?XdXlyM+JN}M`NYV%=y~|f$6z~D9&mQ0~%$BC3%+Tg|zR;yOqh2L_Gl8RXT)J3bX>3006f|=e^+y{O zl8@=o(%D!=XDpM~4jfrs68;sgq|Sz|<7Na`9V;vz_XuJ>+I z-?pc;S#-?PW=Ou+)wfG&;$0=pS}xwg&o$nhw5u<=Os>_BTVsmuD&I07IRNO2Q4L@o zAHI4s$+q|c>-zjH$4wUPf1s3|B^n~3Kofw07waO48_Nh1V0-6c>CxTzXBE!Q5_MK} z%iubMjMFS46J;&W*{&?0wHoa>@HJZSI!ANKJixjdv*!`>#BP1~3k7XR77t=#;ZfA@ zZEfN{N`LtLHwoO<*nf2$EXCaA(qZKz=Kn#yFVRRi&8RT&C^%i*?m8Dnt6RG$eVB)Y zZao{gmp1-WhHd4$D=%+4W={PgTum4wKh^Ui_|jqL(AYjpr9Le_#KPd;dQXzXVI<_m=hQe|JNe zgz&ZJQXbg?W&9@(si}uQ1pH{Sv1Jk9vf0IO?h&%Nl&>&|#Qmw$@68QWAYU@R z$Dm>pGD##q)hBs~Y!JH3q^rP>gb+ddli= z)zLk6JWaBlf8r7#OoHedNk*r{)`iLWLHJH($IHzYn|Kef{WuvFvu!?d&T=&TRguin z(HEX_XK2Ahl&<_NvA2Y|5LDVUA^Gi)+V=BN>0|Ow9H2dWD7^F>hkrV^(n(G4_rMmR z%g{SadU}-ZBKSNlcay6`b-3#w2r5Elja}slGML?a3PJGln}YEmShIPiN@Zt=Mxw#s zX}IWH_&b|_DEpEzGBKp6#fj+m34YDj+I81OGmWzTKaEVk_RSU_VFcC+A@F}QB*Ev1 z{**3?7qtPuoWgHF%{39>BWyhkj{l8{AFmMX2%ZYplm8;?H-=lG2r50G_?a#LadZE) zWJG^c_+1xGIY#gPZ)^Ycap8_iXv))@e`^Ete2Z|4;R1>_82=)xAKVBk7f#!^*#5;I zEEytp(reGpf$!g_ti7YMhF-$w-@0fh?}TlL7Y+T(1OFKx{MroECOHI^wF0j$ga1X? z=Wz&OQ+lM$J@_{&t?wN2{+AKGe-rlq3zg_zvnJ2^+h%ObQW@L=HLDGz{dtl_*rkMH zk+6D6n9;C_zJBkJ`K;GDe#6DHF{gQPHMCePxOLXl*yc4UNcwjvg-{1-wdo5f2 zjj%b?$zn}vTmG{;Z7)#*k#E$a%|`*FH$P3E>G_Fbamgy*GUVUU5V0nK@fP;jXeHz-b|NbuB?&YhQ?mC1xC}e*PjdfLEDi{* zW4P;t`Ad~~l&B0lLzF9lw6!i)ZZ4AB7vo3lR>Mp%Xgg)`YVanU4rde2P|Ia$JV6Jd zwbYN8TU`B;l_KHsH@QBnlSj-ug(c4M*G<1FwufZVazHoBxa&rzPmeuV)t`r4&NcugT0&smDr`@$ ze6G2bXioRb!_-QEvOC;X7W+`1!Q}9-lr0yj5E+2bah?%g&vG6g=rXH6{%51rOqE29 z%^Vf=j99kQM;YZBQDgJjjOz0%W9KR!mq-T%EZw#y!qWc=0dwvxf&m_t#Iyf;$GT-? z_v0g#A-o9cQhsa5)B(A6XqwZzW@PpGc1p?K%L%69n;e(wSfK8#)c8+BiTMKK+wLL5 z=9rnnnx2a|M|OkhM0IG0x3_xQ<@88fm|=FGTJ)Dnc=hZo&&KUZV5arYNaL)jF}B&n z?wuIf^=eZO>2^3I>=MoA)_hN+NK0&iR8R(+h~?mQ$sNtvpsx}J3PMBnhzB7=&*&MynHi+F`3DuFQ;0$L{zQv%Lza?bfpIT zK$o)Su(y9Z%2;?2=x{MgCkgQ~U7fvMgHPfbx8t?QoM@fw1xZ;qxGF+`9#VkGb| zRQtbBN-7r_4Tp%J|mwcJ&3efm;^;6LVdMPF1==*_bm2P1s!Ds$>tUxWtAvr*4%f!q9r~**bHl$GnG$` zRru6wlAJg!;c&>1zN-{|=Mf{E(T*{X`G84O;oirAZ^#6ao%v*j?yxW_YElb?e)zKm zLgAO((c)5EAj(yL&|5cxEb{H!V{!sxh}>~c)TFa0sjde<14PeT?ASl_$n=&$EYmQE3ZZ{PlHl!?w!v*Z#u;S_>?^4r>)*K35kQ4#(Z^En7sQ z_|}e;THa07ZNdZBI_MjFnl#5kxp!45_B$L+r87-jG&fymr49t#8VErM&2BAzlBfAR z_1w5W!i=cGOx5_1_5_jtc&$vznK+W) zIHEf=Su0ZK?rfF%)XIia(@UC3*yCL-^)>+Z6UUQoRXOR8Cfp^Ohxm@2N>R1KE0pNm z)+4jn%6#%ogUKF^rxEW1^vbnvOJGWgh)%`b+e-iG29d8TblE01o$64>O~W<2_IrpV z&f0*saE3xhIOY_n{%DQB=eFNr`MVp03FCDC6%hZDKKm?s7?dxGfEc3s)I-wO}Kf42Q^E)J1tUf63(m2Q`4z9Z!|SXu5}ORqG>Q|-cwett_I(H z%{79;SJl;489#OisYuK;oz;=?yZ07Mk$OBxjjq4le-*atTtX)?e^sQ`Xx!z8?*MZu zie^iaWsL1+i3wUieQj|gHO{Nq!ele4@Vr`o-C7N&??8;+_;NIPN8oy`M5`mKZinLz zDocGqIF;xvdqnV}qvE^mhK0`pY|(S~eo4EGPWA`7kg93@D7(47M{Jtbkca(%ULBi& zA$EG%lEM%ow=hbU; z+h^ZcM%AOmqf2q&L&$cC)bS3zNv$FS;x2cEpvL3qh7{FNnqF~bNOn~Xk za^Zm9(+EkB0Y5h;4?G|~FVtxuJF867+iZZDgntTKZDE5luz3-*Op2V>Ekx3D2H#Ju zknuievPnie`XG%{`x)L#{P@L{w|j39H?ZplA`OI_q`XP~0zx#%?qtJyr`W2ZgTz-= zRJJdDcqk(<7T|KWLBKKxSla36`{5_Nq#ob$c5~tL$6jnBtanhJsvZHRqL{keFsbZx zk>3u<#`SPfeYKMf`@;mEsp(JEu-N!5(sanuxyKJHZc?s);Xbuf#K<~qxB_0>SSVgz zU%b2Um{m1*Sqxx5sy(lwK%fWN#=&_qjn@>!+tV!z$V#}NafSb`;=$Q(9h8hpT zC&$|8iDb?VaJmjTzWjmL6UiG@RhW7nn76^3e*$=aK3D(3^Dv*W1vyuQ|03pH=gi#9Ch6!uoODT*U#aexS2>Fo( zDOv!%k>lv6a&thHUSFx-4-B!+*{uJnc$Wcs_M&WMQ9R5L@H(kQaV1~q8U>kb^OR}-@#mCr7LeNbv2#T?q@pyZ@$cECQ_iTSBrfN`t z+))V#cP}Bpe1MA`G1b5m-D~jyCd!ow3trTa9p85B?A`{qq|g`dKGE2H)ZS8h0he~O zR%)zUQ>M*5Y2Y!**Dmobhs)XNd+Og#bwJmDD!{8QW9c~yx2_lycp8gl-KCmS%4h8W zrN|>gjJcfzIMNGAngg<0rR7Ad((UDAk8=iXRqM-`m9|wH=70-;!Mks9MBF7kdSX`j z5f%#aJLVKP{GwpmTM{tXlo=q=g{C&k!_2aZUIjX57a02X?JkCh$nzaMyY7qo8peG> z!UJM{yY<=?)Fng(4|h}OD$OmiIGI(>@uZunaSIsnyb!{nd+|DPbv#TrIrzm+02=mo zQXX-U(h>rlWLf8yJWdeEU7gE1$m$fJI}tFyJF>Ir{B{_@$E`M@YoQ}Jp*H)WA+eWU zUA@crKey8bJu~8?86(LUdvZNL@%ukoX<^Xj5W8 zJq(3O_4RiCw9P4f(4F=}-=qO0uPB_koDB0VFEDxgcE1SV_( z4=u0Iv)}m4piBUKtrW;JBwg^H*0;*AMDlSe#(Kje*7^vDK2MrU9TaJ*I$b`}b$aR5 zdWN&c$5k<_f~9otJVx(S@g9s>V(IEy-1L%BVN)>>!IO?Gl`$n$cX#J5FX6<72nwyd53g7qY~mkN0rKjvi+%BbGalYYp2^T)U8@jDR#I z-?kk5Yv*)>zn z*RHZ3m_Wlg!Rk2rs3l@?gdAR*&)5e)D|;jFEfE{Gm9m0pl$y7Mu+F$INR)mum8ZtFl35$l>Wsj3#GtFQPkp9`c#-&BVLC;VAfrmydts>&GK#Px_(PX?a1ZjG^`qPX2W$(lH|SFx%V zPLb*bN{gT}iMK(E9poKQ!_@ecP?_7RrL7_EG~po{&;&CHui%KtN!qpVmcS?C2m*Nwt@- z_dQFbfzBHGZB+EO+eThsmfTREUBIDl-(9k;R-Wp`>vt5}^9mH6MYUU1Bs`5y?q#{! zX${gx>XCfRdj7gU`{JaVQ~6ac&$bzES(7ZQp+wDnv)nwQqGId80tK?n<` zT6%K6w7FhkviV*zTON1%aF-XhkHbQKfXmjvW!ncQ%|;D=Q*=OM?;^eW#-{g zgyZP!KI%yc8uYW)s~b^C4Avm>siF+#rA%cX0Unxb{ik!{R!Mle_h5#{!?kTIJ{&%^!C3GxQBdlosYP0w0 z86DZ~$(N&$sj)~VmuX!0T1&ke<(LHJ5(BlF3j5-T^MxzQni@pY3`L}v#PyFt&zl(! zc8}T83e&R(l8pEW7(SZQ<^?lL_3o6~{ZJx?C@mpkhNe~+OW@iKmTWP0GWMGx;JER1 zlH6#8y!7_;3qd?kx#vuAjr(el1$2K9JFHkf|5bAeVzMId(Z=UJqSzvTIBPr z3>HXQb0;N}v-Y7SP$ls?8yzP;TlE{6&6z1hUy}dG$_AI~UA9mMQDbQgDnlGr<`M`b zjX-%Eiy=;^h)i$3a9<-|5L{-T-Pz(gZ{1QCvFrCVE9t$XXR2p+SMlNRoCF5uX1C0@ zAgPr4o*j8}P-v}HBquC^5FT^Vd{{Fj7%h;J>l>ks!AJIK{>RJz8 zi&jQ+_#9v8zVf}8Jx|h*$*j?Q$UeUmrn`&e{N*}*v=GGQ*5CySDA$~zVRt@7H`Lbi z>7Mpk8(SC%LBjy8cSUJDJxePsZn*@N7*y3{rf-QN68P_T3WS45(=TXsv!2 z65SJAqh&2%ZKPcP_Sno?S3Wry#F?+inXAf}BFVicXfW<`wy9;f+#XvUOnL=q6)@_5 zJkvxkI3RttF(~f6PjuY3#%=`ljA4W(oPV zGHYhbRBLS?cGer-`Og61n6BnC#a$qmxt)i(TH^=%wA=0r^;S)d``h-3uOiP+Y;f1B z4~d>yA0r2c`>@OAK<1SYDJOvyEsTiNORHFWom`k1?E1Cac}cH>#JW4h(G)t0{B6Ab%oQ|y?k}{Qs#v+1*e8bmE+7&3KlbjTt#Z!se3|? zT|CdyUzl^~XUvVIRat4^aKdYh8j~6&cEZUR_YI6xb;9=SD|vb!#pt=G+Y#v9sgmB|n1jA^Z!}}51ZVelOXNjj|T#e7UWHw`(&nBx*|0vj7f50Z%+(RRScR~To>ogU(n*JW zeg38t%cum$h*SM`t3^x#GN;ADxqa=?!9RFs6n2@YCFSiiq3+rxh`jfweN?W{N22;lTqLqOng?qi*~ZG*hRP~qwM3#`U7 zj#W_m_=Jhd$kW+jHt)gQS>VpcHz1^5Ece7@HsZqCTIr&=t>=-s9* zeF$Gw*(CoADBYqEQy}Ci)$JH1*l-;>iRGerF)E5%>;=Z9)?ta)7*)<6ySC=zbgs+3 zD_9RIdN@Tz=@X={uSH?x zH1pCf9e6L+rp9s6Ia}l9Sa!_;IUP$e4nKbSQKNYv7t{X3U5?aZ)K^4~Ejh5xZ+R6= z_o*QS_!ou99qAmvqg)X|ify-%9HBvxsF6`OWsU0Qd8ZK`6 zTbMeDJsEvah&NUGU$X$Bk6m8eexLPW9*7^QivZo$MI3if8jbMRUhAjnQb$oX!NPUI z_h>CQOW(Lvod)}F?)2GKNrmo|mB0+lR&~QSjSM>dyg;{T3wYxOy3Z+&)HW=485w_N zE8eQ9R=U3BN3d}=(C$ihJNRnWdFMw82R^M4;&xQ?=f(M??V4g8c{Eq6YW!y`nNI?! zE9DD?+oJLjL0^qeoHsMQvJHAPvavF(S^Z%ByG96aUXRX3{}+%A{fEfCH7GND23RHi z$ssRY4DGCAU#feO=WYzA2?G-)B4QLu>V(hy3UnK7YwGMr>rl})Ku}4FhcwH0-S^Mf zh|&vr&m0;`EWy9LV_xUf?>gVcPnAznA#&r7&@~S0NWs@&|R&#Sz<8T*fV z&cEHSeg74c^8^03zduC;1+4mIdIH5QKc$=KN^RLij&R$D>#6Y1Nl4>s?YL+d3_ zV6@Wr=#=)98}#ZWGH(6GZ1nzZ`F~#gczAo3{hi_1t-E~nA$WH?4>EyEQ(-D}m)Ct>+s@g)Ans_C90P^-fiE&Kb_1&Iuo)b7(6 z=5hmXzMxloaR^^dSjX>x5;p@`Ikzfr3-h<9LZWw*TkDDq5^M1d)ZTk9P*9?3Y1t^3 z-iMJyz%`rlhLg5}QzwDgPk z#|e;1-(xmlE4)SXa9sZMOOXp8$C#}$xHvW`kQvLU@4RvUH%a|-r-Cqhce>p3_y2a` ze{ByQB4F>s6Z|Q){Lhn|s-Nou9xmtj|64cS9lrqfKF{$x$Uoi*F z(t-a%S*onB$4KMm8KvD!fllj9L<+Z~_Sa%%RVL%!2vZH@JWV2J8J^9f0gV6SCin5br&ks5Q7ny(vZ?`G)0K7hQZ3dy*JgB=xG(P1TKcuiWg)#wf<&0kL(WX z;IQ{cR>T@i9Xd$J)`Q{nY@MGy=!KkNE$$VF;d|TR78n=t@43CS#`vF4D=(N)frH`x zbbw(9P-CjxY=58@Ock<~X`=SUL3Khc9X#W!E8{oTx(3j4Y&7nS&PUrH(2GTMRCJbs zss;kmaWws1n51#77KvxU*w*B1@Lo%~m{>&H82sX86wX8G0ClVdC*Yme6XdKf(kB?P zjOM-9X|S%G3svd92>3A1qu|bCDS8?xwq3aHUGX@sR+Y@7QhU?11R$@NoDS?bNV^n> z)q9(ttujH||DRtT4{ldUZ>FVmtQq&&lf#FJ-##f<>I8BOppKK)hFYlGp z%r&&6Q+P~5TBoEA!J5k5msX@wccIlbiiF(G$~0U|-vSbKtf9vv8i?LFVj=nocccDMX^@9 zU7n|o=ZdRUJ2yH@U3ouSrQ{>~um7Skl)Z7g*ll_?yiP;_Yp^(uJ|@;47q#HBKVLNe zA8x6kuKm8&HB)5wcyctiJ~3=Rt_?;On%$A-jKl;9&nF&A3xBTE>`Fh z2lC+3)(&jd6SCFW8V}HPq(c;Vp6qiidA|uP{T@@bF&xmZ-G6+vOFYwQm@Jo-b*U+! z`^leZv~{ZWoI@5~ovW%nw)yAi5^Tdn3VLdi+=U5A?l+Qe3@3FNbxrv&xlFl`251$k z5){f#a&re=`0+GE9=?Pka4S3;-9k77cOBV(cto%A*NWv6ssyfE_Bs&!LsEQ%N&5F~ z$MPJF;IID-y3^4{soBrt)$AHW9z?T^26b=^2Y+9PaD))h)8hJTOfwKe1Yd5-V4R1_ zXU82BvUCs4o7On*guU6yjRK%`CUrmVS-P>5nb|7iC{e|9kHogw*NV%_H7OWQj~-ht zq+*NBB8-V#4K1;a+3r5_@F@!Yym@b_q z*OJ28nXKVeH_zxdOl$Q@Oc&|kEFVZ?XA##(G!{5kz*!(iyp{w(`^W`(CuKCew#bHC zin|WT=#u4lk26JpmRif}FaAj)=HbhV%l3Rcs+;3%!P1EOZ~$chF7^$!_g=P&LVCi* z?kcM`1@{6N?I^MkZr8DhAFWNvH0Z+iL-!MC05)CvE&s)xJ4fCf zy%*n-fJwWlkORLUEo0m>!P}hs*R9a`=Ur%Sk>Pnwf7be&z$0|In{0bNE{M^7>-%nj zFeS;(T-r(AU=aP=wG%d-;QKD=bip%f&#Ocf7PhrP58W(Op zUswZd^@_Z=k5J72SG*;!qHFn7Hx;nE53vDCM!D*J&3~$2gw7Ige2t>hw=9{Y4;)PT z$31p}H%4}mLUd%(2ifA4zA>(;)@<|>?64)xS!Rdt+5n~+RU?BV`ORok9D4VH7HC`P zZkI=PNYSU^NF@))3F=vIhxvEMDy9MhBy>sT$OE0e8o4F|G)xwo423GKUI~S+Rv3N+ zL!~bl{ln5D5oz!P^CM^dp^REK#s;OKooNcEJvEdLN+6EBe4I(D)al#CQVG3X4-;}A zjT2NmvzH+iZ+vh#_e2NU(cz34^KrN`ls{bf6~eioL$pz$+6m^m)4I5SUlAV>J;K6t zPBZNFU0;o-W3LXa12+Ha*P-<_JNEi??2!6u5S&^SAMzM!u?m>;_f#xk#-H7nhnqO)@zRk|f>&6oRZV{zG!|KNd!&pAl&fylFD#_U{WdzxStWcK8F0`YgmXBW!2<%rNs2834b zUf-TzE(@@O_wJd^%!>Gdf?T(vj|)Z$FK~S2mTdN)X^i(dS%ul6#AnyrN&97+LM99= z(_=lWYlpy)8LRVr!`(Mgq%uqUhq!P5r2*dORHMztjHHC;{IyhiSDf;#0s1ybI*_(9 zx@9!tXXvOkZC6&ID6W@O!GD4FNJ^E4G_NcQt1Zbwlfx>E2%?kwhTzhP8KLDB|Ha0_ zaw(TTpFPugqIm=oi!?t`EbMEh95k}{XNtghW_tzkNMGJXaoOFlzXykMQvIRb;z9J> zaeSptUuUJxhy0eCRVqmF;)8e#?7^QE%F7YiZqc1SK2dxsyPmE=U^3ZK%r4WLP#!Nk z>j+y{6RhIAoCWy$ov)@m=9y36rau);`weWw%e~n|h@7`TZxOMGf^e0S4b_M~(a1 zxVRlLNc9Ijo+|FC&_$T0bEU4++5`CF4H@v1Up`3-5&^^Iu;pL(?*6+;UXdfpPETiP zIV0$)q{p3By?YfL^NsT>L9^4z_uq>xQenab%HG9DQKXIp)N+kkmEgL{HlPij9u@vq z=;hC!#8KtCa(x(aBTHms2WQ&`oBF>$7fWR{`g!Q2n%pfUHp~iQGu?>m>S_kT6cgjV zc7n7o7ymL^jx@y{t!>@OxG1G_Au7(#YRdJQjmZJ>OzFl%YdD3sQGZa;f4C1|mZ8&D zt|Qs)B*Ok;z11ucsec>z_clmXC=K`68)$u1EYh5>9Qh|qlLiZrs`y(UEd%fZxpqFK zP|~#_J)Dacsuz1JmA@R+al3B(@@STmQvWK+J_4fLl6M-GH1s}-G#c!ZT;KbF!v@j)0)}NPQCC>U72~Wc&Y%j{3Sq=cV(Ajk{H0O^M|z~bTBO=70ge5 z$#`fn{Pcc2Ro!nmvbBQ1-#WX)?6Unar&jOsa5R@ascIxR0SSx`S7%THBJiq5m@IHi z&K~&5Pdw!z`|T1|v~O1g;OoSZ!lQ_lsGHJiT;j~l1P4BrON{kZ*ar;YjW;rjpN|`1j$L4mtID=NBspWqA#e}(a_sN8c z#HxO7NTbY>Gf>fbDuveRuCCb}w-Z3Vtu~ijJEk>r9-BEkUdwb_%RF^*zU!-gN-llmd~bk$G4k>#vH~y76Xh&1SvsFR>g!7=wfoW+ zRi8!xuX-Z| z;SG-mS`06Oy0SdkO70z7fq?xf{K>Z8Df>?+d_=)^saOdus&i=esouw*RGy5mlu`dEeRI)n+n;r$I+Q?_fG8*tYF73;Z>uY%d zbGbs3>;J6hyMTU?JUQasiT1eL9x>K&ByEP_s7*{ggH?XLCzJ*nE?M zk;9;zRpR!z$kJEzeew@+-n)kSxDOaa#&sHj$I&1|qZlfQoqHxVZ85X(T5(@HJ%q#9 z?hzo}PMPkf&a?M9b}UDB56y`|XjqMA;@bk+JiT_(SrKCOthhl>bO~m7Wy;mwIL_R| z3DyQy<)WrCiA}TGU2HHs#G{#d3nj4W4cK2v_`R>g7tt(#y-;XRF+ZLIivQPrwoH4Q zY+qPET0037g}zDPhqEP1dF+x2cNpzLVoJ!yu}ei}x@U3u5ebr(nWvR6sk!xKtVWIb zn7_8`5Qy;|DRB2260lW|3B1)NC{+3P;9YrWZI$t3+TsqQ%ed1jzv}n>;td5)%h{Yw z3-x!lO<`|Of}Q+v7~{Ke$W2mzS21oC>oEy$(m{o{2MHL>{R~GE=gp16q9#Z;fqhb0 z@w`tKY`jy#n-(9ZvA|Tpk|#-1O5lk)S;A+`(E_5&ZS!y%`wK?dp7U z7(0)h!Q++t22}?06x8J+O}l^j7-F!NHnm(ZZx|{rtc6hHle6E2i%*H|r9e6i8{Wuo z%kjT<)r(2d^7&IT0>2i*4~$X9 zcz4G}xm5C_=jBoN^iDbNM|qyz6-Rxesq?_`o1aVa5aX84B>i!Wx zmGRp+j@X>cRnC}XxwYWtJ=}0kS zv|1G7gmMdMyKYOaYu@qd-Ms0~ZqZ@I)SSE^e0xBdO#!Mi4w;*;dT#Y{%J$k`j%hiO z-eCq(kqEE9`2ApY3;saNVu~x{xxuGDGfa%;HW;TAEmH-Ggm&Ll+0KisFCf*e2%3jR z1(_{9r~i{y%w%UC%WSwozM1vSZ#I?gFYNM9G#~axXe6xMB2n!^nP#4?~C%SHazot zGVI={v~kS$FS*ixw&=&kfC^qT-!SZO{#m%!WA4%7^qp9+i|=ObcH=+{!g3+?0&Xag zk8_YkMCNYl0UoVMvcd^m+i8IvSYbewu8WI3Q-!%VV$^!?PmnU++8%JF6J6NmW(8Yl z0kxnZ^t>w`R6`{ubUO)#I^WrSwT)Sb`JBArJMAwXy#*#GBlA#lv5iTwuL6@QPpjjdoRYEqJ*XKW=%(eJeI|Dl1sWjY)Pm zEvznS*N)T?N^R=Ky?Ah% z^mZ3}?nTSu#`RbS(|yNus9P@S$V(tv5Nc=b9jy?j9V1-|BdC(v30EqIFTu_*N!7DH&p!-u!9DU|$O@DaZs4+l7bVDma%Qu+#_XAN^cM=mDXj zxYYSrP3m=qY1Z%Wt`4I{mX53zM++4SnI#xh4&Az0o6eooPqFCCEuU`Yon~`v^pkz% zZ1P}%I@CAPSLfy3Mke|ehCJRg!f*y~VB;E#?&D6wx zNrFh9uQfE@H?4p^d-6=p96n9Gc0Pn&kuO7QU)z0D1MQVbc2Ll74xD(3uV#^lK6E}h zR~_;7;h^iY+?bue9^*}?YNbA2KTbC~3UhCDwk0^+E8expo#$+mH1_0mu9VcmJ~P)y zWNBz|Tew3m1Ssi)s&xuUm$;OoS!*6LXTZ?6W>w%7PCGfpGw4Ku?__#!QzNUk#cE<` z5)_KiN9f5<{fk~9mRU3Tqw=l9AvVBatJiwAUz0Ob68Rj~5WGN*tItp@h4Pg-(xh?s ze%#Hs+RRWV5{Ad-;JyB#SfD;%F$(w-t)vh07)6tDG9uEfbvJ+7vWN~47RFAs6OTJy zXB+tFcnC>BJ(SLSSyS`XL1|6^&6}266tGnTDWCaVu3QzCoL2S5UWAAHwNjCKrdiR& zbWuTSuDAul-yJJ?XRgNuJen&)BJ9jHr5{I@oc~f(YSp`!nPHXr=`|uS*uW_lv45o@6b|&(&ls;QA@w`x1q&!8sN^1zA0ZEJ0wnmQ`Ou8Z6 znZX*NvA~6Dg?!g)?V2DQ>LM&g$-O{l8gzPbBXBO_?yw)y(g`yRElI zM)YQf>UU2BfhxU~wknlw^9U)B4~6^PPbBtbiw(XzAEMbic*xxd5yHJb6iav*o*!IG z4PME6aPP(4EHyN^Z!~%SOdlYykNn)BEKWC*OKqpr z?P#qwT9yBpZP ziKtS!mJV&OW<^`n-b+c#>xO)a1Y{FAU8cX3&TBC?^A6juNOgRI!ZVZ{@zINzDF9j> zfaDt-H;HLyGLlk*7TtwWzP#UoZ`1WYa&?+|SpXJJ(D0qbSBm#9ZZ_BJH}8nUs0AC7 zG|Bt?V8y&($CLPKIl{@z?Y9nBeI0pj2+FCj4x%Z;i9R(Db$*u`@9m%PE-y7n(WHe; zFWunT%t*dBVW{uRi%ofpW(-K}Q)0Uo`P_%ya3y#m|F&>=4py36x>CuEuMij!#t||; z7-9=P1A>e)k|PG)myXj z1gAv9+$M2vuH1kl0XsiJ9hQnI2UC3eiqtC1QwJi0FnL#@?@!%5_tZG$@Ys&${VW)Q z2(R#TUWq#;`v$BI32C~`1mK_;Q8ns2x$Qg1{5kR0n^YW{u||G`fDfHQFk}&Iut?#H z1-(>*=@2nOV&Cs$1@>B#R%)WY8GQ+$$c5OC_dnjSvM8C^gvJuxYo5I^RU$=V?ceZ^ z5$})W#trhjp7kQ&wc^yqZ-V_P@%ynv#sFP~*wqoQ#Y?UY%ojcF`SU&!AFN#_-n_3M zQB$T-_g9&4S4JI9k7VE^i6&^l4GCGys0 zYouq$Sks!d!Tzr7D=vmDE@!x=>@vEy3o=_QvuXE~eN%%sMSyI)RMvv%Gzd3^kh8^YMp-O({m3Z>+*{^>BN;rI!9CnfC<9}~aePZL** zcQJq^M1J8&{vIKR&6_Yba>b3Dis!eT>6ISR&b4up%=$1`!6_&Qj>nb+Z=&b1EWAod zGFvsVA}1+l;FJ->&L4(1%c#bEW%W3dg936?nWTCDU6ck>0RBS1L^i^VRLLnHF{v>7EkZz?nnM+>jCv(?N6}QP>8^gw=1wsP~7R$lpq=>Y2>5~E% zNaPY4Z=hM(L-OD42y|p>lrf=ELJ^pe9CYb~aUM&;!8=jQ>Ie(3UfQmDzJHoHc@({H z7OEN&KX1n8ZPv}+O{6tkMZBXi#e~zRYcUPr4NGtGTaIl2n2oPU=iQJ8;w9Y+(p0R} z7Y42DLj)ReK9G%Hy8UiPp#M@XyLLXSuw4qrzQm?(bP5==b~9KuLauN%iG;M1Wo-J79@`PakOnJ-~z)J5>jMc29pSJymnOt0UHgm^#@-{3ygk-(_l7 zzt9CuKSa0W^TS{ea~~4yB;`Q=%;svT_z6K=0#7S>Db9f|7MnO#AVo%2FZ-!UAguJ+VPs&*|fUVho1moH=5P*2~~ew?_;8`D>O zoo}H+BIPWvu(uywhNq&Vh3lTcFnD1X-so+Z*c}8hoBLq!^!Sj+wU~eI`t8OUp z9UB#X&l@GwUse5r?OLk1y4VR56$mRI%@>ZjdE!v-kIOXLbb&>sT=$wME#>m5wXS#u zquz?V8It~!!*f-a#Zn3D?W7zL9W3Y6(a&^+Eu1dwL(ObE9*kg)^tAf>Qi!G+#}j0W zXse7fhdwqurR9719e2?{cnTzIByz0s!|?%Ha;?Pq3vqu%ZplmzmeFZC4&2ZU{Olk1 z569cA3DsL(bM<8<3}}cJ;&FPs$h75X;u(7N_4$ z%-C@5c{*QmZxxgH^>DeEsa6zHRS;c%8{CK$j7J^uxMA0(vpcb;U0+WX4=3WjIJu^M zS~@4Gv`eJz8ruQ5%s(c9nm9rO(&8A`Fi8_ecV!`gz+Xq~JxRK;MCO2oldzjd-G@{{b?~Ya)ezm8PvG4akJmJw)N7a z@}R;!n|ik9c)`fRN+-|+3<_jDe~mG|gN0G7{$r1J+Mvd__Qz1^J51fLHu7Edapxo5 zKLjUw8%!8P)e)dT)8?T2@vo%6e9bKglpROdX#GVn1c`r+KP}oJB_sb3uG5zp%i8Hk4qUmj8w56cX^ugL5wn5?^6{8$9 z8u?t)&XI$(EZp*16s(Nqaf&Arij%u9#_gIS-ORWhW6dW6{Wq}9nSp9eBw-@x>^_4)BRSve?6^sS37B8Wa2hU{QUxl+3%lr21 zJM{|~ASnFHz!0#j#9HRNf@~i(YwyS~i>%5;hiJiuOQDU{hMZ`2AiK74v{lRd-VmOUqSZLB|!@Po!sTwx0}P*ht7S86%BNZ zk$mQ8-&C3~P-IVjZp>JplNb+l2kw1xVI%3&ODg`Lf5fZ7-seJHNF-pNJGVJj>zN#df5)x9lV*^GW~M58Fl{=UWSkRIn(vvHB=$i&+$Tpj-Xrv>E>)}(mwi@gt6*r-T}sTioox~Jp{7R9^IeZ$%KK@ z7uwPT(mPmUR>lQM;3Ke~-dN#APLCiiCdbqwFzU~ulpnp%kabV@g~oQ~H`{KfLE*(= z3hL!KUHea>-lQ$*%@EEwpTkW3=J55~10w@!^)7E#s_#t~k!U%4qrv)VDUI=kc}ro| z?+^>K7ZJwq*Z9aHd=Ub0WAHj+uAANN5HQMbi9A%2X|#ibDkX&lJU%{W$N&1|fMAZe zv{Vo#X%6ikLr_JPO1?ZLeMWFTdzJk)TUpvWQQbgDSi9rL80px7FMn?~>{5>W2K4p0 zXBkF+OnH!awQ^#ye>#WXZgh|C@u=KZb4~NL-8GSBXhTSAxqF5`y==jUrJXDE?}7qy zN0C!zpJTu9DsL|x#PVsOybAtdcv~Ap9R2!Nb~SR8 zlHjjz=&xAwk{qWK!>ofmBGitxBA7;sxtP5K z>GN&dG$QSGxl-Izai!6P12_CESc7d`pX3Fa7%t2j4SCN_JVDoQC3r@QO^f6Z>; z4LMruUXAkyJYOJvT11;^vFFap&X&zp7Q}C)7zDsT~&?Q zXPF>eMX*!BK03pc?%97;!}g|m(t8b+d`t{A3f|JK^8KbwRRTN5J^PZdjq@+&?P(vC zVNK<;O{snayZ!V_f>rAD1E1Q#gZ(fFDM!Y>sHZ^{T7aicCCX+8j2sOM%CHv18()-R zWzyFoVh5fV>8lw5oelxWh)xLwU?`x!)k*a8I4H`hsjAa%x~jo!$zA(^(KtQ9?GKTX z$I>n;bLx6((Dg67wOhywiaSuJ(Q9#ET)Vct;Ge|)_zH+A1-#z8sQPv$2ll0(nk2lk zG@Si0E&i_INu4AxNUg9zuIqb5JxetdUg~=TWekH*!uN`Jx#f>hbXsqJ*P^QUFnWIN z+HKI?zufB_d9U)a@(@$yrFE`MFO+XCu$cdztPBo-9>WQrW)pYR*Xjp$t_ac&P3JOE zd9GK%_t!_NTvrn8KVqq3#^Sqf@)cl6m@L(+2k}63++P%g{H(K=yi>>ndyf}Yj!3ui z`h;mpx37g-))Lrq-bxSegqPp3dpAT%tQBY1c%b2h!!l8L)9)+m-iGUZ2A}GaUu}m3 z&Vg_E;l#yx37Z#HH2S-`g_8X-Et;%&ToB{e9^bNK<&62N_8rUsFVAKm;zlrdZ5hJM z_&Bg?#_|$b@E^IuH|9&@Q=FU(2lG%=+q1{EtFSK})4V8KWn3Yk)^xBEH(U3OMIFw^ z!>!@e`RlFqdf9w_+Zbi#rMN=TBld5*YoG5~yjLmcA;%ofuzN*q7%FaKZ^NMsWpoML zmC}Jj(T}sUYl{Se8^AIQ;s;44{O`atm$mroCypd@43qKC^h*QslKl=P3M!QfmD_s&x(Z zWtP|_D>1vP?wWr~*K=un(@*Fn-7>c;xTYrQrj}PjNkr1o%B?Yaal zfFz1?pp?nL@T+5@CQ#5u>sB6D!cB!s&@do0tlNv2F#qx!@>M)%ev`|9CO@J7h^CzK z)WX7qo>Pe!j9N_lAsBb`DsjuHWZhPpG(uh3hI*IMk&lR^TnC!$fP?s@tTuIwfpz$@ zWJ6PH5x^Y3A-P|uv6w+$7||%pgl5jbKbzZ6LMz7SZ;kA%|6-qPi`B2ijTqs$=U1Gx zUkG{+?xG@3~7@VAs&wuZL-Ky`5c9{8s|O z?NkGKzlfb&Zu#_;duU|v#82ykOW2!&gKv)BX=Swbri2r6WFRSJ$<+v{HSNd7=hPb* ztnRKCdm@V`M80%&B%<=<-vF8F#At9GIi_&iqCIHWx;rPbf8s5gGWD8myZ73T!3rCU z2NDbcKDs^Wuj76N<3I~i>eZ?6>diOx>$CUr#=A3T+^o|WUo*RZ#)SzbZnsoYD4 zzFBvR30LM(3dPz?RDCYvSG4%WyMjmx9UxJDHiMYT^?YDwQI9pN-CHhcyvvXUzvwtU zoz6TMz47_1+IEE0f0^U|*gUr@Y>H4q-vZp}!MEf{ z-Sec~S#?!Av6Hp-tUuCu0NRMl6?yN;;-qkaW_9bZ#&M6G+e2S+gUwA=D|_o?!G!7s zGxE);KaQGHBUp;O%q83Qs`dIBXs~^k0?Nr?z_M13YGJhFN7ETj{p{@!zOWo{&=;p< z7kmMqWlem~GRH;}CcSt^d^(3gmv9!$U@^Xj(EK>8VyewKiI?rNnu7g-C#Rl)*LLp% zL17Y#k_B7)Od+EEQm6&$wa5AE`^z)O{@e4q;b;)+K2K0RFJ8W|Dd|}-YWX_Krrh~m zR_gVA$4}MzQyuycNoUj5wGSs3iZN7LV+Ca+6O^9WopvabR+hY&*V%j9UEQ?W%C}Cg5lLKh|1vfZfsUBP6ldk+f`av z%_8(wUIqq6#}xmZu4Xfs9K4u=wtuKM4|Kb(nI^Sk{pfXkFZ(*jw>LuFK-SQSiVzi9 z0>CJR5y*_lDdVii*hyJa=j#n>p$Wma>oY?`hGe?B(Y*cP)U*(dewq?pz1p$R)SWrOQv_CCrt)s+ZBXlm*5J1x+NfPlds=81~`avroK+{c75 zR;7i7PNj%G&tCe|lDq_YZuOiSqa6nM&0;u(p56WXjhzHlRA= zZ_-TF&`nKm7Eg^o1n3^7bN=NH_g0WkmdCb*m>JB@}=iAf7Z`6G#`1?VVW0ICk8)d8^!z|@7# zW?s)JpW!E4j*Xq0Q`vKu8*pm9Ne@NHwT=!RN5z1-{;SvoVr2tA)>L$k2?^<2ncgfJ z&HVROQMe~i7Ov&W%+GF$KdLSzd*`IP#lm72#yua~IcXGo;C1eQymYP8>y}wwn<`b^ zg#ipPz@a#790N<2A;IOW5IVKC za=3h(H7@z^Ru@XYzSG3qQ;qDRW{c4KRh1umMYTAYMt2O2WzFYzMyICuFdNxp6NBz7 zmZu*$sK*+mI`q{?W828d7W^X1G#kA{lWIG942(ah-C9~96~7T*5c?E)DbwQRYwHO; z1D~j3;r>*m>ws@#mQb_1V!y)7+%ib)99Kg|E!teR-JKAtTK*m|S-Zy$Wh@Pij|!*C z z)(a8Wf$BE2D|fet3h)`ugRST<(Ld!nuWVQ?RvpjkEFlRe5S>&BfhrUEA5OuRSiXc!zlUrVZN zBd#`|+bMB)XT{V0d9#dxVXk(cy@LVc%ysxEx;8+o#89pJ+j>B3iYi%urSVN0he(mC zIrDQyV1^bmswv`ub6bWA@$3HfIg(MMoAX2!!6Y}Gu^T8Wz2g-Cl3SmFk_K(TWXUZo&zf~o3 z)=|=6SuHVyoH%6zW)GZF_MAO-(@z9Pa|2w0vo-Yw)u*4Ff4_6ER1Z$-FB%jZu8N9& zd|$Q0>A4j{u#^^?!U$SVNvPG5DBT!N;qoiAoigVa>2@x$MQLz$Dw@e%k5O9wa_+wA z&fN^Wclk=$NHm(kevjWvjV~~Yr=j2*AV&i55&o4E%5;CrV-77f;U_jQ7madhwsr%n z@B;<-JTFK*lscJO!t|YZz6=30>l~q$di^9DzS5#cRJ0P>NQ!^tcctdv z@XiQM=f^x?er*9?$-wl^Du`;|sj8Wmv$b7Kg%1Z2eQK#FbM z8jV(T_6loKz`iqAl8-T)^|_8n7Qyo34OE<2|1Dsdx;ckC=F*4UB0WKW1|Um z!2Gf4N-%|bR1 zLA}QpKR%o0myoYf8ekojkTLZd)BDk;k%#{ff-5}BAo1D!)J*Ygj%!GWlx@&d6iPOd zlt5mOBQ>2Pyl$BW4jdftl@7(w4D7PPp>(z~?V!Z|)SK!n9scrsf}g*4m-oMZ?J@@H z_^wxnX0)Fr{eHeeR9o8H_xeN&`G64*6${hoOZ;J;-8zGzs zy9n)I8QT#?=3UV(gbH_WP7-S*9Xjg)iNczi+!?sW{ix|XCqak3`REbYCMH&b5(gLu zLn)Y=g5ji*)M5&6c#7kUGAE`NsMs43Zv~JK=L54vFW0Uk1wU0dCI@jhs*Nxf2O?c+ zolr4x9mB`r1Gz&E?ej*$$}P#|^*A!RZ_=$penEnWK9*Th)lREaRu$G|X*H%cEHTWP z|JKja!QqwTq{Q&ZvY{~LeIlSjV(n_MhPG{x>b#X-zST~dPw&KLf`D#{-R$Tf!o^DA zS1p9zmS`4pC7|J3m6i(T@!MKS>s7Z;IBaIYnS|l&RTpYM78EGMcJ?_Ptu$iE-3&NI ze%ZfgV0?TV7axq0w01>|OA(}Wv2>6OU3yzv(ijfG&h1DS6N`!QW9xgLAamt>zvEHd zy1tCZ>%wR93N~@C2ei%ooE4D?c?sAH7$(R7Dn<1o2q|!_lG=?(g)0b&jVqBKICz5a zLD`0c>GwBfj;m4y`6UAiV0{Ltn!}9L{)vvX@3HM*K05ZH> zZm40}dyZOTGGtRt^LJ}TE25fmU~y4Zes_VSXlWtLCSR28{4^XnguBh2Mf44ZByTw3 zv^30?4!4cX@-)F&xIVUm4j1Z&PK8b>|MF0xH_ndxGzg--!8(ChyHkL>#kIaRFXO*9 zj33Ph28qRa)}i_O1Q?C{DQjZ||Xf72V(Q z{k9i9ZN&B=EY9~-LYj~tF5xxbtu#%Mvj@j)S1Y0#_bN##i08DMdW%sT+)kS3?#JPs zV=p-0BXm-JJDbufnYX9$sJq2SKEAZ`*O6xbv3Vv4OalS=TbeVGG6RTWYGOLWv z;EVie-HwipXVR4f=xky#pH1lo(g3rnYMO8XuNOdzQB5}RNV0k#d^V#Oojbkw36;bw ztI&WPo-{4LOEa2azRs`GSW*&lu21eL0M4Q6drhd%>AD37FM@CHozrF4^vU7M_VMbI zcIT{7-%cW$$V{5j^cLKAIy*AFay{`pvJC2B$8 zewU)DB16_U-*0a!ZBUgNlFJ1H#6rwxP3tqNgrsR^akCX~XvA5osoB}6-L9Yn=7JqT z&mmMYN0^~lLO64Nm>yp^8a9e1|C4A1k(!Acl*q9ndrA#>UVs!bvmTg2E zdm|e|@KsvjOBE~;dTDfJ3fhf{VMCOd0b=hOA^>R6VWun$mEsxiz%FwFwcMam+*cue zJtQzz;VRU<|CyT5J9>Ijc8evH{nbVoE<1Je1GPaUhrTak`ZbAxqONPJr0yu;Q;u;w4SL>0hh=NIWztDch{yh`v z<1+Y5SYn6y55)-xSb9aa1x4N9QBbS~@6Jq+T&pSb3LcR34ZiBnP`)4Agv1XC^3Lrc zfT`&)I}o?-zn3PfkT&q?VJy46`TiYoWh9YzE!0D1En4s8x6RX6U(#ydv}n6%)a#TA zS%?DJDxMiun{@c2Rpz%b`8jpB$Mgy7`-0tN==DWZBjNopRg3l7EmZQOYm%82oY@() z^Rtt-ujFUz^LIRiJgT;I#lCE4a9O`Nt0L&ZXD!w6X`9bc zmu(SN4A3%kbiLv1~xqlIJmftDyo754n2^ z2VDCd|5%P4L=oe1B(E!oW;z|lnd{T-+`aqMh@A1$^+{s075P24BKQ_`o*AfgQyLSK z26A5|AF~GR_|;By%;1|>WTh?DXFS&`UMV_{OijT^f=!J5o9oE}>k^~8-j;Obxd$}| z()e%DUHg*7n_YuFBA@SIOa{yMfHc-)XH<5&t0K~4h3C%xoqyXLjIJnIK+fu;-|2Hc zz6@UwJZ?&SXWG9(Ay;JqvMh^pAM)$nM z&Nb}({1;7VhrJJPNX3*U6~Cz}ac!!g1&*5LCMu54=i`!lI(# zx;faHqbu5V9$ULQ^|%b7>L&Jbt6L( zp#)aNV7}WYJt{X`kgyzn%Jwq}Dz*oxf^gvLd+DGLUo`zwpkZW5X73&1)#^^xtsH?< zW`hOJKpGH{w4K@GX4T=+TfM6a-Nz7K1P+9SR(g*0&|`PB;9}_#As8ck$GeMItxqQn zG}M7LfJd9<2u#bl?U5yy$g@s*O<^xyt(Np+oQzL{&{I7DeCohnZUy>_`xj}S9IR#C zc%N0>q?@qRgF<`iK$BIc)?lIZ?LhV<^+iPbM2>B;`XbeJivb+K3qnb}nje>XNf)oP zp5=De!F~&#NVC_)VYWWBH9v6t*f0SpLsau4PQ&$}s=7S7%E+9Y@AG4WGE&M1kC3gB z)-QeYbnJ@XhUablajs?k3Tuu=C_hlM@9oS`>e-o%)6p+F)YZ-KycQBf2&fSP0mMh~ z37eU$`WlChPDDihOTMJ@u*a51#&@-a z9(B!SH-h?u&7Ec{@J>;xGc>_t^vr_N#}D)^rRovwKPy5$xNN!GRLdmb%#BH@xU}7v zaRDo29$xskHV{uj^c}@p4+KQjQH2Tbd*FT4zSpZJL_NytRNpkPLehhDv+d4eslXgyML zJ5YI%h+9em3XE?#LSg{Mp4=xVG;IpdA$3q`h#!^n7WlM9rd2F(H~Djz`!{LwBi~wF z22c%kD+kOVsMfQ&t5tWd-S0K%DpxQ%J{F|kI;|&gxCT4UnQ62kHPu;m<*X>>?iAQq zVETG03zjY0iH+9U5+0P}UrgioP?D<7(5k(e$AsRH@);;4>(1>^ zRi_RhQ|~*2=B>>(K-!wfV@1qRhbPW{5~Eo00T^k%T)p(VBM?%%?0pF{J@nlYq*-+Z z3hzrM9l14+m8;Jn-8GxBC0A}yAw2d=N;=@v8ocv_pTkl$EilG^uu!4HceElWFc^%{ zy-%wKu^2FE@@N{^cCQ)`j7Kj-cKP-7$i(Vd?Bg{By7!|OJE5g3lC$;tdnM4d->AV{ z$ZvVHq^|tQS(}lpbwon32P@IMF3A{4Nc2f=3Z(T#=sryUR=FDD_f-5=tN4v6Hl!{k zfczfLmHvX$xYYNziijs~!j7hFW+YBT=V2*J>ZrtC_gj9%?;TP>burBh;fC%j_&c0G7O2lw22YoAA5368%`ZoUmp|N+ zh9>;p(=rV81R$%?e0!dfrjKE1Sp*}<8o*H-Nyq&Oj-8b!8{Xzt@G zFyo9Oy+Wii4nIGdu{*+r)Zqw+?CW6s*d`*fn-Lyd4Ny``+!-(UEXsM;jzen(u!#`n zXDD+ApXJ9e&$q67s$Xh8+$?+!TF}L8wf10Rp+0L?OwxX=TEocFv)t%J$I6mQ32rQb z+=Hd|rTXmhZu*d0Fmqla1>F#2#GX0Q%aCR%K&Gv%SMw^&Am?1i=}ht^H(!P$BMRK8 z0SiZ_!o@lK9^yb@5uZ57r9CDhwg=q}&#$bmC~BAqN*@CR1(SkNKM|gq{Iy}%^bI20S-s;BAZGskWs4; zN_V9aD&E9Oo#(f|H=sI?P-wGrDV@nVdGcit{yTps()P25OIex%MZ_Be5~{Bw`=G|z zU{@dm#tO1E(Vy!tcit2=HWGxmW(wL7!E?~6!){x$c`s|xNn5%#6I%LiVq1F+q|`d1 z3<=T(LnS#`OX^5CEPf8rNZ5 zus=m-ihVq?%Ttv>DPKhXSgsCBC$QL6g+D510a2=aH8s(f(2J7a62^FSBfA2o_cHH| zzkU2O5PkpC@RasUg}AfB*egb@LdQMv zg1Snu3j#F~TQR`!rxm{m1}FI8Nc8Zth zfP~nRE@s*O_#9TlUD^*dDwaIq>Aq3gjooQ2xiz%K4Wg|uaq>DkBCjae)5_zbN3(>Q zfaE7cw!-5+bCcA@r~!mN^;4H35%@pIg)(@3R&d+yL&hpc^>Z194D7S0hbF^9D8K-5w2K zftzR6Gf$TD9%$It>C6FG{d-&9K61`RDoUUm?5C;L|4|rgo|aVhKN%~p=CFOM27d^v(fTd@)9RXuIam7b&!q1RgD_L@6H~n0zR?5dttb21uDR#=mgzH#*WsH>4AOdh+}gd6UtAob%cNIQ2~-3z1@kLwVe$wp!H_s zyar?Qni3qFUlStP>?^`8$~1f_n6-H%VmamtyGMn_>NE(ld7a|Y zA}>mR$@Kaq5#;+Wc0d+Ash2|H08W3a}LbS3=T?%6c)85Xkd+MSLAAXHk9z|br>gWvE-0dYxrwBjM#;Dg?rrYpA% zS2ESxxh10QD^?E+KHEccd4&`#G3VN^?5-spj_;Fh@msMc)w{>1=kCQ<%3it|4L;D` zMU#wNM?$@{50gS*70ph@k0D!hC$FoZhapd)TPhwsFBe|>hfFC>HG*CV(lmz6&Jg7r z)x>W3?N7R{_S-Q-28X2CoS9TFeeDwpsNBe}(tyA&QAfVW2G1(hwVec&I5X>*>+{5P%cLWCSlfU)FMfx&-l#;SpM z41JDJ_)HjAr=X(j!Rmnhs4(f>#=Q$Eb%WFBr>wzKdTsK+-yLCbfB)XrQua&8L>tYL z4?ug$k}T0RmMIAYSh=R=j}6;z0MMe=UECV}1y+nm)jaEVW*-#D4^XfapapMob6Ys| ztGbLP$s%>%%x{Ka8ml#7P*Ip3Ou5Jxn@k0m4N&Nb?oZ?r<o%&pg8 zR}75bGw9n0CHV+^?lTGEv!miZ+ib)_8MJg}(miA`VMo1-R6M#s-j5xO0D4w90GIcG zDEB^<h2PJdLG^%kRpy`9m``=DNZ$cV@=%Ik0|z)W}$XU zd~)R&oLUvRRcK~)BBersl3#I@N@+th!1FKqT%j#jjIU?G8S@`^2NWm)nP_50SD^o# z`{N$q2=zrkkm5Ibnw4U>Za}=)W6Wj>bfEgI0hrlQCXi*8$r;(QclZNmu4-*K&xA_+ zFd5)qf<&=eW+x&w9vw}QRpj!q-G-S&vPRo|Zh$tuExpi)ux24s+=}U(_Ek){; z(x?wEVztQ3LHgmwX-+x|bcmI&50*N&=Fw>gAEzEu;KUB*juyx2h*MIRaL3Igt1J1u zK}}KRt}mk++;aBG-K%49QW#2Ap*p=EFX@$`=tD2FD=jeEqfzofGno*8x^La+bK+#Y zMXy$DU%9w-rlQ{8NCXb%sV%qA?A$z#9H^hg{hO)$HwOGBnjEUPQ0;^P#G*9WxMAS5 z>$S>!pQ1U{t4!Axq8j^?%7~u~HaSoV#5H~Hq^1`eIlkx?Satx&$LT%+!70RVx7x5L`A%d_(lO{?q29lJ2;!jh zRWdnR>aqGRT#M~x+yYHI`a@=91i)Qyl(wJGVye_(N!T8$3C*n}f#!{7M@e|KN>2~( zW5lzaZkK;`i?5y$0YpVH-yOj?yjHSwuQmHBZOeq9!fwEM$J*AZSopt)=}$!PJB9-8 z6!86Y7dVsOOHTW%)c)ZbI7XI2o_siv`Cs4fxA#sJ@#L#PUaS0#w(yU^>-DBr4wQ8M zmt;iKe|XUE8V!8K76ow1$vX^Y{&DDk+!RH&5^aS3m;dFd2Syf(=T(BO$flT70}MV@ zOndXP25@n)a=qm;X8c_|1iLx{G`%A z_se6Msd)?qn%!aMh@X!U90<_*G=u}=1X+|XUG={4`Xn$7`-W(pFw|rHQS@WJ>ONs; zD!adWg%kebx2>+0H!HSP-_Mx`vH-^Zmo#`kle~3T3Rf-TXNx}gJ0^~v>YR!*WvZ&fZi>+N zFHGRFX6cO&+OokTiO7Mh2a`hp-C6J6sC1kaQZ}@MH~)cW_*l@nee0QBvN3GgskNUo;(L0muoV1D{fJz~dV>&VR9HPgt)* z7WSck)vEtMC9jVpz*07>jt#~CG8uxmY9L>c?mO+3e_N~gi_-mV#yvN{avxieupMwa zcL|rwu_Pb$*st-}ePn!^Ru8_0lp8L*P2ms{%8Y!o2|tvkpC+Z{1kE#a4<9mf#F$K^ zc5T+TwGmWk(DAFm$^7=tF>%oGOdHHt=1Kc;Hu-;e3RuWP4qGg7*2CIQ#Z<3_-wPl`=TBg-r&dDmgr^J zB22`{F@5ZsnB)p9Fn-}?s{J3L6UhT>tSf+EC&B$>^LAppv?nADD_iC0T#?B>C;Vd{ zej)3GN(L8RzY2O9$>CJh`?s!(N0;Z*`5U@%&P)|Bb#jJslJVP6C;-MD5sw@la zQ)Bta=mD#Nfb^q_?_VcFgbQS90L7~*1qKcmMylmYm7~%#*!uJ7xz&sXcn?b6h=$*K=I4Bc%=#$5Ln^xPHSb>yDzIrDJI7oC z0?WVXAdwAeJI90g*EaSHEjnTTbr5}lU={+s+Ryt(EOI(q|j^qhRh1pJ`jhg!^40#Pm#%42G>B4jUaUaNa_Eq8(smT1Ru5V9yu#VJ5%}M zF;W2Uj>n{dkoqQ0s_J`7DMzd)e#?q6I4AR=C8;?q59sn2jOUoRt%uPbHPNa!srlJ)C``Xq`+ zX6rJT3(pSqK^7Cn`#tA=TW0>7#s5{vyz8$qZzSNy|L-IJdglw-&tLcj?`Ku$AQ^{! zY=r01fVVl+)gk*IgM3HG8?yiY{5zZEzX<-f3F#{a24FNzK4|yXu8jXXUfy?vzyL5e zKU=@9;r{a{QUCg8NiR2y4QzcjK z0YT~mh`jHyyt&KX_e08U;rth!6AU8n&bR3mGW6NqqUg3XBjBbktUaGw@OUa~S*pVh zo>*ZUDR7o^K=#(&Wm}JeT3NRpoHA?`#qs+k8{m{?s8AU?Wjc4qKVe?07{as#y0pLT zPw}{v{5e7FWUl-hOX*dQ2;>m*Rjb}o&3sod$#qOrF+)Tais;j&-v0LcIP{Qrr`v1_ zX}KFdx85`cA$x>uFT71pd82)7!4%t539WT42>!B(byUuZ*-+6FIDzG&eqmLaxy*TI} zW&_>PeG%TtSJUZMl|>e5;I|*e(-MGta6@vSSMIJzXqKe*hO4JIAmFCc14h>Aa&&g4 z1UAd&fxS7rQf^1v;igyq$qcSeG+$TI_OnPJ_{AT|(*LnK*$CdgTWJ%mwSh;&87_Fj67UsP4C*wM(WgrlmgTSVAgyciyv4PE92B z(%w2I(5;E`o;hZqL*{%qVSis{HIYJ5K;rhd9<9H51HkzqR8FI0%qPxpp$_nOc#y<4 zk7#^e#)a@~vPPS-M%}oHk#^b6g9qI6M6Zp=G|hR5Tt{^`Hj!}aUU8S%6ibEWvA(2# z+`p-Y2tgbqSjh_14EC5}i|8$UTQ+&r#V%!$>joL2kHpLYD?&Y?-Ax!%&nEj7l>Xi@ ziX494g-fg>CsIN$V}p>;=hmRnDH_WpbGtMii#++Xf@Qipd=9t_WFth(Dj2tm2g3E?;^8TkkSjB)q^g~5z56%naGZJk0Jy{P z;q4MmZ_n?m|7S2?q3Sgw_)h(O`4|0*m%X1f)Kp4%W-^icoE}3J(zn76=W2KS$gKt! zCKyv@_9EZ#d*pNmM9;?BA~=SW<2ROm^MKTB!#y81qu*nd5gQ86;8P^WSs-`USpI94 z|2GxzWP&mt+!c6tTJMSqn4QX$m40W}s$Fdgld4@uitT4lw5h%I4{R(@TBrfNO5Gcx z6#c9hLAiFt!Aqq}yscrMO3p*EM`048gu#>B{n**Sr3DnKvyd@5K=1Xd9CEE{oI`0QjYLvvF3YPjZy zh<`^Zeeb~vCrz|#-s4+pU9{9m(S0pUoPkRabb&~#WF+p?8tUB zXnH#LHb?WHsK{$b;OE07_~x7oNk6Gy@m%;4y=|9<68h#+HH3xI)QOlq5me$Q*(k{K zLvFCL(~v-DI0mB&qNi6!@tiK|%_MdS8+A`iHoDw-%5C$OmG3jll1{V8^`BJ%L9% z1Hh*VSF6*lus&jRy*w7bHNK>W$_fb&@A{68&l)l*smkTW%7YQ3?S}TM(wF-vGReun2Mjsycl8tJ=1?tMCMgwiuOKr_HUX zdQAezlx5d;z-%!+0SNT|G*Ul!Tl-v8nHz$7Y!0Ci*BSW-&^r+~2nKw4{(1G|A08Db zK&-7W4H4PZ;en3XtS(TI2-A>-J%CM+M@;LLsZ{X7OgWq^wtMsB9yBv<73n@+de|99 zCZ@~z*R~xjDrAmYxHNWZssz8&$+ndQa=b=&&fI>3h!*gKT{Skfr)M?Ngk58>LykaG zzYAijvYJl?ZNWP>KWq_6Ek2cJ7gWMQ&sVCcm}&`29H(M>zRNQfkBF z73fyHzdvo~N@Q~;*=Mx~lPz2_+Au7vJw^yQ^c^?CZ}y%Vd>QMGE1Z^zpP5))UpG->^QE_*~k_OhDRs{RHR3pRzAAQ0k| zAKC}c=2Jfp!?$*}9v@j1>L>toC4C7HYKJr#zN`LFcBhfw^Z0Sfw`Hjf(mBdsX*&oTXm@ zdC!Vd#}6?ZL4fZB0GqGfrKA5D)?Ty7DahHE680jQKHr0uE%PSgmU=Of#*JeTm#riO z{&;2Csbp9N`~41@3Pn0b8^qn4>iI6Bd&g~66eXZ5G;$3p(C;$u1h#9XDmSs@M>kcY z$QdJ>t9G5HR;}3$zfJQ^5G8j_;E2NpOIdhu@Z@?8rJIaRdPVDd*jg_&q|=;T)6u&-@#$sjG0jT713xDodj* zfMBZJdTUv%Qi0&&VBX%WA~%-1!PJQR8%OewT#;9i2uSCOFRL3A|4f4jyab@Sx)vw5 zkb3lvjEdpKWv}Jzj3`NlY{r&}g$L z#aFDZzMDTMB4AKnX?@&=yvPmFYjZ`HiJBRj`hxe#cbobAU?VR5X!!0gF$$Otuc6m+ zjG+r~uI6855e&sSzYb-ypkv7yp=bw_%^y=aTW5r=e<2{6Fp@E~Grm=n<+ihPRb!Sr zo9Dvxu>j9oxaghUA^ORlD-`JYP)OO{fp06&a)fro!Ic+6Pf2|wy21THsm107j);iv zV8h?f)4cTO-zzMAB(LdfG(^9_9{>Wl;AHc1_gN86>1`rbt2Jw6LJT(`?uMF3`2IO& zn&YJ7&gyWkRYt4rsvYA!);ZL`Mn)c$QAcb;%eyh}>c*fh4j==>b zr8|Rjj6hUYnlonM&&G4szO0Ub|qF^)8NUX;Q7d)MBpO zPUn!`J}>~jl%By?apE|wv6DlrZo0(q%u9DsQB%$M+n$Ja26n9GrJ}3hS_FDD3(j=z zEYRqbNxQD*SPVXYOD-5GfG!Zs%XaJeKL!9?i~#$!>_m*aE8ybsf*p8%*LGuD3`G5# zI(vc*fnnTthI;?q{(paqKmHrM4-j|)KBwvZk3tL}??k}Sv1G4;|0y5+FCcZtU4|om z)(R@YKf|!2R7_gbY5wv(^Q-Wt(%EKy>*RQSU?e3}*K&Awzg~d#{wyMnQm?2R&Xg0- zRx=uFx}613kqUT1do1P1339XvH)n(?yoYD}Y~AU!UW&o}ib(%-6B4hKelq7ob?Dz8 z$Gc62;6nY5Job&77}WSTZfo_@i2Va5`>x&9M{q=hM0oU(3_&xA_e7=Y1B;&ZjS^A> z6eK*ZHd`I7OAA2H7GKKMeSsvy@Vebw#4`Sxd|V_Gq)M(uPV@P{%?E)S&>Eb^oepaB zmy&lf?@&p~JeI1jag4&Hjvdd|`jAf@DI{%7fKle-vu(5WGaBGKzCJP^8h^ zos|ul9RD@S6Cvqh2(VNLr5n`+50`3i%|F=+jwWdG+e)Bpf)6Jl_Y!J?8huIsxhI!< z0Bah*p(@}zJPvZ}Q5p5k|~;?0geW zyN%8!w1C6@EwM9r_moI=C>(|fUH z;j_I_arHyCki&=I?I zSSYI@W_HGleQ?PLoSe52gg?C|W9 zX>c}Axvo1|7P^H(73(VR;kKh0;dlVEw^$0?dKszvx{*OnKo=VLHKrGYqgW)rc03+q zu9SVz?C++s$_>o!jml~8pYF^IQedfBr#6_ofb|0>7P%OHd&B7j%N2~TKVh-5=ttx6 z4jLhJX};>NFnc@eR_UvqGPw`OgHgt_)uhYMr{XDh2GgM!s9%1(*px^Y>pMw2xcF#p zZ--HzwRuTpKfw5$j;G+0y6iceEO!HK6NHwn+LPv!RC7l^+H0s2u9^W>f7po;jid~| zC^NJY(f(z>9)sgn*t!t-}zCP8n&TGTF=94X%Fq3XwT;H z0S z>J4#gx@R54bys9UdauL8cFr|I(Dlwy95Sg`{6s)Yuo9g{7bUGaeiOZ6(J-FDMek9{ z^v3m(n&Pnf(qgqXZkFJoG>Y`PT{(w97n=2_8b8OPcBkJqANfys+Vce*ne90N+iMTx ze?B`%2rwd0{h4}#_(~O7Q#F+CmpY_E0SoWxvTV*|4Sv--nc`Bb(qwH#){oK6;iqGm zGmX`U@;_GME$R&0zE#oqiCq=CXc2oX`K_Ty!nOuYzHp5znIvKgqT1FIejo}bc`SDVjO zZb}XG;SgDto8NlhUPa(8!=R%^H1X!!iGUY4TvtP9TAgD+`^8yNn02+gE!6Nz*BgIr zI%u)kVbPt$zok3Iv*%Y8Q9p2Ns#{OzK$l%3as9Sb*BoDtR<{neih&x!32|QMM5iw8t*_&|Nw3e`s=U=xI6EFKpdH0- z=wR&?fID{HRpC>gui58|Mi)<_21WYqOSdw;Oe1HryqUSn*L&YGTl+LHcP;FThTMA}h}k?o_sP^Ka;5BbF9(kiT|dF% zkt)`9^>KUAVo&Stbu_S;gPgzHlO~ca*Y8K#9L>Pj(b1t$SMKgsGIH|z`MNbN+uJ)< zB|Z^Sa$0p89QU_g{a>St8;Jw1m0V~K$Ibs+)V&3`5?jBrKKfYQ-_d}*&=P-xAxyl{g#ffOdSqNm(qOm z0|Q$g(k?W2oaxm!y*<0wpZ$}Zc7}I|9JYs`78`7Y%{lv~uc-SPZ9b?dIb(#~@yL#p zpR)@eyXLDFajWnffduSn3w{jPzhzFQu=JNrU0z@<)g6j$ppeP+M)WD?cDh^mp|6k<06lFKXSt=IWB>7*s7$4_1oS)f0$(B+M)-JE-My0Cpw&TlKl z#hI^~(hoIho8iXzRE<57&Y2TII#iLAQoaAKiSUaU=3?O&nR*_#oW^f>bJ)$p8m$n1 zVzCpA@Mr1Bup?HLGXZYpBJm)dsn5K3R9B(*&Y#9P;44p+bY@SLV2lZXXtc^4+Q zp{SuSc3fi0(DT{hF*J5CE?7BRBWZ-2EIY4*j+U_!IJ82p=Ot>OVxQVFY zM@?%e{IPnQuT0xQbX~i-!pOj|?3U?Ir)}u2umoPMn=5h4!$%wDb5tsw^DVHppA>2Z zWY>qYo=rL0{n1=yt>%t6@3HoZ-O=NRlEUSybEk1#{CgP?r^(3`w-O{jw^%uE+}xW+ z3u|}_|M=yeCH~kK!|8mPKmVS5XuxEdQNQo|^*lRurAh%U5ty$h0s;shFC?T_A1^P; zRMeVWBs=6=LIfg^8M-u|wiWEE1^wZ4a&=nVTd(C1no;FS{lPGO{V5XQPk%-rIzHXk zL^0kUFEu%LR|f=CFeVG@*3G*51!t-Hzx#!RzTaW1-%kJXuo59j7F~yDz7l|v?4on? zAR>!b7hFDvN4RxPSNDBEUx1?U3h-6k>$jjDfa zr)6XvFBJ0(EcK4g_^x}7c6Iwt#F)KqK9)r?zrAR1xnB)}29FhH=(9^IG0Z40j{W$^ z{qw9m&Ttq$vuEqNy}78nGe5i(fg)#k0cXCE9Dv~7*X-BjB~c7HddAK^d*{ zJchXQlV?I)wFICv7t@r^bKCAljt#Q!`dWsZE%xodDrSg$Yj0#?k6b|Us&cmtV9J@+N9QYsQb2KW zD_^xKXzq$%)q3f?LAlK}+=RvOTzB%8wjA45wRbOlQF*a;)Z*`|Qj~6nZJhc&_~m zY`56NB1%M^Ld$x2ooNs)gk=XqF}NCe6;Y$gg64g}U9)4j(*%x!vUfw#^cz9cDP*Z05n*5_6tQy!GNT#RgUWkA>;z*oq^|w!LCrKsHO7lgoyfUElrtX(+44m#4`Ng7!(fpgCN=i^o`SJb1|7T zZao(0ttd&#!nygz&oF9kPa8S2nypq?=P`UP#{(_SezrA@!@pW}DAn7};lJQ6gfOC- z>@CDnsG%Yol}!bp>%1E_MeAxn4!E?tU0v+=+2?fI{*HQMhB(@xc!2BtUZ!{L&PPi` z(n08#(~1ZTd0=!upk7F&saS877Kuh4U*CghRz8L|7=<1_ux zZ4*7_R!CneZN%8kcg2L8$EV(O21{4B$d4*wTTiI+`J|op^B>x5pUfgN2)NxKPU|`0 zX-VXzb;_!__32-gDlimGKn53+2Q?|?cb{g=w>nj%G2Yd9=6pwR%8V|EPap zCrdD7KW?lza5{EeC~gZZWrn5`e9y{R1 zyfY@ifwD_X@yV`s(R@-CJ(rJqvB?zFb&oKuWD=yOcqFMVvNE|tV_DXPQFy`iQ4{PqbwPutN|;qr#DjyyuNT^lNSqRm%Q4`tg;4N98XZ3xXYOcp z92FX+taSyqCRHyfyjXF`M4>p&J#m&VZ9$q5E#Hn|&DyU#WVF&H(KaPaVv}16rf{_D z&Pm^u7MHx(*<_*z0P{Kjxb^~t_~|hL3LOA9hBBwR0)&@ZL0?{5@^bSh*`bsVs`%kz zYVf|4rrzvm zZ2((BnaO-$oIUUjiNp&z;70c{a=9QU^n`Pv|UPuQwbyzjp}5A_#Sg zwM>ISR{mmTN4Y{5d&=W{HJ#&zyqmgLAC5wcqMvYZ1HvIT>wbG!e4o>lMGu@B-EdTc zU^c+zOSpxFN<`R-;Y~#Fx?)0N6dYG^E z^wA)vQEmf0Tkm3J7NlhQ?AhtLIg+MkPDZ!U9fB?tY}WaUXlEp?^Wo+sEV+U;G`E=r z6@Ov=UM(Lh+aK`rS&&}Dn`?~jj__7r-k`Zt4${l3uto4$ijCBpJg{H6^-YiXIDb^2 z*Sd3d6?09AUf<_&vX%0ao3A$Uau%H{mXvBWlU;7P;BT{ly%IASxna;hq=~WnAr1Ko zNKl~yvpA^SO&E;`5jvqz=v0;id9!>)FL$6QE%c=6oYeRBHXc^HH1R2l9M*ICBbkNq zDzzD&-VFA)FJ^H2?o9NGVa9mV)XY{6sq6Ulxp?rnU9nyMET#PV_~iQ8p?Y6cWbyGb zt=|hFP#dH;^;b0dyXsKDsWpt5t#<`w$BvPT#YzwCn$1=2m1NBssBZ$hvXTiFx#6qj z5QG@+B*8=xxM$%XFvOO~Mf#Ofikf7G7#(b#b?tr=tE7RUz#Fyghh|TmkOaLk>2!ud zBI$s!gjd*n-prgQXGUa7cxg4!!>(oJ+-7%ke1XFmWzQnLQ!?H3+)#}iWDS9*^k!CB z<-jXKADtw3-mEd`Psi>SQ-IX>%d3H^H$t~X3qhd-(S$zdtbp2TDHyO^xuxp1Sue|| zDd14==_le!=jw5BMnsHSu|qAK?y<_-!(gq>fCgnWe=hub&EY&!R(_@Vn`T*##at%q{j8Lg6~O6>;m z7rgmwnjA^4jMJ*_a&B{LpL-oj$$krADNWr=B8JadT_R~L*^TM3>2|t>o8jo#f?9XP z)0j&zXXUHENClB{?#`k1ZZe=lj&mXq5CX4`LD%oxT}t6T%N1aOzbXr~yD_5y;3fFJ ztnS}U48VqK^uhX-v^YO;qrIg?uPwIjbe*Yi{9J11AN@i6ODM)mKrpO{GTv#TC5cC# zV*$=WEU8htHx;3BahK?n)5iXt{Cin$cI&6GD7D%`0v_L=QUt3 zV(YCps7P%KWPgBAW~($1AaR4auH0(pr_s8)xiWZ6u|N-h4*|vD&UR~4%1G#GLsmUP zPamDsF@y$1O3kU3C@Bn0Z-$I3gvfXw#V?b}(TGl;5%a^{vk$|p=Aq|?StW1p{v$I> z0JV-mhd8vJj#$UUdBTpNn5N@FtC88bp>#sRXi9Oj?<`2mgQrZNR^)TUn7Bf%=FVog zS&2pXX$Y9Zw+|jy9oW;837N+69z#SQqkT~(aVon9pFUibq&WO^pZ#{)kk^#V+WRHU z`0IEha&t2$L&kc21pe4do^Nl^F0-)z*?6ub%<@3o1nFDmRAd(s3{ma*4q8W14^z(F6P;ra4*D<8P0E!L3P19< z8(OZNGIMXwIw79q;g9E_izfGE3e^?#hoWJyZ!|u8g9Tm9Na9LhE#UXUpU5UaqfwT0 zD0~%SP<0!LBfk6Q2_3Ya-Ohxs9rC51?CdV;sku7m_C+FNts{(QJggd))a%7<+G)(Jb7rn6`K%>|)!A+RW zew3apG<+t_l_kIQDM7mRN?qWocLD8$u>V)&W+}m$ZO&11f<>u=hgE{1pLrKX6_o znPW|3f&A1cozK6UEMmfF+^ImF!=WQ5%lX0I;MC;-OQDdJk1FNkq>;|l=+)rMcd%}I zme>b}(!He;*g@EcIX>Dr1ryot+&(8o=v173Ygg9SOB5ql_Mu6QU_bg~VQ&ljp7!7B zDYNGE;uxj0Ie<_FlxcG0HHy73r{9E^a2%|NdKB~5ruM9{<1IE7?7QC;+fcKn`sBJ! z56pt@4)`(E+imC)2wsgyQZwk-YE|zo;%hvF{z^`4*>PMrEPOs)E+Igja!p%WkyH5r zQ*BrUao--y^#K24bRP5|p`o2o!Nhj&;sO4eV%WpiGaeQM4nQx+4Gn6gdBgDRx%;S@g6A zxa@>A{EFKMDCHUf;nS|Dw#(rCqm+1R4D0o-?s!nvNN)@7xdDr{;+q2AwlX_1^GV3=D5Yy@0)Se#N!nl7H_NezXlr__~d`1+SL?(Tfs zzIA_eyN5iFR!E9>l=2ws`aG2^$>1=rY^L1u<|@TOUeQ3cCd4R0Z?XIs+Kp4L*-i@~o3zQ1@i?eMA$0aFjzi3 zBx?QCynO&WUTH$y_4AtU?<^C$dM|%tG(eG%v^r{k9b1v}u};Wm+uUQ@R{!t>VU>a4 z?tDuwrNAWhg174DG)<+`=t2-Od2H6Na@>GS-L0D3pVP6k^XzJ-A&_49q^lc8euQqS zYaI<6_oZytAIXuo+ZJe4L!awyPk3z3SVD8l30U8EXjH5_ANZ4+6bWSVry7=y*Nr(~ zt!m#~Um4FglAoheVy*_KGJ5UkyFJjjn?0t?6AgJjUK|s1Kb@dA_oGzrC?>*^;bJ1P z@qAAkYe*Ha1j)(|U(P7&pZd{gh1(n?K;xwoFS>_)0c*u@ILn$z3dFeE(F)+;OJj-P z*iUt(gy>wcMFDo>h)N`^Hj*+=^E)X>>l*<#g>vDPtTtJog)^r9e1ZetZOSr@SL_IG z$(ShS#HVLf>*4J`IFOPutG#^}D~KKbjwagXwE*<^6EBuAIR+aGn!HyRhQ+>nyQQ{* zPc?STKZoe#Z1L;JC5Rbia?Qc}1x--mZh^VzI&(l|159^_fXC^Z1fcTzW z;upE9{(&i)qKQ`1C97)nL0D>RG5yDmvt;sU)C*cMNR@J{uZykjh(!q_ick;_uxa@x zm1c1q`vyC=Yg}XBRfJ{j_1m}Fq;z`Y*+qP|6jcv2B^X~LM=RVKx^Zxry_MX{m)~s2$t~K}T6X`qx8R3qzcUlpz zIr6m*q)J7zh)-`zq2;txIab?0sMP0WjsnUysg=qCexsRV{V=?>J(OUrGJY}OAqY9j z+}XrUyKz!4eyZtniO1$*FOz08belRaKm$@GuQdfE+X24bd3mJH+s_dBJDWWL5>aK8 z6|8x;K-^kokwB;AtIrJs19O8rxc;;CBP?|^LzY^}=gVlrY4FD%MHQ-C?AiNaOE(&X z80s=_b>B$)<)cO7b;X;CpgLl;njc8YwRu9eW5w>y6$4`LV7%TcZNws1y~;((KKba1 z*%Rq4r^CBWOi*q^82srs5=({7;Zt`jsX9YS3 zbe1UhpeR?@vSCNj4$;2Jj;g~Q=By<5kw42q(=?*>nV zbjRv8G}6*kN+ks}pbzdClyuJ9uxq7(l&VE+N9>20ss?73wJ%d(+_l2SjFxd+X|G!F zzXZg9yER;aP|fq^Xn6$fj2-X=BtW#=-CgSDkuj^8k`3L0zi5L3p?lxT;rO*(8N7dZ z+UIZR(cHR7z$KfQSNRcL{&Du=NjL~1k}6*&1aqRIqbHK7Q;7+ro23%DeneI~%q{%^ zd${;xFJI}bUM6-*t@Hv%oVusK8qAq^#`7gkeKEK1B#(PCqnqa=`5zz^c{xVT5EOgz z$^pV@FP`_o~`AmrBI;->GpK#7s}Tk)W>o_0=Eos#lF#mKePn!swe41Dlj zE+F%7d|mV#wC-%5RpQc`f^!$;wr2rGG__J2 z#P#J&tWG zQ5nMCs}awJZ@zEHeNZ(}nA{Kk>J3xn#Nkn#wo1C+mm8(d zXJNOPN5R48vV5c=+cwt72T1M>YaUKZ#)H)f?EIHjF4qTi(FFt};tG5ad631^jkL~J zx6b~u8T{R3mZGz7XVbEw{Y65-*VDBhV|#OB@t7U4M5XSkfeR)mu+BP3KLx>UzHWEA>zJI6$*l<8V#nx z(J^CW*~7SQ3PXuwyl06_ksk>`C%*<0a=?m(<8aQHmH8b}-C*_OsF6UvB^0W zdJEyU?a@779J#@GZ4}Vg-iOZ#p1Ryrrc(9ho;&&NHRS)FMiMB598G3`)tsYMuv}cK zh_l2;8{XcPv^Gxa#{i`gUie&=X6c2Kg*{wDjP1BysgTtRqZoEH7*`&NV zkN>@>-|9#))l=2BDnYLGs5xw@h-2RWW{1NB39^VOS=e5wT#GZ3iA$TSYBEB+gR{X} z$4_{DTug9Sejf`4iC<^OLD)1Rrj)b0pNln0glVUqVf~6C&+!QVF>XUnZf`VAn9qa$ zL{W^aYuM1f-w2)a8%&T51)5^F#98ejb4oMnQMC+j0L5`f9?YPI+c;N_Cr2rC#q^nz zY0hPaheE9qrvhvfrAS?yHukR6l4mg_I)Brk0jtwE1LYO&CVI zgGs`;kzT+tS~2KjC0{V)tbYPu#c0n;khw;tDOWN8r)T*=;O4Q0}O_JgOCs z8nj8ad;ST2e@^ArztZd}-uGgylJg;Y3Z^mvwSmj3_5N|Kt!;JIwnrtcDxh0qwb4$; zPjBl{{NTK`LJ+b$)dfa=A^+23eLg&gJLLYi1t}K<9iBTSP6yYQ@{#FILRb%Xonn*o z1JaVf_hc_YF%n=ll%9Hx+TW9WiJeFiDt+C<2USLS{Yg#gR@yDOx6eOOY^y3~S4e|D zK5leLw|bn0z+V&Sl;~SHhL!at`bAz3lUrHg-`1+#W_y(LNwehVNx2MZTi4$NNqW0YsDx#`I3LpImxfV14KS}>K z>dRrc&Qp6f;>Y@)^2fcfangv007VsjeAh{npbQE^bsC2+-z6q*;$z1(Xe`>1 zU<-5i9w`-$VmHPpUibZHVK!c`3Hw;JMbw*G2%UTkv_!5YI$vGI ze{>;oc#PqW+-i57@Xh)bVcjO^IzMcjVy8I{19ur_0+}4aQ%QXgSW@w55-O>$v&G7B zOv!PRtyR8UmP@t%%DZZi;6xjf%jix(GYB<>Y&2?ZzDY;Qdrt7ZXgmFvOHLDnaecPA z^M_l7nG=8c+%4EP+Iyqm8(oUQ4uT~|REbm`4f z@pC@9o_UoL<*^HMMjn=NWN4Nu5>=MM`|;ltKH-oJUIplIiLk2PC&U&ivZr>X^1pR; z=J!n{*zJ>q-dRLi?)ruc+! zB9qp2c5cQ@&97li@TPB(s(&FTpP+cf)&G2Xqc0STFD`bcWY@4Uu5S?3=+8O4qS zgDUAvTi13G5eI7h_ zeT$BEcA(Tb>VLG_Tpfu=%;-XB{!soTr)DL>1{oj(n!#T1U1OaDy25FGsXruy>XSS_lskNW!{=d?#I{}QYjVFEZ!(cTFfZ$94~(EV!Bixhh4+iTSJY;l=$_vl zzR~_Ir18Dy$vpTZ)q?=rDu{T%X~Uvk(yCI$vPc?X*knu8w}~q0x*LtP@HTJ4b7waI z(9s?CP2N|!3X zTwZ1o%X-)F+0F>w$8^mzv3iqc&x*UE!t?2n`o-1^zLi&Zb5Yo-jcIASikvcAobn|y zcIx#!tXT?Z#jQ}#AUb8|rx&vZO@5o+74szf=lW1FS0-(#usroWW@j)>MyW4V_cu9h z$yBTW;a}2jtlbYd1{INR=?EC{jw+mHIbj4S%sb~W0E z`l%Ns<|{YF$kAsySt&Q5R5g8f1mkt< zG)bi=+KHaIU{^PL>mMEE?98LW{_st~^aFVVG{sL*DG=-YUdvT|#Xg2zXtQbqSh{Ps zyCl>ZQwHPJZ|;nh;07dT^e-2;FQ#_tFR$~7`-;{lYc}4ozwwOnzAUQ(7mk6%PC>(PrRW?)3H!IVM_QtAK-Vh@pyugT{zRqSuFw9MmTvj{Hr9rT#)Lyhg+s zW}-sw#LQ7kS^nh`WQXlX1nhj&5GA4#|17TT9MDHa?j|lC0`bI-M%WQ%Ze# zF6_DnaeOtbu^XVW`30HdhYdpi!ZM-ktB_4H<&Y|IbXzD|4ODI}_{ey<)xkR|n-_4g#R@oZ+;hZ!wQ$jSQU*$@GQR|j2yKq{&|pueu_$TFHh8At{tUs8 zwC+47<=pSjL|q)|0OzlKw15?~_7>9!oCg(1&>JFKELE|J5C&clZ<12X+5{ z;7}mb;5rX&UugqwID`^X3 zqAs3}HdX(#k{g--RIpW=m>+gMM+C_5?@|-J3-W7mgfJQLyOju32jA*_L>r^p%U%c` zvU#nCpo(vwMXll@j1Pr)=bt+r4qIpb#|7?etDe2hcK-{NOTQ;j zqYs3LAmI0JLs$D}3zY^)G_;nJb?1?gn99IM>&tr67Q3-iWH-diSW^;CcBfZg|HbM6 zf0BMr1xYJC0`d9#+-jgmLNTy$Eq0W#*KIGm=z?sy%7kRF=R zCI75@z4e?HWEYx{5y{7Nrhb!IsJ7zNa5(Ul;r;YwOk}XsPk@D#q5B(ID=#mj>5+48 zb+u>@`f1Lggt7WmQs?&;^C=gXYBoZ|a&VnT5+OU74jWuyE)2k|umGY-*(1vPM^+ao zki8L4)WbW{C2W??^chaMt+$zDil-B6b^+{hyl$HiZ+E(p1Zx!<%I^Xa)Y~p5P2^ww z7-uo#_i{ifZ!pdzPKY=S$)RZ1H!gjwca6v`;b0yHT-nge0$!~{h*$=7-|p(8IZxlk zEewc_4In;^f(V<3=6 zC)VF00k8yvcg4xA^%lpUm33-7Vceb1BSgHU1j-)EA{p&>1}SUq%=k_?qw!UWbPVO@ zPs!#9{NA?{y^n5pgWI9@KT7cqx~r@Wj7aWd(l;xKUSKE&!7D%Up@!ksAKKt*JSJ3` zWa$Ep>K{|0`2~S~8w!L_kIV=B4@#HcYecn3ov^N^R>zO2(Rc-E z0RC4fuyNTpeK4EFHdm2Yka4zTT%)|WD`|RdOvrh0chx6l>ehO32`SE;(`+h(r(}O` z--DcUS;_OTKZ8`Nf!d7uY^~F#bd|!xwzH-OKqERapXjJckt@ipTEQ>uynGpda&NEG z>vr1(5gr|%^wHQP&wfBl=IUO{LxWEp#Lm7|PzPpFNlFlWRJlvp2IYf@yyGx*V7(Rrr^#wp%v(CyZtq(Y2=Ha06^~K zQ&dq4(R?4%m z$(WjjF`dlOhFj^dXKpOkcYPU2n84hhgd-hM7T#1qyY{#{;s4Zd_-Zz$pz5Li1!u+0 zD2QK0Nw|j&Y|6vT^!sxO`1A9&XzwJGKX3qbAp#*OgZEA~BBV8FTyl7K;WufUt?xTF zPSEE!{<&VBHJ)d^f){XcWEuQk3dR85ayr;^_;r}SV1@^{LA@h`$B* z$VTy}X3vLnMKvZ<(Aqj&{#uk{)mNm#bX34r=<$JM%+oW>jH^(#rkXZJrP!)mVM|H_ zrp@!S4-x5br^}oHPorCi4vXXCx2s4Wz*Iwef%1L|?Yhlzk_WIua&WhsFD-I)A%mj$ zYaJ1To7aYx>y=V|8iy+*g1Y?>Mc;pC%H*)AG1&7wyLZnkUd|r>Qfp`<;X*j`&*yzK z+&mg~ewGT=3FSlNL;{WS_D-}D^+gcON34OW6uG(e5ScNFyJRg+=mhD`3Qi zEqm=5a2l?ou?TlpYmzP~y>&YUz&DCgKr4s{zr?Bc0FZy4R+P#D8wfbLdr#{=h40kg z-gn`qHoku9c>R3qC9&velU0JEv3x5G9UO+GpkC99mNMU{pTC?zAs7#{7u@lN4U4}3 z?}**d^Y(hjKTBijUy8X>^YObv!Pi9z$I(-XqxVfA(3$0Ay3*zdwrxTqFr}~yTRxZt ztY1``K$&aUpOq`D{BT_VEWjlU(qRjF7dIXVo4fPZySl2aUO}8olE?c$U{>0k2-GPK z65?2u7;F*`_v84sC?vUVQ@S?Wm+N9FeD;}%Q!5n%(*e%6_aID0!{Gqb3+#un!}yC% zi!*%Dv=L!o<0 zh0||Lw0>dz=Q}o(bS-fcB|7q`n9c;*j*T|!Qiiuy7Vf$R`t8KkyOf0fw*eyZszK(G z%K7py>2*U=zXW25HxAQKN7&<3HOs%t%IB<*<~8ma!4q+Tc06B(*bPa>cQYx8sn8_` zt1F6j+HPgb)kPGFo^Ux|4&v3G=wZkU7u%Z1elEfzSaLWkz)!G#J!U_LOo;NRv@%^} zXgZv8qJt13_=ITl+=lVuwHDD1A6~ee=>n@(MI#USg&*{kYkAYJfINOYBS(Qa>`@o)r`6mKVS;Xe@e zO8IayC=^n`3PHoEMrBoH7JVb#lZ9#_-1C)c=vK#i>~OjtJxW;E;9GY$G`coX@B|^gd_(VJK$H-CVx0_ba4|ROV8U5_i*0viF`dcEc^#99poA; zDY5JJm9m}&bB_XNcq#bNPIw4}Y_#Fh-sL7!pQM|@Pxh!v^7li+l!_PHhk?-M%F0}; zBkj{P#;j110lR$LiHK5(4VJB;7ktcXjyZX%Pm1=X5cN6#D~?k`VxjA0YdY-xPpjkKz+%18S2{lIT)``{;3h2CB6w ze)|Vk zf5^F-4PHb1*zonHSsJ*@7odMSqr1H=1(JG6{mN7R+-8@GjkDOIT~{J^!|vhM19rPb zqgqD?fVu-MFvbZ1=)8Y|u;WPtF@@R*N^+cwH*Wxp-d)tQ^HJ2wNqaCxYe?}f-RIf! z?PiQ`O!^nuh9PQBp2R zz?Y8?U$Iu(xP0rbKaU(2IZF9T0Clu|xP8Lme7U@Q8w}T|8A%0ghPh0bYQ~# zi@qRTbl#0|ES>9gtIt?zA0fUPGKpB}TJL&`a7D=B9-T$_GnpGh=|ns-9-iz>SU5VJ zkj2i`7U$V&dpN4foY+@Fk_DniH@l~PkzJ;EMoGGuS(f?cx4uW#NG4^APWHexyfdm# zrCHbn6A#l6Y1zrvM+>X8I=)|gO2KM9M^}%v*Z4ez>a`w_xPPP)q}Z*R?&Kf5-h)IJ zR4bK-rvA}DvGmZ2*mW!aDOjd>JYw^fY;}FU)+~DFw)hj4wAy^rVK!H^4c%XQmT@40 zTA`JPtxPAgln|7v8jLj(6mUBZ`X&r$5*d-=EarM5r(*oxbM#Z*9|B&SW#VTQ`(Z^S zs!R7O3Eo2;aM{j)%lvC>F8OoX!QsJ`kHMM$!{;CMjN4jI@UCvwpE2<2qW*RopFq3H z=Pce20ucH|PiF0E(Xoqt!~ZWte^E&PLQPz|O$H@2?RJc*$-GR|Os60a`Y`95ClD-v zuZ%LPxF9NyHK2P$1bU6IO@pG3hFTB1wzBW+)S~NdpViCa8&jz`qHrS3>)kqiqOgRM z>hfcRyw2mvqbWy)u``yGCwk|E;~I=qsuqEwvsVUS>d8)LoJzXm!Imw8+mrdn z0?N0VHQ|Fv1&CS)A3GOOAVN`RO=WQv9GMjEqceZeZmEx_a$%AMn_H%S$KvhOthjBl z-E&tt;Y^lQm*{ZAhv;~Ym^pt-sxKdp?g9(5NS|$%2QU^Pk>gAV5^N);f6rop)*Ri) znMYg?9fTeS-?u0cKI&rHn){?xC-GmWINc6{QI*U<1ivM$&a8AdH?@+ldb{l1P#K&0 z2PKF1g_ggH4{XjE17V~*ufm2X^7FCH5#$W|Z^Byo$%s#XN||$A;NK>`T~sJ9v=W|7 zR?=`_UqKeQU>eMW@}VMx`B?{Y=-4*26Nlxm=IcNyppt*7sRv*ySS?BF1u_9QZgkWb zRc;Lx2G<+osOu-o&(S1R0$#2tx+%ye;IaJqVcp?jBXN1iy4#`4;V9^zN)_&VM8W-Zitq)kbiK2SMsGAyADQI6Y}aEWZbUPm2GyBJwtusJ>8p;Bel@IbO8 zVtvFI$H_??KfNgrIKdmJn&rBwWmg!!8e8$jGK?h!$G7MXe(K+mnSZ*g3jE6R(;k<_ zG)-f@b$`NEAibA?gLpMjCcPm_2ghhCfpI>~t6a2Qh}5^W%A{cwC=f=oj@q$U zt_rWKzyqfTq}0ZNk2dD4VhLZYw%6v9>QJS9k9Y(5+ z{1+e`I!D=%!;;o;VlIyTe`yHHu!dzqp4A`bVrw-P*R-)WB=yMvKuMmeg2jBJ)ek1j zrhixnLg6bP|J)r4AKA57{j4=N&xlMH2aquz=f#^NF+?%oWH`uJ170@T;e>r`;(}+7 zN-juLil#r;Lbmis%zDApdhRtOBB!5cMPdu5y~+i@T*e&}Xg|^n#@?hb#eA zXLT5$=MO+@J_Hax=!{xXW8)6AG|(L{``-xu5pwEWo?O$6^}9*W`Dn8fJI)cjt%ChI z*zacHcLpG=JrqQdx`$Fk>e~(qjW`i>#{adu}fh4Fez`ka{lQDvu>^UNBg5vuaY9P*I9NHv)PRMQ(EaRY zltRA$b1?sam4UncVobH=KsW{lH(Qwizg-Qps>yWNo}JzHh~$|wFD2blMU}xOy+SMA zVuvnauU+{y_kG0%b5RWER)gT6UdF+ZH&ZfWD?%|bsi4|E5h zcKj5TnRVhVwZ>*m$R?%E)U)tEMNAM+e!K=f%YK^62xVBRohGRR5mFU&kpi!dvZ7~n@rpu(mHHRwVmjG3@rTcH={$`E!O!AJM1(cW1$^8Uo7hzJz!yc|t#FTs~AWAUrCA1&MRPk$a0&CJ{HyqbuYJ9ZCY zPe*4#t&i{*m{LnBM3d*3GOVq2xP?%^rFOq?vs2(*OH0@|<6^wd9YXQN%?1KlXMwfZ}mC3eo zUSj_y8`Y2BH0FY=_o)SO@>+NlsGy3RP>3B8ZR%oDR(OUwfl>JIrT)o)d4MM2e)GIC49TB zX;j0IrE|s>EbezqS&h0+i8!lWi-k%J0s0FgNZoORN_@+A7v5=OihC2&u4J&`;>2Us zqD4%A+t~At%-Rz?;}jgY^qcOC$FNsDn&Z0E zPt%ckFg^#81gdACHiXttE#YH&>jUU8^QQv7_d(g`zQzE=76K|a)5sKHZjJx zsrcmTcG~M*v-`*5QWUk)b#7PvU(qy4p7?|Ow+*U!G*M5XY3tMq@ zh^cI0B=OkN%_Mcv186|+#s${ipbK78i3U~(M#optwrCCPLaB$(N0Sp1unDblr8Pf#=3Gfx5qhC- z0GeVkBmUaUUcA%(9GbGp&1|w@TH?LFNIbeb@e=^2L{ZHX1J1^*+}w7jr%(M8T4MCQ z!_bI;+nLSi1LSNCazm)gwYd-i*6OF6^Qu`0FSfxKmM3FdFWmEknNuoV;nn!_jLnr< zTrY`mS}%A8%UoL4itz^YJ%xiOBJdqlI{jZB+KGB9^hiiuc&b}n^WbIatF)z@a;rm?y|vtUWXER>_lM!~RuBb7|3hZC#rW?E#K3Uy z-fS)|&*JKQ3Q2;pFm_VO*3i$p^hA=MCLnbo0!{}Fy6b2y*wUi!`fp%jk$wB+Yo)6K|c_}MC}d2j@at@_=9vbl{w<*1*r+Q8mLI)1;3Q89GSH_S!=7tuOgi3 z4D{LSYG3ksC^>fA5wjdiWiVFyIXqTa#cAN!;VIfr)#OK#tfTxGX^|}V;iDf6qd&eg zA7&*)yn+n%*xyV0bdw9(!yk3Y$wH zi4NL@qL%~Nj`6xXuNy|COFZ!vU8WZ!HFqKx2Ey)e1?ut$7Kg~@e+&tQ^pw<#$Zh5I zMmvy)kXWXp*E!l%=0JIU?-&-{tEQFL$5sR9q__s@l6}Qb93>4I!pPN_bLpBK1)~wr z155A}o)sP!0A;$;u^K(wa9~ckcqx~Ej!;v^P;vusS&A)m`Ml;RcF7lq)TG55EOxMF&g&Pb2bP*BF9Y^*wF{jG=}v z*D9^P;3QC#*5pS7RvEzpY8wg_C{o9Wz=;BiBtu}^eP^b1%(BNt+Z*qvI9YR=o{^oq z=VxI(4JNeFI7EYE%9WW5xe;H`wBypu-(?jXfMe4lRW-dpdVV9hitTns-rTS+9&_B! zaFVe)$KoW=bLl-0(Z05zmdnsy1g%D2I`Kdz?^RB0n)w(E`H_hE8pBc0Yvia3vVtaD1?Sy!-=j$1-mS7y@qP z6v|o6#Y|59X1ln>vv=gw$I4dBb<+fQ-=@5Xcsz3Nz3pJFew;td z=k;KMh(aDtcb$tKEo`9#&>Q;979 z<;L-qR{!YY&tjroAg=921T~Py73MQjH`yVzFW$blk!bDc?*lq2&d^{C9T}>v*T+V()=QYH<*K<ELIlk-zbMLP|fD5`s1#&pz9I#awEY9{Xy)ENO!D-1DpGR)GvWo7e}A8bcm=R=iN* zm>m|&^4S{$cb?kf9hSi00~9ZNmZ!nIr0lbw&@EW3W(jRbDUm=K6F+Dw$l^>RI9cf` zx)`6WwT8}0nox?I|5CRYBZ~2TwZ6gRfCi%eF9ZF%Q?0_*M#Ir434@3x>*F`Y5&GME z2g0Yee2FT!Cza+w)ULHf;p?C42A*WkD?Jyr;@l@w1)mVoe$mlVb+tQD*P6}4EDma3 zodkh#RJzqjz&|_%S`SSt=Kb!Q#pBE*Cbg&e@WfaysDTK<{)0zCVxd7%n}saTq4`?f z_GI#h#x_~rwdHhEM!U-*m1CvLjZi558VX#pW9|y3x(!fXS*SA3ldzjWgA-0Ln^69W zaMF9>X(^X)fE(9P;L+d{>)^p!l?+8+K~ABW#dfyZBsz3BU%0~ZfntL z|Fsl9=k~1 ztU~wPnF|Z??>lnd&qphiBlx&Cf%9~;^ycuyU^8@ecc&PS3IfsQdYDS}dj-YmT&-jd zX!mO7P=y`dO*%qN%Bt#Wb5|m%?l1o?kHR@W2N7o`<%S7N4P5BDa+E(8mp3z zvgKY;riJpp)2F~%7_7|rcWBj6Wl$3|-ec56Nuaz$v_mO}3Dn{@27NjV=pfab5E%ps zoZ5*Fn0v9f*=SYQ`~cF0=u|YCa<;8v5Q?-zRxk(uxRQj-czR~Yeq&)fTWPA~@S!s% zpC=tOI=#ir%=p?zOYS>WecFDwbDyCch{8!MnQ7I}voJ>_rfWiK%@2(VJQZ3zU^$-W zfu|`6JyecdqgwX`(B}nprwb!^SmL(h`j3m8f~6% zG@?-UHx!fHO0we!yZ`Jh^YG`$+Owv;i0pQka>DcVSxwe^DSj|psD_z4Z3mYertb-w zTntX_16V2!N4uG@X15A_X@F~7Vs~c$u!mA8-#`g}y7*~;FsR3wQ@QAS zzG59S&)5njRn``l-PWN!9AtmtTAM8w+NqYIU&>&Y(KnBf!D}r`9pmw8S4?BKB5ILAryHG4ik9!Mq-sKYGBejJSfS%_HTRvL0 z8LiO=TlS%i8c{CG83wpolI)sLRjx(UN9?DWvd;t1OMbCkOE^W#m9(!!Fy48&ANJ)w zR~a5_Ej6A}4SSD4-veB#!tyQ8wp%h-t;@!qhaCwH*GpvHihWc(Xy~*wSC==+>t4{1IWti8ugZRJ;T`A!x~Idg?Y?b^9ATu57}X*LXn{^i>`@% zG(6tskR3`I{ckZUepN6F;V~t5cqh4R@EFlF>QvBh*)Y>ikMXwAeO);w-VNT-oSJwv zOE##lp6wD3qw&*}hb{b>x#9^>7H?b;?0ocp_#X@UqTtoe-S%0<4o9&if1=l8a+n7P zOQsPjvcD^pSJp?Kj_5hYyH_Vn#GGu+5GU8u8oC9dBAT_EI?Q3`*I~}{SV6JG_291_GY$@x%5l-qjy-<%1MF??tYE3&Im6K>$)g)reK*RCrh`>CE2IZrrGb%VSZOOiHc#f_lujzk3C&U5A*CkV%~baf3WGC96DI3 zMy@Z{3`IXZZB(Nl!H^P+KXI9?Y$|J)de8m^Hoc=N?fKo_jib2UQEM0`e<^u5{#u?D z)%_{^01359e)Z5}KJ>#UtA{y78@x5irv8^E+oIzB_-RY78{y@0u&`w1!bfaTj+1(7IJNwB>yV<25<_$ zWZL}s?Lfcr=}8B7=s24g=s1EE6HjjeK*BeuL2jj9o*!x!d4MtrCKdJf8i@NKv@0~q zqVC{Qos(doPs?$bj*Mb?w&qUOUbAyc)b6cpYB4MTX6=+E18RjC&me;rLIl-FddK^x zU(buBrh7iKRzR|ePb@BZjt3@EFQ+{l5R?=QWh9Uzde{VgiMcKUgmyd7t3JO%mgR1-Rd7e zRp`HJU3+{y5)Q+JGw2`k=S{e&L#+sKifbWk8Ayu0U4_W`NIAyI&7p~AM8Nl5zPtO( zwG~T3uKed-`LyRKjl>4iNTuY9Ew-zCXG&IYP6FiELB4HY_F<#zYpr}MZ}~u$+8q7c zPrjb(LXCndeML?nmJ07I^x>s|IuQg4Dxd0(*~4s)6Fvet0Yo03WJm0tlvpJI)!LaQ zOU^N&h3ZaRL+qqFNkdB&VH;e+QX4;O@%oHEOc+wTCRVB8fsB*@XFl?{W(AfSE9R2f$4|ZpZb#hc!4LGTmF} zZ)}a`kH-Lg!w=P@_h^niPzVBwQs3mxGu#HtxFGiTceDzq03}MfM$|q!jAQw*pc6wp zz_~`(-y9W3t>0&txMwEZ1Bpa?UmOF_7o7gBrU34ES{o5)YUVqk=baRAx7 z9aKc}>n`ap*E2%KIj9=~`BMEp^P3yyz5C@8B z6Af>hIa3E8c@sv<@n@-C=Sqp^DpCW`t_lnqM|fCYt_mWyj{0iUq!z~kK&up~%}%E- zCVMG9&~~oaV4`0MzS;_KT7_ivo& z1wl(l77o4dxKkXW8ym8-DwYK(fbwabVHOzr>&x-ct@$dpCfmK?*s}>FSAZoa@P%r8 zXE4?*VVax5NJ4`t)EX^9ru)E>|2r3;kDV;ASW$1x(OA$vkc5%Ux|@(#pVpThDa|ku))|r_1ScW)Y>XksTfnSDz zll140^j-h)+|v4nW5|X&ZptdVdi--W*Z&|z0V(|wU?IAYWYGwr5%~{H?=OW`qO;wJ z8~~HS$dp@AXc3w8FTg5h{!T&+>;kvj8%9V-3fVZ_g?i$vcRqe2`>X8Ue_DU4;2{+ z^#43^FA7LwU2QN5_}Bmbj0!S8WO%XfA7?gO1;OQTO0M*GNPW>v!K?ZMRvV7+lXX8v zc#rN<-9sSL_V1kJ)4Y2N!;dmDM|J=G6;cuOSe52i%CV7BnKu4UT=HMv)`2es2J?n; zIAq4u(K%D=Sb+l&$5S?_(kEf%}K0cSbpKfp0Z0l^dR^z3=w{}%ho>HweC*%36BI^&MYkzKpp$xUW2>F36S)Y0NJ|pl zpzaurKm5Cz0FYbdeUp@CgpNx7E4)AkPDTp9Pa znlbVDTUB*%feQG?;QzCL1|;v38;sCK|I63-D^D|UQP_j<=>hnxx5}zQ<%%E`n)jgd zm%uM13Z>FqXt2wJSM7ym$Kd}C@t@86h3tJ^QW^fE|5M#SIneSLQN!haWwX^H@+GHJ z>@33(y}#-s3_5GU-hCcUcmH&4tDxe<&`R>&R_nhj7>ERJ&=;sYO!V&(01M;Y*}Mz0 z!DaznJp0Xi{c96=BUQosGd_Xjlg);*Ja^Zf(fs+u#>F$3EWatY$O(y||J6C64RV0l zcOwf*-}Ap8`6D_vJ-9bn+zBo*=->;IV7%787#aQj+7F;FCj*_!(%btsu`~;_qhbwnJo9Z6K*m=Y_A#oH85du`y*fT}zC#KRO{eWxo-wy|b3}8KfWySmacNn1Ojr2y|ojvTSWysz6 zy8ig6v{(#X&qR83=-*~_4C0-}a|R1+Wq$e|Ts=UWx8OKB>*{n=XrBuro7Yi!8XH^; zomI(>cRI+$dD**a$|nHMf6kdHIWu4i|1+|XSZBK|IZvY#cIgKyrBWqGO-aYUa_B`8 zi>LYcQf4Q-zvtNL)v_{oD$qa5Nn{1eq!3Dead$`@yK*HKx=--opBWQuK?3b0w$w(% z{`+dccOqfP_F_-v^u&A#h!7rg4fEQPGBSIAy|)W^_5LUX2DJQ4jke)J4EtdB4R;iU zKJwLPNm@cg3DDh3AXS75CbX!L3>=y7Pas0_0!kZ@o>q$r8s+;zlk@t!Cb7b(2pJ)` zkkH0C^MD+kZgV6{RI?Dl47iDmU2nZ~$y9$m#*)(^%8_U8J0MetAR8*STn;WgW)Fb4 zr`sH?LY0TGf4N3A;`wKYzZ>BTBrwibNE56tkiQUosvkP~RXpw(+&@L?_z7?zQ+(~{ z*uBh$A3(~rAj~ALv|0qENoGZxJ!0w1j2id-y9W?=Az94kKCPwWC9=A};sh*o+4@?< zYW|e^Yc>9U41n%^UvZmXJI#oJz2N>0G|(6Cm6&%WBZIYt^m-HRacs}nv8qyAT}wEj<}D9dOQ~x6hjc!d+;U5P zJEWrNeD=RuNC}#D`l;*qfw@$(8L_Cebkd^JMsYS*>fdb#C`bIea-4i6NBN&}w13@L zio4n&1Js)3+olwwj^W$qC%4(%WJJg>8zTwy_X%wAZ?8}Ela6H@y3P5s)5B7yW2X~J zn!qV6rFW26WTbu?^pTmY`}gn9(#sC$4F9SI5Y4cGdj_#lNBRAC=L50_0xgU+9h=IU zz5s;tO&-H2wen%>cO>+bzxP$C$e?N=WRpdP|2D#38{239L>U z6E~V6>wyq2U1yM1s{zI33Ii&;9u;t}{`+2nBSgfIqrxloA*ugaCuT_4^L02=8JsR} zq;2^cD??@tF|_wb1ZIdLxwJoV#8f|C709A)6{58Aq>Ge&9Q^*jhY9t30jWz1L=pXO z5Od^2&MBm2_A*iIWgWBmMLKgjA(Ovmn&GE_pak-Wz5pJ7t8Dyi+Qsg=*O(cp^8b+o zqRV^ZvP(}kCg%UW5gdGuE+s;wrXbsO^%GzrIl*c3-WyoG_ev;KDFOM!Ez`+wDkG=$ z8ZK{-#ANaBlLC~}))^RF_RFWDb6{BpdJCzz<=!yVhrTT@&h^0UGe7#EPL;x$Xd@p?u5=9 zU}Zl{I$f9n7*>fa4$tRCg8$=-2*d(T>Ji05)llxg?*<{;I6EsBaDTY`D8sy}7sJ-( z{r8DpF;K&T@&AvjuMVr~iPqL5h)B195+c$dNC*fVq`Ny6kdQ9vFaT+gke2T5MiG#f zmPVRGcf&V_1HXIk=N~@$oV{ny%&K>-d1pptb*lC~OB9P985QyGl>c7x&Vs{nh11=y zqc_s^>qFA4w9FsYD?hNJRW36Od`lT{We3l0OG!zUy&J&FB>t+ud<)Iq|A-ntF`YH~!rr#!w!Up&0^y1H5+d82Wuo zHxYkd_^Q!cw>3)N2JOLyq|Ao%vR%<%eYk457}^&6$OxR1xYNnfC=L#Zq>}xm@n1)L zkQB${b+YawS_B=*ZUX0p3Nt0D7wHRb@9jmvPxOtALc99lV=B==Y)?gg@u~4UJK& zLR!~#onvLRG&AuP-6FTd&GYx3B;N9d{VH9Qy_s7SZ*UGk=WxE3(O8*je~wD-iwGzo zTnfd=is_Qny1KfGS@M*o0~tZ!yt9#FK@+~~+4!4w%iAo@jZyhNTxejDBbxps3mL8S zr?crd-8%nxHEYlbB;`;|0g^3tQxH((6xR`-$C#3~l>KDBKK30P=1) zFASXRDwVHU{tMJLJnpCCU-i#)s;)W;Gtb5nJp!GgK zJzP!nF#A>$dqCxEJ5{iycNhGIodl_O*$>B|68#OPDa~rSVOxXU=vtn)AU1^J#7psp zH|mBos{xmAZ$gews;i5O#tMj ztAPHfE-Wox4@sZyPlgOBx7}hOqh5oyuIIT!r_>r1T#J5{HvRmJ)-pY<=YvIF@Nwu7 zkGtp3#EygR_!I_8q5PwuB27ghwFzyk`2QWl2 zsLAp;uF--GEWFIuuXuS4A>C8f3Q8os`XU9;Tm9uf$u2n|x9{8)zq~k?$#fQhw;-=$Mp;l$V0(77jnqKZa{byZ2o4%{R`B1?r8j*Mhm^kU#^vJTGM%g`myY9P z3ft(gxRSUu#HbVO)`YzqDOhPSx03{}*rA_pDDQh^vrM2#G~Vt0Y`-1=Ft{eP4k(IA{QOwi65E`tQA)S4xJCdy= zCO4GkqWT=tT5&P{*JDs@5Q`M?vV;h*Zbj$tvO5S2TyLp{Y<5$4&2X}Pu8g1CuF=Ah;grOZcLmxq2BxsNAmr9FMOcKW=84SLu|o) z>Y87?2tb(7!b#FXsMriG#O$xrwI2Klpy83MHKhR^(cgq{m<|5%tc>Hbvcy!)l#J|{ ziZeRg-{e2RtF^j=frvR2dB80eJ$^_9Yy^+*eZa|UAuuHmCG@8o++F@Ug*_IrFQU>Y zF$^Ab8}kPf@VCCnm)wKaZn3K`fWE?VGL}1c{o+giErd+XmAs(k7W_nJkx0m9T=~>o z1yj-=m{KL~{Vjl&k40Kk+F~k}0vx+a=ER~~7Yfclk-RuNew4Prjt4g-O$eXM9=3ph zKnytPo{aK2_)6LkRP!Tz;(;Eh5`j>%F9t<^D5CG@N-kzfzYcdgT@YZ{k#$W)QK$Lt+juwgV$xRo?%Aqc& z2kW!{{!ub%mK9yz?Sov6Nux;L`{?wv^LH3kOG}Fg@DqOj6Bbz@?Q`k)_pi0+HWDM! zT`V$2AW}AyKNKtDm4z(?jyUj%Ueke|yBEqQf3SxQN^n|?8E$qbUtC+1J{2%ai+MqL z48$ac`+LP!g6FbbEY04(KTPXq9P@_v8k-S*mj3p^K@fk46KK085Fy4pjcI0XF7@Le@G4*7qHb_I4tM-$`Uu{>7va&-T!;~XHZBEbgsSrb zH=*8%*gJydYfRnsIJ~@YL7?wV;H5Exg^+NVd0oMm87M@^7c(=(x7B&$PQ4G6j z_?>e8os1k7D@6Fj)tmYJ;PP-*WOn)+Z>uCliYE=Ka+`nXuj_`fRiPM%Q^f|pUEf%R zV!5oy4s25%l224B)s$Jr(jLw}y?X8+a>EIXVS9f+64XEy;}a6n!F5yr@NE8@cfVJ_ z*709xUx#+@EabZwBxytgBrlzcK&+vG{X(@b7YiY?1yBo#KH&{q8%}Yulrn?aEzIg! z)}xrTo?cvBoJ`*#$g}KA`FuX3poCBZ?4lZgGjatW^l;&T8FRm%MK%a#d;ra)y1H5; z9|xF6@mPH|EBsfe9~8{xesnaweu3zCmcB!GEQb(o(N)xizb{PL^u}Ld#v1inVIC98 z+bJBbj>wsf6fh+wCgRY{UV#y>X4I?}`}(@NCq^9+Ku!MrMD~6D{m0FuaRDu}v7@^7 zl9&ud#=T^Z+-Nn+Xa&uD&kMbL;65n~O*vOpW{{l%z#d#RH8tn6Bb)gz)mM^(_XP<= z#6^LdFyL406%rCsW-+d+m?ob5anN~pF~~+2_c8p*7r{-goAoq$p6BHNgsGwAKFVAu z!@nNFEIsr)Pl4?`Li?J2AnDdQCA%sDq@;_IovT*3p$}mvfM_zL1-XDr9N zG_^wA;&w^R>shvj-X>JCZ}OG>dS$5CegR&Z58eUL#b&oCeZ19-HK&N=eC4bkO6!Bz zKD)F;WA;z0ylNJBgh$d|i&q^50+0q?xlP&M9x+ufikd*^*mp|>H*-V*RK{C`BqIhl z6TkeX_tWk)K;~>_gS0j_Hes>}JS`RX`fqT?oqNyreeb+9jP<(`dsF8I zKCbe(xbaSW5#6i4qY!KXU*J!`APIVFG}ju0nFB3F3<63)4+N#KVts`PZrw&9Qm?j6 zM}P1P3KzFjG_!*>M(kC>fUS0$hK7cZjpqHvgo{l}s#r*CIql93#=r&OJwce9u^XY@ zGe*dcn^XC&3PcR5(jYI=t$49>)$JEmP+;UD`*{t{yBV0Odi5?tKSWIUuN{f{eeA4? zL4ERTxs^2mXJ8-7BjS%gyRwqKg#Q^~P2SN2Sz46^WXX23zfctd_DcRkm`{O-*d?~$ z@dq9UG%21vlSj#B(XOHZVvaH!%ArJr2xuKDT#wGq&J2HtQuZecp=f}M0_q~yO^_c= zzJGPW@6^>7E8~h&lmo4%FBCt98RcoBSO!BC;lor;Vq8ZIlOh-bfLw{)l57l!+wAX# z8H3Y95i{xi8yR@`ypKa>b-l|035c*aTUV- zN8;wpi za3FPj+u9ZJ(L8GG2M@Ab6?65=VG=s!W-9MgKA$4Yn_{!!w7_k*o2vvMW)L?;ZnCI{ z(g5*%U7!1s$IAKbZaFuspD_4fe}bX`b&-Fa)&ivHfOTp$1&gkw`({rbWXFwo`v6UFcUpXRZlNn@psW%z~QR3%s1G8EP zdEalCm)BSuAi97p{_u9jh4@3`4TR(NlDP-Z!|q}}qy6_8W~W6j@+ba11!2Z3Zwf4?enUFB^wv0#drQ zsd}xqy@^bljh7AGLm~GO%7&fb6?@V=IjO^GHO06-UKzs7J#y`+$S72=;H-E z*L)66q(}}#KO0V0fj!d0&@kI70jwyNKcI@|QTL~{tbgV2vP4Jgz7ybnWn?xMpH*zs~|$XYe` z;G=6d@&WlCu^gn5*Zfk z*X@V_x3iulm*{vU>n@yQM`Ur*?~Gi2gA zDyKadP%t0<(Hnl6-P-p6F=7ALRfMu5pB!$o@6eV8j z@POAb@96lrz0P@8xLc3!ZP&LaV~Vlx8X|BO{qVSFpTvX!@Yc6(b8s!q{P?(#UpW;H zTrw{-!1VEThtS~8Z+3H&h2CTxr1Cm7_FrC58sBK}Q{-A2)%eHGy^I;{S_8k z5=s2^;81DFjfv_V2NUnS_kEGvM&8H-B1 z>ww71DDFZ)p>T-2!QIWxUe0XDF7Vo3kv<>+^ptVwUT3!yOy-P_jeQFPG2DB9y#e;- zbrM&`L8!CT?I9ua&8d2}_Z!;IoArl`EX6nVRv#$vtRgaZc6tRam-RVluf?K&3eV~; zi4}wp((i(R;#xGh$C03D-~&oInZ z#cv$v7rLg0LtnDH>k&`;r8gb#E;)qq2>GD%h$jG7p&kXby$(b25O|5C#89HXY8HG7 zdXsFE;(Zq@58}hlL5#;sVg^LIljgmJ&gg~AB#&?q_hXp1Z!lvqzPz1sZgv?+t5YxC zK{}9&&!YP>Dm`?91ae5m|8*Km&u>4!lc0lAM5p8z4>|Ew&+@2%Ch=@cARK|W2{Kr- zU~)a&h%9Wp(42BVoHQBuipYI$Bc%{J0i5noR0DkOqJB9^`LDO*5H-umuN8Qn{tnbM z?uue39?MmFdz%8p6bzL~xQD&4Ut_`OKwrE#kbwP=z$V~T5P)V(kWThjY2es4j7m(h z(sD9IqTA?tI{dPm#;qEe;E@}o7Hnz`(tYB_YLJVOO^ImJ#tyBI_4B=+ch+(?lz}uK zYI%_j1)rap@vs`BCDyFAT7cn09TS=c}H1Ah0`SsSA=wECP#sqQPEGGYHDB_TRXtK^rk{_ z;q#@A`qJ%8`)wXS?SSe=G)hQPN2TcZSd81$Mee7L@cL#1y^7>ZK0d)~-SI{ulwXBr z>F)!ElYLPQsJEiJiTPLhG?`Av&F)S=+RZP0D@#*I*lonG%1gsM73#J6zI(}kzWecK zTVqc0+=SDR#P(#*^MyY#xwf1C4CugyBxT0f=h`*$F(0s)b*ia66YA+{SaiSA^d?GO z#LMnzY7o)IuxM4y6J69YExB36^p3cfe}5wNmVc0!H;`;VE(sZSURTYwVhbq{cs6FE zVmak%F6(ys!{B)CkBGZ=xw3A11i$?Q@s_gp7wc5MK{e;^cN;9FMn7<;)&2X*sV;r4E9VW<23*w-)*g?!gsn9W9Rc z2ISy1{3++9giqBt>6$W7WM?hX0M;qbJ>?NfnH>BQS5;TKNC8^$H3OuuRg9f*UaZ9B z)zSF=cK*4Vg^I&ZjrNH(y3CqLbuuHhji))%M`;ylG6WtGX|Oalk9Ovm`U5#s^nn45 zq16PE7DL^})^KVAGbNi6w@$i*Hv8Gukn<-p-*CPa=oNg8dr!9VL?=!aKZMMxeV8hG z{;Mge^yuDdw$#YNWS_wD`MDjh_~?hx?h%!ly2&<Yicsw3apB7^GZc`f|PN4mA}9 z=yu;FaW}S4FBQV(z9hWsESQTnYHlc9TwRAnM_Qe5v5}X;ea$arQ1X#Z7YDT!R(KXO zsO;?xG+y)fb^*Whjxb8LySuv)@Gy$m;d2GDn@(~`gjWDWU5f&Q%{TW|Yinvcz`9eq z?Lx9dvtwlAvw=28X<$1NR^dgj7 zNyl(oiyGZpi|UKU>5@oG=9Z1NSr^NCVp)SC7|hXI7{aw|jbw;8v-$Zg>P!!-aoNpT zYy0q0UWLV%Tq=8+AFrn1_k0zBsnFAoCiIJbi~c%{BTw#sxIRu}cEnm=mA8vtb+v}# zWq^jyq^XROy9y#eKyl`lbkmliFX4!01?Q%P7TE^j{U28ao6@Cg57dGM^DlcdXs$oZ{rVq0_SbPS5kE(n7S0WFEm}@{1n|5umt?NJz0>hl28cgAJOnuXMhP{+d2#?Ecz zq-C!%x}b&C?;$yze-8>*mFSDCMsp|F9-YGBw>^Z?KruzC-2rY=K9G!LC$ks2h5}Jm z2ZP#{XeeBfWvyE;2RP48Eq11ZzFTc)7QE2MRLouFo63C>ltai#O~8R)`0tMqP7hHg zZg#TMK8ka|eJ3qT(9rXdhv-~-@sh?Q{v&P^W(xJmcy_h!n_ud@nyFxw-|Aet z95+R$)+RHWRNnIJN?A|NxpkSql;{-~7uOZftpr~c)pOhayifl#;TmK@(cJd> z#IDMb=VEc37V^akxA&_lb6H4%P3U zlrq#pW1+oIU}wn#w_zey8fExhz+s7T82Jn%%5W!V)jie z9Z=T{F>K=U{Ea;~e%#)9@NIM3zqsV!NY*MgWZjfjx7FTsPrk06#@*=9M$eA)gxG3r z(7IZ(#dg&$M}|D`oKo5P;$v(}x@q|v+oe7{jW>ILk5v zENtGt9|>TP1#8CUxWW01f6^sB(=N#b=eU^p|7ajrB?uwM{3OYFFsG?lY|cYWH;_Ar6ie$hz%s9Sn~!vAXE}pP#n?bF$G$ZVtIKB#sijGk-%jD%%p-# zzH*5%cB)J(=2Mju*{WHn7XkiSnTaD(BY-isVU3y`k!1S%8O(_?ZZ&c`|{G#QcyMdT|qBy zp)2`ab7kwtD9D(F)iVlloH#9VP%2{OcyhWoPu$&)Vcp-=l*R_LXUV$U3qJI}PT`XK zAV3w<5wfycFD55M-5mz@^BD{X4$mCt$+TU5HVs*Evot8Mgx60f8jyaTWd{_#p#yo{ zLk%c$#K31_Hlkswlw>FR?z)DcpyNM;aYUO%ZbIzaU&m5hbhRv5iT#aZpUdtus|sx? zWBGo=Y_XG!=Ij?iV_L;YI%NoKk(1U6W?V58UnR*9I?u_FB%a=u$>#kdMV_nskT>{P zlq8GHY)_KQ0x4Stci5HBby%IF!9vT%*0x^|7SG*$I8m&-UiJJsm;8nl#i)_;aR?+1 z!)Od_1bz~&4SIzWZnW+}+|kXJsX-ieFjZQLbhJ+wb^qd-Qor$k#dm0bETs9_!-}jT z-oNFes01Oq3yGHHZ7!82-sRYpI#_5bAzPrO>&afVUrPjung35Q!upidj>6%?;)(du zw`DKcGZ;`cKk*Gp%rz~UEhEoJiTsJvT#zv8`$B6@{1A?eia{CTkrpdh0fCl5Sf1mv zs14h*73xu$lGQbRUwJE&L%^zQ0=L0}3OUBAT_yc!vp`&W?(+u&O~dX9wJH*_m6|U^aQ*=KW2YqX_EX>Y1{S&Z$cS z0s^K&*zOQn=zd?ylUbByrT5GI&|s$IJuR1E8Ok&;{)uE=oi^mF^BCXIcDls=zQ;Rp@-eX1UF*QMR2G!Te2V5Z`y8c4SBi zlH;7jR5^}C_Jt(DbWE8KB16EHQ064_7AL{OQd;dz^#VK3&#azj87=r+WG{^ar`-4^ zUVUj?6R?Vr>A?4|vJ;6_PF^LU$3uZf4o2jZOBiz$a`;1#Fgj^?s|9kI5%>F$kq{kp z$F7j!V6-STZ%YfHTw-XP2`@+#B|Bx@M{aC&piS(sC{X zS&=pbJm_FP$Q;*iG3I`Zxzv;@7{bMegH=mEMxzc!IZvdFK|uXxh3s`H{RY@{uT z(U>>5|GnEYZ?M25G=6`d?@?REM+fq-zs>qUXf-!aLnZr8qQM) zDD*7058>7?+&Q=k$loT?q$s8v`Ztm2>?Kf}`>dMTf~w$s~Agp>6jV<5cf8K5Un-bHz$(@8%V> z`k+#Q+iL$5$0BE>{N#0M0~GKvz^N)6@ap{WT4`^d9;F%3^s6L_6*mhI0Wej$O#6jV zazS=aAH()p?*WVG=P+7bBtjBFy?wk%)%I^b2EJli32TAHw6jIWTHPA=7kVDg@z&9K zdwZ9x%s6j1;;<9#cMYMW+eH4{5j}R}vFPNXHu_N9+V%PJ1EH9*QT%>A-GcvoM8252 zq0v^EyBX_|c}(C`N6beTtz56ILAll6rv|F4w#v{NM^CxzLo-U}owaInezzZWgdvh! z**NT67Aij(tzXQZTG*b=a@e#dDtL^(mpw@EcS&5!i+SbZc4-heeAblJ98?=(C|Y>3 zW7enL`eC0VaNp8x-ik^*pR>}))#!Zcxni|28TY2AmLu6ND@wuh3@N7Pu7g}ELGZN^ z3PI!}7=xfQ9Z=>s4A1QLlS(F?nvKW8wa~Rl_qqwW$+2Au*g5cf%VKg^8z6JxsViAMG5h*-pk_9sgcH5K~Lp>)3vUo$CfUaojE9f zTYDfQV{n=0TunjAB}-;sR+saHPr=)wJiDYp_r1DH*WV`$74d^hgDmBk!D1nmU2SHW z9t*GfSI;{Z?Q)*?$hpdP7K(-#4Sd*!%~BX_@3pF+#SLUTIP-8aTk)$NH=G}{803T| z@Q6<~(H`cjp~V*ZPMXSR%Y$=T&YcA>*hOP=dg;tMW2p1bc8nMvedDB7Rzl4wcUQhO z&YmmInQDBPl58c_pzX$LU<j0#`H`I&2JxVg<7X zRK$f#C7KhbdG|f1N?|gWf0@D_FQpEz48m{fNqgPwQ)2z^=cjt&bi_K07xV+F{l~N$r*dzcGH=i= zu*RHxUaOIhoARMDN{h#U@|anM;de$m`RBMy&Gfh(Qwt5gf*4Fgt-T+8BKTZ{wAwG0 zEr<_oWI7-3U*zfT&ws6C@Kf>$UAeF|k|?}0o!ja`5fhl$6K26Q#Yk-di$rzcHmFhF znV0x}tUWAkI=_U-l2LpJPc{ zQjO2$19G$2M$Ltw7#Nn*4fPRpN}|e5pdN+@e`{J$nSTV>*c5;=6$#b_IzSa{tya)c z@$_+z#{9*}ZJUC1KTZzebK!EPhl9WFk#K!sm*C7~FR?%3J*gqtc4}RI8Q%P@q)>jN ztM!CO!B>@p7-01fn49g+tb_CGv!sq_Uab@H%}mj#o;3}*!+w63jo37)udgIErNLpH zJCowOIc2*j>`^QN*#;Avu`iZ>Q-~y#O8$jbEBPp&kZmLlab!HNrdA7Fk=t2v#9esA znZM9|&|_)k{2R_Gz_x-YTyg|ej>}`i|5x6$)cTB*W1_1J!%Ra5C3|R_(STNWfh?eV zm(z2lUJORsJem{HWkKdYauSfVS7!tAqYPW=LIsSLxbwLro)}ZwM7od_4tP`)#4iU< zCZ<0Z{M%t&08@`W;r_m6q2lsu7qdRX@<`@ToyHvQ^d&5FM$>^b+yTC`f@w64GI?s{ zW^A!CXQm0*zZz^04V@XV5(Qtq06+|vg@vU(=J{-7f4;85HJ1lq9^7xwjf3GUGfPRH z%so9ld3-nw^=V-*ta_wVJl@nANhV3=ME2n2n$zp+6q`&;iRB!Z$kj`G!!g-i>nYE z{P4l7v@+T0^R8i3yUwcLg7!!bO0`iT^1hF~?B)qu@<80;@s{*0dW(HU<`N^9ICd@U{wKxFJUeZYh$vo`sIuv+n$ zzfzPU`(E9ed>t3Ve6*{87+*x8L8=;1uZqzD`IkdJQ{>9wJ*^PB93p3!GF@6(u~!PL z9ounrI>mqQooo5GlT80>aIe52GaEqdWyqRmw z(o@J!GF>!tS?&yC#jT?O0*2UdfewQiW~ZFM`QQPAG{((Ri!9_b8i4kh@b!euQ^g-< z99I9O9=~KL+z#ujB89HW3=ehn0XK^c@JIlW_G=)yz|=p!8#YKb+F*`~6iO~wp<)7; zlZ6qn=swwhI>3)GRb0R>aQM)t_zaF5yr@tab!wCW<(V0SlrsVujqruyfax_s7?bHw zjtWBztAUUCbb}5cN|AhQMflV$TtJ=Q#&x=8V?a^iG}Ez4D;7XDCXkbplZTh@!RG*h zYH&~wzW%wm_!Pc;RM>FR@dL?(8UF2C5Ed5b_mc$-y4zRJdLx6S&s0}AqxNtStn=Qc9hOP@`xg4HGpJ%*iHhP4sK4UWS|fl7sAY&9Bw#1mHJsFCW*S$m z#v`5rkN&*Reo~b9@UMsX+u)0*Yt+Hl>f7{+BE5yb7BMXp*k8P+MTST_gaF5evNuT} z8aOYp@bH1_pW)oXg!j{-%BD)~y}FH(71|nUI-4P{m`iThg>P@WD96$LMVLLs8*eeJ zB)>RT!^6dLt?)*ZhX1v)i0vo*{WPm*$K@kz_Iw|!i*6SQYv*&&<$#AbydF5c z@S44mf_>!j6Ea$R>{Yfh`ii97jlnmW?_t1n{arp6tJa1AX;yEh+xB=qXyIDDr{3%qJe>GR^9+VXHFHPvH#b(FXq z^mm&IoGLW*A3+_|_rNlGgPbdT-BT|^%)wq=Tde6Wd@t0DP_iUgZ-HX8kH0><)8*q; zf4}ZEdOPXp0Y_Z+MJh^2i$#&|M4h7)|K-~>e0qhZw=hwW8zWgkwSV|Wv~lxj+jEur zU2tB1NvEqsVOZJq;`IZ#FQUY6$1W1_e$;jr2rUm`QVrsv)Q^U22)*rynOjQhU7Aem zASGBU!BDw(sKm938|!nLv?AoC?cWFR3@^rlog zZ7;$!Fd8e;a3Klx=g|Ga&P0S5n0OAP{i|8C$u$R;l|X%aT~lMTUV((8kj5eq`d=8L z2>k|;)nNPmC)>S+PD<2Dv?w>v{FnyIvOe053CW3CeWUaNd(5=XFPowSPv^ z``bBt$Ft8FhCYlVXf|G6sc86&iF$WQ`Eso$tEiOjc1O6rCwswzx_%{U>u10rVIDM@ zUIh!NUz6TxaOWOQ>3+(Wj}s@KwDoUD2-rm*py?xkDfoqN;Vc_%mfy*Tz^sYs+7rU* z1*@ZW##a-;aQ{emi_2%YCp54Xdz^B^{fW{dx@sOq2g|?8+OL1#MW>^5KiCw2@}9+5+~dJFW`-UlCE8wC(CKY!-$G{ zP040Lhi2Wy2iM^NyNK93S?N@A#h=<}gJ`-lvHR2_LsuC=YV!Vs4LCd4IT2SI+w{Le zIvaJUR--KL%jvzqzUb$u7&$I7KonT=}e3*^^8yV zP}mK-jU24T&k2RGU1^fJ=1gII+k7rgX}a~74{U$)Lu5uGJN|3yHL|1{Bc)Krexz6D z<(E?mEL1nL2b`#Fh$;MA50s$R^AkHF($RZ&E}aKq**}M?F!9Us5JUS8Y!%yr|61du z+j2RF^}8z@bK5LONE5R+KV211N0`e0-AnMdq^bEd;o`Do}in+zHy#(hH*wnr)XYSAZ%8N|*9T43@; z{OUPkzW>zYh4L2=71CiK_dMG$ZRYy#Dt1LqCqAh>Xl7B(Wa}j*! zUd4%F)|Fp@<{>j9O$uM7kG#+V#--$jUh_VoiOkn8PZmF9%(-txB@FD+Ag~qw0w`{iZ z5!agd+VB2-vU|c&*7jDtya$xoY)v*R)Q6Jalxv2w8Re)ie36-*^IVVi#oAb9B*+<# zP|j9td&&#ShPQ6l!uc=nLjvz#1^iD0%Z7VoJhlh^?D-1H!<&4h2W;2+4XU@{L=Of_ za?ak*HFTY?gwFTG3i5BtKX0$(<|jRx-xgXK-CkEGsJ1y2X%DAT%iH<&bURIfZ=tKP z%|2?Mw?`s)qNliB7QTO>X#=j0lT@icuf!AuabGl#(mP1{+P7}jVV6pS?rkRZ=->wP zg&|D{ z%AAPXW1qhdTowHpjrXm@Jjt!`pP#*%qOMQ1YV?;xSKR`rEdmw+2{KuGqgXJsraeqsK)JvPE8a^M2m_%-e=~3J|x2K(Bx7V=rit z*Xh#v{I6-x*1rZRZMfN&jsXucLo9w0NG-N^P}@0FY%38-rXFgxfjt>*g8C~~G=^v4 zI{j5vo&gHbtblCR%zdP>yMzLk#JPssw zBw}OSMtNGhGXEa-7b{ddSskPVd3IiU@IKwp#ha}+zaL2My=2-iPH?z6m-$Rm!XDV` z7g@4KE@}6r2K>{1l}Ahw{j)MNcgQ{IVtE77#@y(qI?5QcJlMjL80CsWyguGLvsq4Y zTini#U7KR3I2z0i#=Rfge&a{pha?PovCqRFn zh{sB#UTItG_I%E++NDK6IU~M0-YBI_F~^o;{` z_2|UBRQ)p*bj{tlml8`Mfpofo(8@7>lTisDIdm9%sb$OKn3n}1{L{s!*eg?)cJ-Sk zjTSSG2>+XO1+hBZlb`KmW9c*$ytiOcUBseyW$W zYdbWpMn;?#g?dq*O=9 zdgs`H^^I>wOifjl9Ex8D zJgk)}a2HoZUeKCM<4IgTFgtmm=J*RYoL*fJ=WMFb`umDed|`z)+j zpTdIXXUOsmm!dFEG_y+x7FI9M>FGW$CBB;~il3-*6ww=Z-zmtNA~F|*xQ3ic1(o#R z*xEblv558RggKv+Nbvb&i<+8sZmBH;@b@3x_*i z*FXGv7i`ql4}l1au5D*4fE>UtvWcLvvp;hc#kN;HPuRjNIfBUByXLT&%Ah#RbPh!_ z16?CcqLlOGWRq02J=(x!#eH9DU$V+zUgel2o{u{9t|NY-!zp!rv5ACX-1w2lB85Zp zV``(t!__zCP)WlDo}b4H<8mF}F4sdvK28xv#}jwUZ1B03o|Rn22b$_Bjd9VwVwf8dslJD+3Nab+fa58HteEl2NMTnsmrf9J=Jv~=AA9>5Znj;{d;8S)0^uEn zL##$V-1~BX^YC$&YCu#Y}K}}ItATn=#@1WJFn2&=Ny!(;eo+q6d z_5A%A23Dng8|#hV4n1}2g*S6mO1Rcuvb?)^LD#r`7%Arx^ha=mz4Z+ZZ^frHE@Skm zkL7$3(jhR>1B{7h=_$7GtM%LOc+WsbCd2Q!vCGo9gx_OEw`_VL;sJO>) zMnj!i<)^BkRR&T|UAjt3?ExkCZ(}(bhw@snWh+LDKOIs;H8m~itYE+J6bjJz<9L4M$eb46E1pR)}7Y&9|PBcUC6)ZIRHV0*c$d z`|;`wbHZ{nr_KWJCeu!BA63ufkD=Z@}@Bg)*sw z0;nrsVHv2+I?b+xr1amyt38g`vl#Php4EI<6rR&WVlft$al3 zD=6z-Ty@q@XIU^NmhwWeG(A%?Q5AsW;eE2|d5;9utfvcZu=-0NAzeSat>_LWLwDwQ z*LY_kE3v;w)Ps1mus)BKQ1a#OrVX89T_Cp|TRk(Z_N3}V_fT!>R-oaQZjbQ4tA?N*u%l^*;Jndn3YJa)*G1~3a zR$8r`C&Z0(F5iAMU#o#<7`{C!j9f5uaMVZTyhSk|@aa{{EB@4Fq#up859~B@^AG=g zap8GYE^wI3x-&ENs63a8pvqF2sqjK$Zy$faRQD<;fumlo5PyGwdbN??$`RAOQ)9#? zFfUL{1;+mU+6e@(h7h*4pL8pH(v8s zOi0AAMFu9k)MMi@>dPb|-c`}#QiT`?=;|L=c&13t7oT|9+lRPWZqYusyWTT{gl!G= zmWz#uRtl*3?Z;Ut;Xf~!I{er`{h$ri`Hb=8IE<=u(`0Fvfc7!9Wb{b4$JugnnVGU+ zY@Mz=KZmFVKSu^OyTwSs@#^GTiyHB4HQk|8;|sow0j{jY$1-LFgd7XLV}tuHGOsdd z25Fkl6bF3krb|-!UC&{jC4U|RfcmV?Vb$lt`Dj`2Tl_c#c!g zW6V*Lka3zmJLoh1kmGSbH=nP=-<*B+#U_u_X2<@O0l}00k0^(R2ZIW)zuG+XZt`cM z9Pkjf)=1K<-cMsEsd-f>O|C%#n73DnIr!_-y)@}CZP5yM8Ck~V#|eQ=4u<2+G`+(0 z$UEXd zQ2_=oC;d{HvDupSGecf?udNjSilx0DS52(s6BAJ9%4H&+$ir&Z)MmT7=-|$z(~2${ z@>w>V-hrgr3MoMbXe1eac97lc$~EqtWu<(cgK(9p-4;0d+ewWsGe3=q$d(_9KTCaD z*pa`gwzqxPK$k~MPu^yPRe;-LJ9d<&F%cseYBmea4R(b`(8Div_4NpDL?z%Nc6jl`YH@(-myz*@+630A8#~+AFoTVLgc+y zwIc`gK0Z?ck)_&-=cqfYc12)J>F*{5rS#1rmIX7f!ii@t!p5C3TDHWgTvR77fuHcU zHp;@&2&6cQYQFqjJKUxIN8ua+=)egvX0GnGgWw3JH;nRy+rEos07FJqz9S4@-0$qj z#`BC(8|9Sg4Qo}uReb4}-5PiBT_9wD20iLTMLSJ5{bhgc059&*Sv7NRThsO;8`YSC zpt%ysQ0<$sUxLYtckAqU!6cK!wmmJGP#0#c$?ru`nV}h)fr5k#-5o=BhXN7$K1N_w0J&5Q+KwfW4+L{OhO|_vB-d#5 z#FS)|i1{5qp%HH^IS7e)z`kKx@^CTCwm3ZV7`c_Kx1-cyBivoqvJWqj;0j@@YXfPA zuyBPxi?iODUBY~)WU@G$BG|DQ1$rfY9VS?P()N3v;+KHKJ6^eRMC0EF)GuXVu*T)k4V76pv;S1NYPn`XAoe-WFm( zAdWCjLeXY}1t5CfFIGhcOZG1ybu|mc19L+6TjdWPcCTt_@K|U&6Bm33bk;+GUeyYRnD8Hu>8dp*9S?6_|6 zHkfTx=NhlfY_yxuQ{ys5WN(7XZthmH?7W!u0F-C0{)#8w-^M>ZX(-j<@)X~8?4>3u zy%5mg=?G`XcO}lM4zQ9U{XKp;4`Q6|1W?xS^lXa2yK~kT7Phvf>&L%xya6kxtjT^O zXd#kpWdpur(HDd&(N!X^GpLl-S!hf4_eA+@bE30|8*6kC?f5(vJsdk*dc=-QeZSYP z|8@)6-e~mt)O%GW7793vLq_{iNTgzlDB}Yhy}5oQQDOD|+>cWN4&WdK(2A?pG`}+B z-f)rhFsLjKJ7qQcQU2I>?)~;|&3upd=uz7*XVppL>IgYng`5TUUBW z$UQN_7u`rir$5?9{$+k>i9wh=ulQ25ErRxry+Da_2_i9Yd+{+EBy~{P>o~4w^04K0 zX(ZsXvwwCq0`iP;jyCEk-RzxbeJrMaC>58s=MN(lCxeM&)T^9hjM^*&iirqZhBC#*@pPgb zHnJrh3T??5oiK#s^_YTHHqD*cXLS=b9PGp77o|p}@xS$1XA({sE#AmiR=~+@*gC6C zYw11|RD01wzDIHLHqhZ!7b?*>t0RsZA& zS^b}z3!jccR!*H6yh~&|+iR1LXaHgW^jez)l)~|Ggi;BTKNaZC)95>vR>)x_%B~jR z>MCXCK?B!hH8N}V9c=B4+sMqYdz^z6bpOhv0_!5&6)}y~Q$1L|WYZIo2I-Mo-j#JZ z!foLn&27o9JWlh|xbU@uoNWhYD$}ELB+YY~MR#d9XE5Q$EiP1)se2UvG_PPN7d)ze z&xQ>yIO12}ZkWk|0bcJUv8F<9spam6_6o=6H_o028*J4@)Wa&t>>oyFnJhY2 z%*Nt#33bI^o0TZ`*acG&v_HrqB|d)Y5R=|9n$jjO?73;A@BaE~56{FM7jA8nFG)l0 zxAKjIb6L zrapp3v6^<+^oe=AED;6oX|T2EbsmoJ9k$38Aem1>xs$N+#rTJc-8`g>+A((DNuD?_ zZb}^Vq++WTuVk_W*-RD}QMUIVWK`NsiUk}0NumJk8U(+d@Gdu+rFiniTtz^UV>qo? zxe}>A2qPhe?Ah9`keQaqbp=@;n~B6rxMnmSm|f|PW3tU#<@SkS8MQyE?cm3`uBH0Q zeX9>VsV@pNZi#2@$#7k$Ww1|J4}RE}E$?HF{4Tbm7duMdhyc=7UP6G)<6a`S*r>q1 zeiyg4$HsxxC;w*MhZ0{&bJEx+&+-1$`&rXnTrdL0kRbO_yXD=r;>)QOH+#_9pD(Dn z-sfB9KdBCGUDcA5Q#kmv?Ye6#klgNS&xzYrQl$0v5`Z>#LEVB9MIk#Hgq-Ypc%GDIQQwcbtZIrR#_oTRP!K4cOga4MZQsme|2larRb6r>hbr=Hu!GlS7p*JSS!GkO;6`Diwj30`vBCq0W6%?B%)~@l4=SOD zt)-(2CY;fSX5MCvWRDla5OG4vd8;v3*w)8~I_FpBShORFY?&Im@5P;?AM}+#WEkGGRq%Lo$w#}#WO`HPmx^?+JgUme zSXf>>tY=vO%<3Bc4~cqWtT5v^ReZVlr(x?<xE_ z#A#&qvIsjc0BAx+4jsAWAY&}>4}#;v~J-MJThT2J=Z^uDX?6v51P4^ zg6%S=jFQM}PT!;ie(Sneayt-~&SO}qnEV)tGN;%#{??=;jX9HA*gUl_S(~VoKNVP` zo(t@79f_CaI;Li@tziALu)j;Ys|5vM-JSB$XXZcx2-@Uq(^ccvuF70YFOU&sA4 zP7v0xHnZO#TI(;LTcWYT=UwA{#T7IZ);VuRgwQNn)Jx%qyrPl-YlLs;tW+#?F*RLQ zi^%Dlg!@Bc=W+GfDji+&s%$GbzcZk48Z#sxz7Ue0+S6|v69?x6!5USz4*h)&F3{+f z;X9pZNJ|qkr5}Xdu^#jx*N~E|Povkumy*g>CV88iHmD=4Ab&|VIPJk;6dM!p7Uipo zS8vbu^;*UP{hP@w=o*gtvHU*psk(#H;?225K*_Z53NuZk@AssS_Xf2g(_2l~tI>4r zF_ApF6e}~E7ro2+RxR6tUZbLP;Y)D~o3Z^QIPan*$(5<0!2%`EtH-lVT;;ffP~w~^ zw*jfOMIf)yf0{`ZI0HsaY?8;L2&H8LF|fm9rr42Wuy}-#Q8;3lMAyT2&R(1QIGosm zP3h~a-g2jXM(fcd&KftoUP7qNrSlJW?t(a zSH7uzi4L3312D-o2b2ERCYPZBe?JsaO-ip*O;FP;%Of|97D=vn0of)u|NGt0T$kE; zwUU+>VBy{qmy?U!sRX13zr||5`WyEXTDjS38(mg7R<jP&R!O(Ai`XHo_XtgLv+9gKSi`b}r75m4mGJ z^vflPPZO__%Mv%^q=2J_calV`S|6GvvAk&6ji1ikw zwk0EFOnrZM$L?9z4*rm;A0{z>530fgby;}~5v`YhFcRDlI6ql|4Q+i(J z(CAi`al6mXCI_)qF>wkQ;r{=4-np%h+wmL^Vqg@ z+E?uqs);Gr%a`JhdFVGZ<*qg%3sC!jkgIc}4RmRW!>K>r&(_J%(Klz9h%`wL`T!Q5 z>X;=X=2^HLMOdGu@M+?N>2&NLMW;6HTFX8rZq24 z3UCJ4;Iwjs8ji9RFp`_U*oqWGCf_wq*K^_rN8P4PHGf0{TSfc!Td7mW?p+A5e2|&4 zJz-dQkh{7_xN(R4kU^*``o9YD?bp3sGm9C-g~3vU2=kQ5e= z6V@ow1~1!I$FR-?CERz%>iZ~%pJo;%X6sGo3>JbO8Hjqq8b{W}COxRfHpU&_RQfcP zjPD!6)wB))O&2-bg=g5sR4P0`z02;gyGy|y(<9Pp22iZM5x?k&WEBoG-|L|$Tmy0x z_g`y5HB2-Uz^~q)g#gamhQQ*@tL+b-9dKEq=iz>8n3S0pc*6#~CZXGco@>ivmBGv7 zaaTeU=z&l!@>w*YDmhL%YnRm1+qJaKdXkT|1h0cN+X7o`nHNuE86#v%U{GmUa*y6) zZOfErEqPQ2Qejd(H<+~@D~{&g?uT9D&4^Vzqogm0>5T~?ME3}*mpOvnN!J= zC@W)`D<*a7wv)$35}h@|0?{Z4$bK@(s3{I2YVqN?O-g){fBhI2e5BQ=b9C3V5L4Q_ z-r}gyWM;`*mZ2Kwmen^}J)~6M(l~;XmTA4uQn>jw6t}J28x0|em zQyYZIc*Y-Uu8~|@Ng5Vj4|15K$g1Yo$=ujw7E{atk`akCE1rGeele%B3CCk{+8&B% zm$@cQsLHbz!y>sz1}pJy=oth}wE9-3P!n|tCey}29a%V;?6*0!U9iG2To&>o?B(=| z)n0oYu{X9FJ>}Zc_6hNQ8co^>TG1V@xI>yJ4&mtXxNh$yhfx}}R(K8{Pi772*nas9 zG+CzZlCHA-LVag1i8A04C!k>=B>`~|7r&ZyK&ac4a}cNAP0y0D%(OCtCuVA&0cuAV zNX}Cd19Q-P>ihBdV+_z$a$@I;0r$?%!Y+@4xddNb5IUPOLU9mA=OC=%qoEoxSm(|; z#9*{U?^ADQQRlQz8!zA&cvBmC8`S9Z)hy$qqt2<2n2Jh^v4=LVJ@)XZ2bS%cRYPI9 zWOHQP^b(=hb{L%h-D~>zCX`VT+C#{;bqqg_W_H3MGgA9FW^8rt8!b874i>)7FJNIl z;?5o|=vVYczRq;sB_JjuW3JG#`Djfd9`TG6=r8HjShRA0va>4AwtGvVRaLz~^vR^! z!7vxptKXF&1RdV$=>5v5TEACDK<&05zRu!*Z+nK)raWPd?Q9AgDX01-!^dPzA3C__ zRhPaVe*Ku*yXJG|*`mZ<4M$rHeQzXc6n3c6xGO52hs!+^}|9}A}G_I|Q}H%c9)btDm2-Y%{{ z>)aOUDPj~RN~n^QC;tpBiu|n4J+9`t9-O@k<-2V^Nn}q~kzlMHJDYbr;86jP)~@~9 zbOAgzWphHJ5Vc%fm@L9GV?1^X6R2&1&mga!7s6=B7O^oVPmgHb?o zcbh_w)+mE~?n-2oTmz$L#)V(ma=If+8)9yy#FVi3J?IxljqFV!#Y$K{daLij5tWV* zo~B@T+{7(VlxKZal&VBgIStKRO-kB}{n#^->D}i$Q&rX11kwBstlR=c53cOvSdAbi z>=<2;a59(76&O74m2(@pVdq#*fe38a?&StbSf@;Gnt8C?j5WIdNSQbOeKU75)&M0> z7OlMDZ!?zW{%*?RDui|+&EMNmL4C|;d-pOnU9jNeJp@bGfqLvf64qvmi6n1?eTX?M z>Ifzi;g0vjuLTH=qg_Y~5+(53-ODCG{ozoqrtI}v^eWCe=~87fwI>c-I=)u2crUo< zX+Jegi&qN(NX%|b(KdW4rLOh4l=5D(OgfC)Tz#HCBY6)aaAhKJ7#vFgYu@GK-J}(e*;(RVRMI9i$yMI`OaKBa zOe(K)f+YML>pFkIWZ;kjl;$VHOx5?RFi)ZjkOCp;>0CFvvE&Cwc0<*`PsKkGA}r|D{_4~Ox38&Cp z1bK<6m3lRJhQEAmhA^EF``LmxZpbhYWrd^?D~qnk<-isN9FC#O0XlQzxftCH&bLnF z5{;5%?t0gf`AefyyqNo&pD#AdAHC$9SP~j(P|H7Ysb5a1nUfb<6daoDPZBO}nK#!- z+x?aHw-kLaM@73B25g+5`KBu-oHSO7=xa%UM zG^jGgT#MY6#syx!fG=91q)2Qu0IUEkJ>YXJ@}amgZ<($Y?O$e{t?G?ZX1bONqgrIU z)Qc91qr8xm_uR#iyx~mLT`AOw$UMXq?MubRqRP^3#ft0TzTPfXvG3^!eLp7@7prDY zhRm*%R18XrieJvCn~X0N{yh(+zvj6*YaascP`#3L#O|p;fqXG0{6(vub=NcmQAOSK z26Q6^>Q75FEi#^v@d8=1t5Oxt%o&CEmfC+w2(lJ2?#aS4I1+`S=-8le?_?B{h)6N! zdD7>Yx!KYD@$NU-Py!Ryw$Ade9Y5%wj80l(~svW*K3>}|Tr*||p4VOeR<)j?! zyBh%27Mo;%(xRt?=>|bS5OL`#pV+aKuwyxwaKGDLB;7=*=xRjcu9Z4GoQ>9HuOAdG zqDt7^;L))|j;+Y$^cPc4oN*z*?qf>%R-GiV>jSasa!q50R!+Y8!JtjelGWHJlkgOi zxs?-6u`7YEIfcgeif`G*z(d?a6pi?o1hkjvdD#Y z37J=I;#TbL6pv%?unu+oI2wB5d05H~o?m)@fQn#p&HaEm2o6TB#1dCwPU3WY zx?2=~&@?pu^+l2tbVYAZyIm^$#cW={W%wS+)}mM zOm8?`u5W@ea|QS-tjp0CA%hB=^fy64ird8fW1BA42H_saFk!C}?T24=!sKz{u_BJ2 z<6x!dVr=h!5rdI*y0#@6MCxw&zhKvjcjGN!X)&%yqnRWyH0B z&fAE;D4Ly|JFoaoScALz+f};J_KBRi8LaIw_omgEkoCI-3q;KBDM6L_`oJtl+M33; zY2h1~t*aiF>_P{;&_!#t+z4~*b=J2ZIo#Q)V)Hb|m=9^2l@)(<91K1(D)V}mPd-ZM zIb%gZnl@6AXDaBUR5-f61}<#dA3W;B!QWKOjsJ=HL~LDhQ%9Gz`M^K5Vfz|+3=f5EAhL)7BX;Pbw__cIZZlAvAw#(n& z^L=P9ydUMw%d9l&ftjzfr>lDJTV`k_`kTr1yiW;} z0%wd8xY}M7f}{O3EHwAZrKdYKgpdMgLT_2t&AZ}{zS6WiCr_%YDjhDehc3vHMj<}& zww%;EXV-UTGcQkWRLxYk6%0EtW_C!&|MCkvo#GQI9r&U@E5j8JD7m4B7Kzp(w8`J@ z6Oa^2P~32r#SSx#pbGeLdrddjrq$R+Uz#tjd)FzO@~s^ETJTA684Ggi!X(%`!vOzn z$p6J+&I)cLD2Ke}bb7%oBxOimVs)-FkY*WUZx5fqHnt^Uc)6WIf#OL8|PMmPi`2)Ij!@b__q^T2)fp zX<7>sK9DSO5Cj|FtT3(4p}i(tI|->UIEAcG<%4)$t0Zqp4L3iDacubRKu9(J}*FU?1iH4+_08%hyLzAj!A;BK2{GU`{0 z9hL{^hH|P-uF_4CU-b&xc6cmZ#Oc)RI?t`d?~v|&Gk~}Ok^^ICISXPC{FyhF`cI5| z^zzJ@DLna|pQ|W=AxYadE-qrkZpY*=RE$EF)bu`EOWYH5DHp}5kOs;45Zq;0H(h-( zX!!B&n65a@evD%BBQ&&*0Q20C)wnRK*LXkinrMIWY5q;`256x%zY&i`A00`vvDqNj zAbfq;@726sfNL`ICU#+W_zn8e!To9)k|EDg@wP0RqOUwM7$=%z1UuWq)1vrx$l?!E zChr}c>Na4!Ot(^OnOJlZ!qw#tVw+78;2GZ9+19Iw@~uuW;=IM!0h07p(0sjlPmBy` z<+ON3vO=#fx5BKCr^2L0$Yctx%1sM?YW}e%fHuZD+mr+-LHU@X|QePvrf^2jeO=hKeS) zIySTgdvbv=s%ek=hH*Ml6X1n!ygP9yDBD#HV*12y{G?^o;sUyjW>=i(7 zDCz|)zBp;K?Cjg`+T+Y5+#YIhcxTcsiWjZd9vAskuf~qGgi}ZD=&lNjsVsFgxU2*! zP7|krz^s44(QDb#UMB-?8OiiLCLe?^0ZG2mSSlhgm$_DX+3%{v+R)yxc&tB#BnD4MiD5wS>|he z<8a;19Vc}5(wBSt+iu=IvtB}eXvb~!R4M!c*9nNg(MBu-VNwx1@pZOq_yO|$$nKBS z?b8Q}$#@!PG=j@*Mr>Pt8oqwoaX#h!>?t6OP56eTf1v5UahWL}JX?o_8W$A2N!A~w z@6)_=`{gL<2(I)ASAp%o1?Te$c*(Scv&#T5C-rO^Z9z;np@N11!e6hsyQfN6^2W<$ z=7+djNO8}J^#{ZPP&7Lq8fS9Ce78X4<5hD4teIftBw_C-BgMw%c*%}?G#OyZJC9UX zTF``)2UUBUGfKjZPYovW$j2(JhXAWjK9M3(Gx4sl!(*mA+D2*povHZ3}ujNDG0{*Vbc z_nR5gT*;QWb$*x;d?+g>xI~WOrkp!n7Ab7;e}dXTgRf%{peOjob8I813qN2>J&@_4 zI*ozcxk4D{^@8vM*(<5Z_>-9J`l0?zuw1{+v( zV>JDoo3)}`GErRyeCjoQpxW(=-~N;ko*MpFbpv_^)FyKau5qGKXSoHknq+6U|vE2T0R)7v|5IuYD6x<)c=yFLD`en(-+u(wADdO6&$A8;f0T zi3HHt+u@sYXZ#vPEZS802&}_q^;hLo9YPr@G3S(P#zrMo8x@@Mz1=MftBSM<-+jOh zkPC&=rSZNz?sdL9-+wVE+G$xo^m{F;ogF5mLP$N;;FEYyI4_70P4fU>S4~uZa|rC9 z-vZY!wtuH`8CS*rRfb@#X00IS9UTy~{g#0}zyI%BUJ&Eq`AM{N&o2~AZ>-Op1aeEf7lGLoV=Ht-0qU|{#$8peIh}_fQ~jq3?>Uk0+Z2tT8Z=s#6vb={I}hI!7?wFrijGBj$s6XU)um7y zAFy?0`?`e_o9sD>;mWul-J*8Gh+nyRrlLB#6YmGDUhqH@;vc^-JsGAYGK9r`6*Bdw z_g|kvM>wjF4&^^3>r2T>F;mMQT9(DioIB}14JV);4kW*OrPn9f9~n5lY*Qt51b!bFN3f5mvw*793i)$ zk|ONbl%){OGeNUyN-gS<@Z?QUgKy@_MV>n2Q4;pcmVI1povQlO@929MP_;yC3v44% zU{#LYW3L}#&V7I^_%eWFNPx8sfr4E^;&0GWz;CYe?NE$lj=sq?kM^kh?4|Q4JK^DO z=!}d-Z$t|7gCd^}qY$-xt#UWn=!V9(?j)$c*p+s+H0|wZ4kOkP;hFlO=&8XO@aCgw z{m}aUNspD2N$?Yn^YZS}^S*Q?pMm}zjDi8(B=UJtWTv&gDBm+=3Q-g_l^0u-xKpQT*Wf741r^uV}rk3?t_DxRNG`7q}iP(4>AmcL8RrKhyl zX~?GU_|jzlkFKLXhX>H@LL{HpR4QfuWwL%t9s)PpZ$GPj01)D2AHe=BjQHmpl-!zj{G`} zzTqEY0RJ3?lO{0A5DlHb5d!}>Y``d-!hu6`P&>%>mz==gKJHZrFiMj*8NWBV`P=V) zqj>lLWxNfY(Rx1_uzw!r-(ML6#L9L0JIH?y?vM4DI&V}UR$Td;w9U^Y_{)UPW=Lw0 zul|>*B!1tDuun9fXeYWh?@*`eNc8P&8n}*IU4et-vr#eE^Upy!g|yNd`4i#Ne|!2? zEiiZqE^EZEeAroPybq0Y{IxfJ&QNQ-B$8N&(GF>Z@F@FzA#2VTg`GQ;0YHML;ps-$ z;|&QSE6G`+8z*J!`uT>UoeU1qi~sRmTS?A`^h0|tf#^MPo)&$q#md)fDz~=;17PSD z6s)5hJmv2Cvs2w4+wd<}!fe^Kn|3C8PyB!O==k$Z=dZj=YoGq@8_qtu`uJ|?@}s1; z+V@|z<}~F>0!C2adM%$uvKh*(Zf=f&T$!|bS~}P@mhs>~LV1H0@(C5>7Q0n=&mK$T zLDyy)#PmfX)Yxn6qFCPP_a^yl(*~(>FRTh4l4ftJR_E^@=6p5k!9kR<4;9M~s;l?! zAD=mPz)7X>E?k;ooCOFD&5O3)u3euNF;rYA0YM#`Ju13!|7kiYPD;VG7af}S85vdR z9p!}`%>UuS-2Z%|a(zZC%O}w4mtS{qtMa8Onmmt1w;_q>sjI#B6=lBIH~19?DSvis zj@=6igHM|)F8HowxKvu(*Tph^C{im%CNk~Dm_+W4a0(eTG}&YI(E_^zc5FYi@!&HSF`MZ;!v&__W$aC%|5yzx-aQoZfh24KSUe3(~Cbol>N4Dx*gW( z)bW{QL9geI%TrAAeiPVar+DIluxeKdjlBvrbrx6c@|B|oGrntZ;`>LjCEE7JmCn-* zEG|n3lR9>r&x+S{m!|i3Xr4kanR6(du^N>S)4ER+qrPtQx-;8d$k7*4>+!mLA?0t)nUOrIN zt`P6qwaPxbzcyoZ#2Ru&KZAf-a$DR_X>h5esyqgV2)$%{xK{|R~_$c$wG zpjY1h$#dQVc87Rxu!7k5Va<4o-yFYUdSJ_>xStF?PY&N?@lqG#g1nk$6KqxCX|*J4 z>B}lA8(d*hq4vO9%Z?5tf50}&U_=6sH#e5h>Q+^eh*2%6v$8PZ@&k*4sK*WZeid?v zWN~hWT*614tD$u_<|7Y;Z}Ey%Y|ea$LXFsc>SmYFl=@MqsmdJL&Cn8N7>2d&8@yGP z!tfNo)7>T<9STcuy~Amuz_WBUfd6|{2{~sXAGZM zPZ-YzY3(P8r>pips`>2{D&a?}_oS2*WJ5g#a?KJdQkvkwF(aK&;w@brZ$=6`TCH|jxD zuPZvvL{73sDtqNfUil_dM4|X}@fXE&jO97bBX2K)J%1>tl$xkG5XGv+5?feD_4*4n zwlaFHjt4FkJu192LOnDmlpp?@B#@ zCUIt9=xcA(4J)n>4s#XhL-U<)SDEbVb+j^!HM{j6mvAJN>!?wSFytbwR`#L^iY)t+ zbslUg3Zn_6A*|#R|`A^jJM z_TN3}!%e*HVtI^z3)4Y$1iv4fLA7hKkGk~gKh4nZJ5Lgi_#Ao!$W{NA2Dwdp&M@xG z{lfQe386puS^t+uOeJK3Kl@}eite2P9ihXbT7BuX>bLJHQhK#{J(*GBsK7>lx;!1o zsQle_d*Hq8pY;s?*ey<(uZ(l@TW}j@xvmoKGt4A1g2v>>R(cP)+ z70St@uk9xPb-&(Li2KzrSYeb&S@U_HD40KqTyE$+{_&lO$*QbiN2LN zx?*5kn*oNcT_l=2d{;#ZXmFof_m-BI0YLu+ePLG9!wmb4T(0QdTyBMDBfsS1eBJX8o~u@lY+2}Osp zjqg|d=?wh+RlEF!nw-;b0#q)a)lbJaxpU8UVi}As&Nq~GFw*zR_wb7?4A1Q~SGpXT zD+T$n>>t9B6)`$=!nv4se%<-r^kJ}51G1oV9$IJ*0G-{pSFj4+CQQvop z-D-Zp7A@*XH+l5|Tu%=Bc=`^61HS_@H)JYt46+h6Yaz!0;VB<@hWHohP0 z1}+G7gVaEiv_JnKtoM7F(8O|JelX(4lXD>0ox+mnMm=SKND*aes}Iu?s^_e$ZAr$x zR-xFAzN2$tqxhm}bQVA(owUE){dqb_mC{NrC3=BZzdIp&bzxi&ldU*;x!h&O#^40W zQ*rj=h@Gc#@-C}hPHCf9b*xUFt{U^r6b>VSwTD=Bz}_?&HM2K&`(+o@L#P!}A9_U= zf@%?KOIQ9SZ+Y7ke`+QZ4F6%AA+eur*qIqAZX1VVj*YG1>|@ne%Z(DqIpbR42}+zR zzf;Qv71oUucBG0?@s`PQA6;~=j%BhUTM?+vEt>gw)(180*NxRbu6{Jxqr=LI6S*1f z$jtfabPf#f&dg;}PfyZnmNQU{68r=d@X7}6)9^Rt7llR@=_!F$Mn?)4IpdTkbo_K7 zr>FYJ)w-D8WAo-=N01T7zb13sR1LYl?<*0{>m2a$&?Rl$1FM&HGkLE#OC@=#kap}I zBY?>YksgLMQs;ve^rNY~E0hKX-!i&}{n8!DTQM$iQ1LdB)#%mr=LbG~9 z#P{mwHyD<&g@JW&iszuPh1dKBbe6Z<;;q_+3}!u*sKc3yGzS|7c>N>Po(`yf4HsU~ z{KI@n_jkrXwR_(|h2Lr9E92sWLACgO2>27__n18@VT~jqA7-Nl1KpAp^KQfFmCCvWD_ISu~7c0KcHrUlO?5wj;<VkIl(|b#cjyhp4`3@Db#vvOn^^5Div-Xn+lsCZl6*#vG5wfOw)DXA z10S+jTaUrLFQ7*`ae$$fmM%H&iA-m*fPXQLWJ_k!X!2gyGvGUu$6CHHQG;n&E{h`0 zC5};}_Ry{_k#gHrQywMNu)AawW?^ErA`0S^{Hlo5=9l|t)aLQ#kK||8s;m7qkMcYn z##u&--}DR*@Gi(5s*cpHUZcPbDG#=1i)GlBjmJHF?QS$fAh5oB z<(Au*>-<)$jqBW0xeBh4xi{OGBvj^?1P%rPj_D>7;BgE2paVayU_bO^`VC+D{=zoV z?_aKoR_L8tl>4})QQBr@C3SBqEB5+4chK1Os zYok0|miDRdO)vSg-A+-+URPjIHFe;qO3--}RJ}5zB~Ht)TQ9KA;z`fY!ChCfcW>Mz zd!ow@u4z(er+IJuMd39@D>zj3vfqBSd2bYg>g@Pc7?1b-=#Er>S>#hp@qYg-ck-1+ z4tG!<*=Tk=F23C2G07vY)G96J(38PSaOD_o=47lV54PZO-h*UrP;{M*%h#2;#kf^= zZC%xkLl2SIiCp4;u`*j(mx%N4kPbvTKIM|lg}nr_O_iFdESkk83dJs(D3^)V39uMl z?1>J6HR)rud)Cu?sVxkS@;%XTI^sB?rlIP+_7C*}u97)eET(!XxF#kYIuorGjePP+ zUxvv~(lECZ^|6+dZnRj0YP0e=2PJ0yJ%rVlsqXj*_pvib)KeCJ0=J?_O&xUJgmaeY z>dj?Am3;@Sa`royVS{r~ZZfUNCBrP|wn%MJq(Q=`hWnJifz$H>^}J2k&40==Ro6104rw`Br|+q zf&~c=H0iAp|KQCk-GApVUiKexT!25_?HA!_pv^RVyu#{C7E1M7>x@^P=WURO8yq7V z{&}|p+1B>4%;QIV4bzXV;Z~Zj1;G b{8K^-wv%tiO%3p0fPadzYEO!#p9TIO>b9UB literal 0 HcmV?d00001 diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/housing_regression_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/housing_regression_torch.ipynb new file mode 100644 index 00000000..eb558bb2 --- /dev/null +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/housing_regression_torch.ipynb @@ -0,0 +1,1982 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "792d95f9", + "metadata": {}, + "source": [ + "\n", + "\n", + "# PySpark PyTorch Inference\n", + "\n", + "### Regression\n", + "\n", + "In this notebook, we will train an MLP to perform regression on the California housing dataset, and load it for distributed inference with Spark. \n", + "\n", + "Based on: https://github.com/christianversloot/machine-learning-articles/blob/main/how-to-create-a-neural-network-for-regression-with-pytorch.md \n", + "\n", + "We also demonstrate accelerated inference via Torch-TensorRT model compilation. " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "75930360-c5ce-49ef-a69a-da88fa69a2ef", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import os\n", + "import shutil\n", + "import numpy as np\n", + "from torch import nn\n", + "from torch.utils.data import DataLoader\n", + "from sklearn.datasets import fetch_california_housing\n", + "from sklearn.preprocessing import StandardScaler" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "1de685f4", + "metadata": {}, + "outputs": [], + "source": [ + "os.mkdir('models') if not os.path.exists('models') else None" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "6d5bc0c7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'2.4.1+cu121'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.__version__" + ] + }, + { + "cell_type": "markdown", + "id": "8754b174", + "metadata": {}, + "source": [ + "### Load Dataset\n", + "\n", + "Each label corresponds to the average house value in units of 100,000, which we'll try to predict using the following features: \n", + "['MedInc', 'HouseAge', 'AveRooms', 'AveBedrms', 'Population', 'AveOccup', 'Latitude', 'Longitude']" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "2bee64cf-a44a-4aff-82db-c64ee3a8b0e8", + "metadata": {}, + "outputs": [], + "source": [ + "X, y = fetch_california_housing(return_X_y=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "8644e508-5e4c-4cdd-9ed1-9235887d9659", + "metadata": {}, + "outputs": [], + "source": [ + "class HousingDataset(torch.utils.data.Dataset):\n", + " def __init__(self, X, y, scale_data=True):\n", + " if not torch.is_tensor(X) and not torch.is_tensor(y):\n", + " # Apply scaling if necessary\n", + " if scale_data:\n", + " X = StandardScaler().fit_transform(X)\n", + " self.X = torch.from_numpy(X.astype(np.float32))\n", + " self.y = torch.from_numpy(y.astype(np.float32))\n", + "\n", + " def __len__(self):\n", + " return len(self.X)\n", + "\n", + " def __getitem__(self, i):\n", + " return self.X[i], self.y[i]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cc6b55c3-dc7b-4831-9943-83efd48091bf", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = HousingDataset(X, y)\n", + "trainloader = torch.utils.data.DataLoader(\n", + " dataset, batch_size=10, shuffle=True, num_workers=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "d868f39d-4695-4110-91d2-6f7a09d73b93", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[tensor([[-0.5241, -0.8454, -0.2556, 0.3256, 0.8226, 0.0041, 0.7623, -1.1132],\n", + " [-0.3852, -1.1632, -0.3439, -0.0999, -0.3060, -0.0110, -0.5767, 0.0198],\n", + " [ 0.0802, 0.5054, -0.0854, -0.0261, 0.1170, 0.0557, -0.8062, 0.7485],\n", + " [-0.7215, -0.5276, -0.2073, -0.1787, -0.6115, -0.0134, 1.3756, -0.8686],\n", + " [-0.9009, -0.4481, -0.4374, -0.1855, -0.1788, 0.0477, 1.0760, -0.8537],\n", + " [ 0.6685, 1.3794, 0.2930, -0.1939, -0.8066, -0.0586, 0.9402, -1.1731],\n", + " [-0.7873, -0.9249, 0.1057, -0.0384, 0.3025, -0.0482, 0.1817, 0.2794],\n", + " [ 0.2764, 1.8562, -1.3384, -0.3592, -0.3219, 1.2067, 1.0010, -1.4127],\n", + " [-0.2856, -1.7194, -0.1692, 0.0904, 1.0177, -0.0792, -0.6938, 1.1279],\n", + " [ 1.3093, -0.2097, 0.5685, 0.0202, -0.2477, -0.0660, 0.9121, -1.4127]]),\n", + " tensor([1.8440, 2.4870, 1.5770, 1.0160, 0.6780, 3.1560, 0.7980, 2.2500, 1.2220,\n", + " 5.0000])]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "next(iter(trainloader))" + ] + }, + { + "cell_type": "markdown", + "id": "1e817b9a", + "metadata": {}, + "source": [ + "### Create and Train Model" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "9a441b60-dca4-44d2-bc1c-aa7336d704bb", + "metadata": {}, + "outputs": [], + "source": [ + "class MLP(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.layers = nn.Sequential(\n", + " nn.Linear(8, 64),\n", + " nn.ReLU(),\n", + " nn.Linear(64, 32),\n", + " nn.ReLU(),\n", + " nn.Linear(32, 1)\n", + " )\n", + "\n", + " def forward(self, x):\n", + " return self.layers(x)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "15cff2b4-9d23-4d2b-808a-a5edb8eda135", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using cuda device\n" + ] + } + ], + "source": [ + "# Initialize the MLP\n", + "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + "print(f\"Using {device} device\")\n", + "mlp = MLP().to(device)\n", + "\n", + "# Define the loss function and optimizer\n", + "loss_function = nn.L1Loss()\n", + "optimizer = torch.optim.Adam(mlp.parameters(), lr=1e-4)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "5e2db3f9-5db8-4b42-89ad-e77f23c4c1fe", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Starting epoch 1\n", + "Loss after mini-batch 1: 0.003\n", + "Loss after mini-batch 201: 0.709\n", + "Loss after mini-batch 401: 0.505\n", + "Loss after mini-batch 601: 0.379\n", + "Loss after mini-batch 801: 0.305\n", + "Loss after mini-batch 1001: 0.269\n", + "Loss after mini-batch 1201: 0.257\n", + "Loss after mini-batch 1401: 0.227\n", + "Loss after mini-batch 1601: 0.214\n", + "Loss after mini-batch 1801: 0.223\n", + "Loss after mini-batch 2001: 0.223\n", + "Starting epoch 2\n", + "Loss after mini-batch 1: 0.001\n", + "Loss after mini-batch 201: 0.214\n", + "Loss after mini-batch 401: 0.206\n", + "Loss after mini-batch 601: 0.200\n", + "Loss after mini-batch 801: 0.199\n", + "Loss after mini-batch 1001: 0.203\n", + "Loss after mini-batch 1201: 0.197\n", + "Loss after mini-batch 1401: 0.197\n", + "Loss after mini-batch 1601: 0.197\n", + "Loss after mini-batch 1801: 0.188\n", + "Loss after mini-batch 2001: 0.193\n", + "Starting epoch 3\n", + "Loss after mini-batch 1: 0.001\n", + "Loss after mini-batch 201: 0.182\n", + "Loss after mini-batch 401: 0.186\n", + "Loss after mini-batch 601: 0.195\n", + "Loss after mini-batch 801: 0.195\n", + "Loss after mini-batch 1001: 0.190\n", + "Loss after mini-batch 1201: 0.186\n", + "Loss after mini-batch 1401: 0.185\n", + "Loss after mini-batch 1601: 0.185\n", + "Loss after mini-batch 1801: 0.190\n", + "Loss after mini-batch 2001: 0.186\n", + "Starting epoch 4\n", + "Loss after mini-batch 1: 0.001\n", + "Loss after mini-batch 201: 0.179\n", + "Loss after mini-batch 401: 0.182\n", + "Loss after mini-batch 601: 0.187\n", + "Loss after mini-batch 801: 0.183\n", + "Loss after mini-batch 1001: 0.182\n", + "Loss after mini-batch 1201: 0.187\n", + "Loss after mini-batch 1401: 0.179\n", + "Loss after mini-batch 1601: 0.183\n", + "Loss after mini-batch 1801: 0.177\n", + "Loss after mini-batch 2001: 0.184\n", + "Starting epoch 5\n", + "Loss after mini-batch 1: 0.001\n", + "Loss after mini-batch 201: 0.174\n", + "Loss after mini-batch 401: 0.181\n", + "Loss after mini-batch 601: 0.183\n", + "Loss after mini-batch 801: 0.176\n", + "Loss after mini-batch 1001: 0.179\n", + "Loss after mini-batch 1201: 0.176\n", + "Loss after mini-batch 1401: 0.183\n", + "Loss after mini-batch 1601: 0.177\n", + "Loss after mini-batch 1801: 0.183\n", + "Loss after mini-batch 2001: 0.172\n", + "Training process has finished.\n" + ] + } + ], + "source": [ + "# Run the training loop\n", + "for epoch in range(0, 5): # 5 epochs at maximum\n", + "\n", + " # Print epoch\n", + " print(f'Starting epoch {epoch+1}')\n", + "\n", + " # Set current loss value\n", + " current_loss = 0.0\n", + "\n", + " # Iterate over the DataLoader for training data\n", + " for i, data in enumerate(trainloader, 0):\n", + "\n", + " # Get and prepare inputs\n", + " inputs, targets = data\n", + " inputs, targets = inputs.to(device), targets.to(device)\n", + " targets = targets.reshape((targets.shape[0], 1))\n", + "\n", + " # Zero the gradients\n", + " optimizer.zero_grad()\n", + "\n", + " # Perform forward pass\n", + " outputs = mlp(inputs)\n", + "\n", + " # Compute loss\n", + " loss = loss_function(outputs, targets)\n", + "\n", + " # Perform backward pass\n", + " loss.backward()\n", + "\n", + " # Perform optimization\n", + " optimizer.step()\n", + "\n", + " # Print statistics\n", + " current_loss += loss.item()\n", + " if i % 200 == 0:\n", + " print('Loss after mini-batch %5d: %.3f' %\n", + " (i + 1, current_loss / 500))\n", + " current_loss = 0.0\n", + "\n", + "# Process is complete.\n", + "print('Training process has finished.')" + ] + }, + { + "cell_type": "markdown", + "id": "352539f5", + "metadata": {}, + "source": [ + "### Save Model State Dict\n", + "This saves the serialized object to disk using pickle." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "b950a3ed-ffe1-477f-a84f-f71c85dbf9ce", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved PyTorch Model State to models/housing_model.pt\n" + ] + } + ], + "source": [ + "torch.save(mlp.state_dict(), \"models/housing_model.pt\")\n", + "print(\"Saved PyTorch Model State to models/housing_model.pt\")" + ] + }, + { + "cell_type": "markdown", + "id": "0060fcca", + "metadata": {}, + "source": [ + "### Save Model as TorchScript\n", + "This saves an [intermediate representation of the compute graph](https://pytorch.org/tutorials/beginner/saving_loading_models.html#export-load-model-in-torchscript-format), which does not require pickle (or even python). " + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "20fedb5d-c59e-4b0b-ba91-3dd15df1f09e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved TorchScript Model to models/ts_housing_model.pt\n" + ] + } + ], + "source": [ + "scripted = torch.jit.script(mlp)\n", + "scripted.save(\"models/ts_housing_model.pt\")\n", + "print(\"Saved TorchScript Model to models/ts_housing_model.pt\")" + ] + }, + { + "cell_type": "markdown", + "id": "3101c0fe-65f1-411e-9192-e8a6b585ba0d", + "metadata": {}, + "source": [ + "### Load and Test from Model State" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "7411b00f-88d2-40f5-b716-a26733c968ff", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "loaded_mlp = MLP().to(device)\n", + "loaded_mlp.load_state_dict(torch.load(\"models/housing_model.pt\", weights_only=True))" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "e226f449-2931-4492-9003-503cdc61f061", + "metadata": {}, + "outputs": [], + "source": [ + "testX, testY = next(iter(trainloader))" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "d46af47e-db7e-42ee-9bd3-6e7d93850be3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Predictions:\n" + ] + }, + { + "data": { + "text/plain": [ + "tensor([[2.0423],\n", + " [1.9258],\n", + " [2.8864],\n", + " [3.2128],\n", + " [2.8639],\n", + " [1.0359],\n", + " [2.8652],\n", + " [1.5528],\n", + " [1.7592],\n", + " [0.7497]], device='cuda:0', grad_fn=)" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(\"Predictions:\")\n", + "loaded_mlp(testX.to(device))" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "13ae2c0f-1da5-45a4-bf32-ed8b562d7907", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Labels:\n" + ] + }, + { + "data": { + "text/plain": [ + "tensor([2.2030, 2.1590, 1.9400, 3.4310, 2.4480, 1.1410, 2.8780, 0.8310, 1.6530,\n", + " 1.2380])" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(\"Labels:\")\n", + "testY" + ] + }, + { + "cell_type": "markdown", + "id": "3bcd329d", + "metadata": {}, + "source": [ + "### Load and Test from TorchScript" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "422e317f-c9bd-4f76-9463-7af2935d401d", + "metadata": {}, + "outputs": [], + "source": [ + "scripted_mlp = torch.jit.load(\"models/ts_housing_model.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "0cda8ec8-644e-4888-bfa0-b79425ece7c3", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Predictions:\n" + ] + }, + { + "data": { + "text/plain": [ + "tensor([2.0423, 1.9258, 2.8864, 3.2128, 2.8639, 1.0359, 2.8652, 1.5528, 1.7592,\n", + " 0.7497], device='cuda:0', grad_fn=)" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(\"Predictions:\")\n", + "scripted_mlp(testX.to(device)).flatten()" + ] + }, + { + "cell_type": "markdown", + "id": "2a3b64e4", + "metadata": {}, + "source": [ + "### Compile using the Torch JIT Compiler\n", + "This leverages the [Torch-TensorRT inference compiler](https://pytorch.org/TensorRT/) for accelerated inference on GPUs using the `torch.compile` JIT interface under the hood. The compiler stack returns a [boxed-function](http://blog.ezyang.com/2020/09/lets-talk-about-the-pytorch-dispatcher/) that triggers compilation on the first call. \n", + "\n", + "Modules compiled in this fashion are [not serializable with pickle](https://github.com/pytorch/pytorch/issues/101107#issuecomment-1542688089), so we cannot send the compiled model directly to Spark. " + ] + }, + { + "cell_type": "markdown", + "id": "c613f24e", + "metadata": {}, + "source": [ + "(You may see a warning about modelopt quantization. This is safe to ignore, as [implicit quantization](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#intro-quantization) is deprecated in the latest TensorRT. See [this link](https://pytorch.org/TensorRT/tutorials/_rendered_examples/dynamo/vgg16_fp8_ptq.html) for a guide to explicit quantization.)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "9ffb27fc", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:torch_tensorrt.dynamo.conversion.aten_ops_converters:Unable to import quantization op. Please install modelopt library (https://github.com/NVIDIA/TensorRT-Model-Optimizer?tab=readme-ov-file#installation) to add support for compiling quantized models\n" + ] + } + ], + "source": [ + "import torch_tensorrt as trt\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "e0c10f90", + "metadata": {}, + "outputs": [], + "source": [ + "# Optional: set the filename for the TensorRT timing cache\n", + "timestamp = time.time()\n", + "timing_cache = f\"/tmp/timing_cache-{timestamp}.bin\"\n", + "with open(timing_cache, \"wb\") as f:\n", + " pass" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "b4aa2523", + "metadata": {}, + "outputs": [], + "source": [ + "inputs_bs1 = torch.randn((10, 8), dtype=torch.float).to(\"cuda\")\n", + "# This indicates dimension 0 of inputs_bs1 is dynamic with a range of values [1, 50]. No recompilation will happen when the batch size changes.\n", + "torch._dynamo.mark_dynamic(inputs_bs1, 0, min=1, max=50)\n", + "trt_model = trt.compile(\n", + " loaded_mlp,\n", + " ir=\"torch_compile\",\n", + " inputs=inputs_bs1,\n", + " enabled_precisions={torch.float},\n", + " timing_cache_path=timing_cache,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "a5da8cab", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:torch_tensorrt.dynamo.utils:Using Default Torch-TRT Runtime (as requested by user)\n", + "INFO:torch_tensorrt.dynamo.utils:Device not specified, using Torch default current device - cuda:0. If this is incorrect, please specify an input device, via the device keyword.\n", + "INFO:torch_tensorrt.dynamo.utils:Compilation Settings: CompilationSettings(enabled_precisions={}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, refit=False, engine_capability=, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/timing_cache-1738007491.6462495.bin')\n", + "\n", + "WARNING:torch_tensorrt.dynamo._compiler:Node _param_constant1 of op type get_attr does not have metadata. This could sometimes lead to undefined behavior.\n", + "WARNING:torch_tensorrt.dynamo._compiler:Some nodes do not have metadata (shape and dtype information). This could lead to problems sometimes if the graph has PyTorch and TensorRT segments.\n", + "INFO:torch_tensorrt.dynamo._compiler:Partitioning the graph via the fast partitioner\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init CUDA: CPU +2, GPU +0, now: CPU 581, GPU 2466 (MiB)\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init builder kernel library: CPU +1635, GPU +288, now: CPU 2363, GPU 2754 (MiB)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Predictions:\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/torch_tensorrt/dynamo/conversion/impl/activation/base.py:40: DeprecationWarning: Use Deprecated in TensorRT 10.1. Superseded by explicit quantization. instead.\n", + " if input_val.dynamic_range is not None and dyn_range_fn is not None:\n", + "\n", + "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT INetwork construction elapsed time: 0:00:00.008361\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Global timing cache in use. Profiling results in this builder pass will be stored.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Detected 1 inputs and 1 output network tensors.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Host Persistent Memory: 22240\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Device Persistent Memory: 0\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Scratch Memory: 0\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Started assigning block shifts. This will take 10 steps to complete.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Algorithm ShiftNTopDown took 0.147039ms to assign 4 blocks to 10 nodes requiring 7168 bytes.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Activation Memory: 6656\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Weights Memory: 11648\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Engine generation completed in 1.60138 seconds.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage of TRT CPU/GPU memory allocators: CPU 0 MiB, GPU 1 MiB\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage during Engine building and serialization: CPU: 4106 MiB\n", + "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:Build TRT engine elapsed time: 0:00:01.606208\n", + "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT Engine uses: 390420 bytes of Memory\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 26 bytes of code generator cache.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 52 timing cache entries\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor([[2.0423],\n", + " [1.9258],\n", + " [2.8864],\n", + " [3.2128],\n", + " [2.8639],\n", + " [1.0359],\n", + " [2.8652],\n", + " [1.5528],\n", + " [1.7592],\n", + " [0.7497]], device='cuda:0')\n" + ] + } + ], + "source": [ + "stream = torch.cuda.Stream()\n", + "with torch.no_grad(), torch.cuda.stream(stream):\n", + " testX = testX.to(device)\n", + " print(\"Predictions:\")\n", + " print(trt_model(testX))" + ] + }, + { + "cell_type": "markdown", + "id": "d2c55e07", + "metadata": {}, + "source": [ + "### Compile using the Torch-TensorRT AOT Compiler\n", + "Alternatively, use the Torch-TensorRT Dynamo backend for Ahead-of-Time (AOT) compilation to eagerly optimize the model in an explicit compilation phase. We first export the model to produce a traced graph representing the Tensor computation in an AOT fashion, which produces a `ExportedProgram` object which can be [serialized and reloaded](https://pytorch.org/TensorRT/user_guide/saving_models.html). We can then compile this IR using the Torch-TensorRT AOT compiler for inference. \n", + "\n", + "[Read the docs](https://pytorch.org/TensorRT/user_guide/torch_tensorrt_explained.html) for more information on JIT vs AOT compilation." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bf36a50d", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:torch_tensorrt.dynamo._compiler:Compilation Settings: CompilationSettings(enabled_precisions={}, debug=False, workspace_size=1073741824, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, refit=False, engine_capability=, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/timing_cache-1738007491.6462495.bin')\n", + "\n", + "INFO:torch_tensorrt.dynamo._compiler:Partitioning the graph via the fast partitioner\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init CUDA: CPU +0, GPU +0, now: CPU 892, GPU 2468 (MiB)\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init builder kernel library: CPU +1633, GPU +286, now: CPU 2525, GPU 2754 (MiB)\n", + "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/torch_tensorrt/dynamo/conversion/impl/activation/base.py:40: DeprecationWarning: Use Deprecated in TensorRT 10.1. Superseded by explicit quantization. instead.\n", + " if input_val.dynamic_range is not None and dyn_range_fn is not None:\n", + "\n", + "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT INetwork construction elapsed time: 0:00:00.002784\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Global timing cache in use. Profiling results in this builder pass will be stored.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Detected 1 inputs and 1 output network tensors.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Host Persistent Memory: 18368\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Device Persistent Memory: 0\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Scratch Memory: 0\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Started assigning block shifts. This will take 6 steps to complete.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Algorithm ShiftNTopDown took 0.237457ms to assign 3 blocks to 6 nodes requiring 25088 bytes.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Activation Memory: 24576\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Weights Memory: 12292\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Engine generation completed in 1.33069 seconds.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage of TRT CPU/GPU memory allocators: CPU 0 MiB, GPU 5 MiB\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage during Engine building and serialization: CPU: 4131 MiB\n", + "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:Build TRT engine elapsed time: 0:00:01.332388\n", + "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT Engine uses: 212892 bytes of Memory\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 26 bytes of code generator cache.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 100 timing cache entries\n" + ] + } + ], + "source": [ + "example_inputs = (torch.randn((10, 8), dtype=torch.float).to(\"cuda\"),)\n", + "\n", + "# Mark dim 1 (batch size) as dynamic\n", + "batch = torch.export.Dim(\"batch\", min=1, max=64)\n", + "# Produce traced graph in ExportedProgram format\n", + "exp_program = torch.export.export(loaded_mlp, args=example_inputs, dynamic_shapes={\"x\": {0: batch}})\n", + "# Compile the traced graph to produce an optimized module\n", + "trt_gm = trt.dynamo.compile(exp_program,\n", + " tuple(example_inputs),\n", + " enabled_precisions={torch.float},\n", + " timing_cache_path=timing_cache,\n", + " workspace_size=1<<30)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "4fc4efd5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + ".GraphModuleImpl'>\n" + ] + } + ], + "source": [ + "print(type(exp_program))\n", + "print(type(trt_gm))" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "1bcf9c47", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Predictions:\n", + "tensor([[2.0425],\n", + " [1.9257],\n", + " [2.8869],\n", + " [3.2127],\n", + " [2.8640],\n", + " [1.0362],\n", + " [2.8658],\n", + " [1.5528],\n", + " [1.7586],\n", + " [0.7496]], device='cuda:0')\n" + ] + } + ], + "source": [ + "stream = torch.cuda.Stream()\n", + "with torch.no_grad(), torch.cuda.stream(stream):\n", + " print(\"Predictions:\")\n", + " testX = testX.to(device)\n", + " print(trt_gm(testX))" + ] + }, + { + "cell_type": "markdown", + "id": "0eeb957a", + "metadata": {}, + "source": [ + "We can run the optimized module with a few different batch sizes (without recompilation!):" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "49f72c14", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Output shapes:\n", + "torch.Size([10, 1])\n", + "torch.Size([1, 1])\n", + "torch.Size([50, 1])\n" + ] + } + ], + "source": [ + "inputs = (torch.randn((10, 8), dtype=torch.float).cuda(),)\n", + "inputs_bs1 = (torch.randn((1, 8), dtype=torch.float).cuda(),)\n", + "inputs_bs50 = (torch.randn((50, 8), dtype=torch.float).cuda(),)\n", + "\n", + "stream = torch.cuda.Stream()\n", + "with torch.no_grad(), torch.cuda.stream(stream):\n", + " print(\"Output shapes:\")\n", + " print(trt_gm(*inputs).shape)\n", + " print(trt_gm(*inputs_bs1).shape)\n", + " print(trt_gm(*inputs_bs50).shape)" + ] + }, + { + "cell_type": "markdown", + "id": "b4fef57d", + "metadata": {}, + "source": [ + "We can serialize the ExportedProgram (a traced graph representing the model's forward function) using `torch.export.save` to be recompiled at a later date." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "876fea4a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved ExportedProgram to models/trt_housing_model.ep\n" + ] + } + ], + "source": [ + "torch.export.save(exp_program, \"models/trt_housing_model.ep\")\n", + "print(\"Saved ExportedProgram to models/trt_housing_model.ep\")" + ] + }, + { + "cell_type": "markdown", + "id": "13631d1f-2c71-4bee-afcb-bd3b55ec87c5", + "metadata": {}, + "source": [ + "## PySpark" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "bb71dd36", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/pyspark/broadcast.py:38: DeprecationWarning: typing.io is deprecated, import directly from typing instead. typing.io will be removed in Python 3.12.\n", + " from typing.io import BinaryIO # type: ignore[import]\n", + "\n" + ] + } + ], + "source": [ + "from pyspark.sql.functions import col, struct, pandas_udf, array\n", + "from pyspark.ml.functions import predict_batch_udf\n", + "from pyspark.sql.types import *\n", + "from pyspark.sql import SparkSession\n", + "from pyspark import SparkConf\n", + "import json\n", + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "id": "6769c060", + "metadata": {}, + "source": [ + "Check the cluster environment to handle any platform-specific Spark configurations." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "f7727b58", + "metadata": {}, + "outputs": [], + "source": [ + "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n", + "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n", + "on_standalone = not (on_databricks or on_dataproc)" + ] + }, + { + "cell_type": "markdown", + "id": "a3b7d360", + "metadata": {}, + "source": [ + "#### Create Spark Session\n", + "\n", + "For local standalone clusters, we'll connect to the cluster and create the Spark Session. \n", + "For CSP environments, Spark will either be preconfigured (Databricks) or we'll need to create the Spark Session (Dataproc)." + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "52e9dbdb", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/27 19:51:37 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/01/27 19:51:37 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", + "25/01/27 19:51:37 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "25/01/27 19:51:38 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + ] + } + ], + "source": [ + "conf = SparkConf()\n", + "\n", + "if 'spark' not in globals():\n", + " if on_standalone:\n", + " import socket\n", + " conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + " hostname = socket.gethostname()\n", + " conf.setMaster(f\"spark://{hostname}:7077\")\n", + " conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", + " conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_env}/lib:{conda_env}/lib/python3.11/site-packages/nvidia_pytriton.libs:$LD_LIBRARY_PATH\")\n", + " elif on_dataproc:\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conda_lib_path=\"/opt/conda/miniconda3/lib\"\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_lib_path}:$LD_LIBRARY_PATH\")\n", + " conf.set(\"spark.executor.instances\", \"4\") # dataproc defaults to 2\n", + "\n", + " conf.set(\"spark.executor.cores\", \"8\")\n", + " conf.set(\"spark.task.resource.gpu.amount\", \"0.125\")\n", + " conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n", + " conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", + " conf.set(\"spark.python.worker.reuse\", \"true\")\n", + " \n", + "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", + "sc = spark.sparkContext" + ] + }, + { + "cell_type": "markdown", + "id": "e3b9937e-2c70-4d67-b95f-4d9d5ab17c12", + "metadata": {}, + "source": [ + "### Create Spark DataFrame from Pandas DataFrame" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "cf35da14-61a3-4e7b-9d4f-086bf5e931b3", + "metadata": {}, + "outputs": [], + "source": [ + "housing = fetch_california_housing()" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "95148019-ea95-40e5-a529-fcdb9a06f928", + "metadata": {}, + "outputs": [], + "source": [ + "X = StandardScaler().fit_transform(housing.data.astype(np.float32))" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "f82d957c-6747-4408-aac8-45305afbfe5e", + "metadata": {}, + "outputs": [], + "source": [ + "pdf = pd.DataFrame(X, columns=housing.feature_names)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "881afee9", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/pyspark/sql/pandas/serializers.py:229: DeprecationWarning: is_categorical_dtype is deprecated and will be removed in a future version. Use isinstance(dtype, pd.CategoricalDtype) instead\n", + " elif is_categorical_dtype(s.dtype):\n", + "\n", + "[Stage 0:> (0 + 8) / 8]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+------------+------------+-----------+------------+-----------+------------+----------+------------+\n", + "| MedInc| HouseAge| AveRooms| AveBedrms| Population| AveOccup| Latitude| Longitude|\n", + "+------------+------------+-----------+------------+-----------+------------+----------+------------+\n", + "| 0.20909257| -1.1632254| 0.38946992| 0.04609274| -0.9806099| -0.07099328|0.61245227|-0.020113053|\n", + "|-0.098627955| 0.34647804| 0.27216315| -0.0129226| -0.6953838| -0.05380849| 1.0665938| -1.2479742|\n", + "| -0.66006273| 1.0616008|-0.55292207| -0.48945764|-0.13641118| 0.028952759| 1.1040496| -1.3827378|\n", + "| 0.08218294| 0.5848523|-0.13912922| -0.14707813|-0.19116047| -0.07136432|0.96827507| -1.3028787|\n", + "| 0.0784456| -1.4810578| 0.57265776| 0.32067496| 1.0345173|-0.024157424| 1.4411427| -0.52423614|\n", + "| -0.82318723| -0.36864465| 0.07829511| -0.1808107|-0.67242444|-0.061470542| 1.9374212| -1.0083897|\n", + "| 0.59671736| 0.5848523| 0.19346413| -0.1371872|-0.19645879| 0.009964322|0.96827507| -1.2928978|\n", + "| -0.9612035| -1.5605159|-0.56329846| 0.027148023|-0.71127874| -0.08471591| 0.5328614| -0.13990337|\n", + "| -0.74344087| -1.2426835| 0.27282518| 0.4037246| -0.9841421| -0.05610115| 1.2257773| -0.42940006|\n", + "| 0.9784464| -0.2891866| 0.24374022| -0.24670053| 0.28922042| -0.01102468| 1.1087307| -1.2280084|\n", + "| -0.5070446| -1.0043093|-0.78254056|0.0122275995| 2.8465424|-0.060435444| 0.8980464| -1.2080427|\n", + "| -0.18690155| 1.2205169|0.015323491| 0.12183313|-0.41015765| 0.04452552| 1.010412| -1.3228445|\n", + "| -1.2551856| 1.6178073| -0.3341509|-0.060125165| -0.7554314| -0.08777025| 1.0291398| -1.3477987|\n", + "| 4.9607058| -1.9578062| 1.4854684| -0.03948475| 2.1833694|0.0029250523| 1.024457| -1.1581304|\n", + "| 0.73652315| -1.6399739| 0.7913185| -0.05238397| 1.67738| 0.01944797| 1.0993668| -1.1331724|\n", + "| -0.505834| 0.18756187|-0.47093546| -0.24297306|-0.60619545| -0.10791535| 0.977639| -1.2879055|\n", + "| -0.88477343|-0.050812364| -0.6318951| -0.15244243| -0.5258376| -0.15618815| 0.9823201| -1.2879055|\n", + "| -0.42840376| 0.9821427| -0.2266495| -0.36083496| -0.6883194| -0.08552282| 0.5328614| -0.12493005|\n", + "| 0.9369153| -1.4810578| 0.6722208|-0.121177554| 0.3996021| 0.01291408| 1.1040496| -1.1082181|\n", + "| -0.80702734| -0.92485124|-0.26602685| -0.1560743| 1.4398388| -0.09314839|0.55627036| -0.09498342|\n", + "+------------+------------+-----------+------------+-----------+------------+----------+------------+\n", + "only showing top 20 rows\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "schema = StructType([\n", + " StructField(\"MedInc\",FloatType(),True),\n", + " StructField(\"HouseAge\",FloatType(),True),\n", + " StructField(\"AveRooms\",FloatType(),True),\n", + " StructField(\"AveBedrms\",FloatType(),True),\n", + " StructField(\"Population\",FloatType(),True),\n", + " StructField(\"AveOccup\",FloatType(),True),\n", + " StructField(\"Latitude\",FloatType(),True),\n", + " StructField(\"Longitude\",FloatType(),True)\n", + "])\n", + "\n", + "df = spark.createDataFrame(pdf, schema=schema).repartition(8)\n", + "df.show(truncate=12)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "7b33d367-fbf9-4918-b755-5447125547c4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "StructType([StructField('MedInc', FloatType(), True), StructField('HouseAge', FloatType(), True), StructField('AveRooms', FloatType(), True), StructField('AveBedrms', FloatType(), True), StructField('Population', FloatType(), True), StructField('AveOccup', FloatType(), True), StructField('Latitude', FloatType(), True), StructField('Longitude', FloatType(), True)])" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.schema" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "751bff7a-b687-4184-b3fa-b5f5b46ef5d1", + "metadata": {}, + "outputs": [], + "source": [ + "data_path = \"spark-dl-datasets/california_housing\"\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " data_path = \"dbfs:/FileStore/\" + data_path\n", + "\n", + "df.write.mode(\"overwrite\").parquet(data_path)" + ] + }, + { + "cell_type": "markdown", + "id": "88c3cd75", + "metadata": {}, + "source": [ + "## Inference using Spark DL API\n", + "\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "\n", + "- predict_batch_fn uses PyTorch APIs to load the model and return a predict function which operates on numpy arrays \n", + "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "1e40c266-24de-454d-a776-f3716ba50e90", + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.parquet(data_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "5b144c17", + "metadata": {}, + "outputs": [], + "source": [ + "columns = df.columns" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "3d608e2f-66a8-44b5-9cde-5f7837bf4247", + "metadata": {}, + "outputs": [], + "source": [ + "# get absolute path to model\n", + "model_path = \"{}/models/trt_housing_model.ep\".format(os.getcwd())\n", + "\n", + "# For cloud environments, copy the model to the distributed file system.\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-models\")\n", + " dbfs_model_path = \"/dbfs/FileStore/spark-dl-models/trt_housing_model.ep\"\n", + " shutil.copy(model_path, dbfs_model_path)\n", + " model_path = dbfs_model_path\n", + "elif on_dataproc:\n", + " # GCS is mounted at /mnt/gcs by the init script\n", + " models_dir = \"/mnt/gcs/spark-dl/models\"\n", + " os.mkdir(models_dir) if not os.path.exists(models_dir) else None\n", + " gcs_model_path = models_dir + \"/trt_housing_model.ep\"\n", + " shutil.copy(model_path, gcs_model_path)\n", + " model_path = gcs_model_path" + ] + }, + { + "cell_type": "markdown", + "id": "2fd143e7", + "metadata": {}, + "source": [ + "For inference on Spark, we'll load the ExportedProgram and compile the model with the Torch-TensorRT AOT compiler and cache on the executor. " + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "fc400771", + "metadata": {}, + "outputs": [], + "source": [ + "# A resource warning may occur due to unclosed file descriptors used by TensorRT across multiple PySpark daemon processes.\n", + "# These can be safely ignored as the resources will be cleaned up when the worker processes terminate.\n", + "\n", + "import warnings\n", + "warnings.simplefilter(\"ignore\", ResourceWarning)" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "a2f45f5d-c941-4197-a274-1eec2af3fca4", + "metadata": {}, + "outputs": [], + "source": [ + "def predict_batch_fn():\n", + " import torch\n", + " import torch_tensorrt as trt\n", + "\n", + " device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + " if device != \"cuda\":\n", + " raise ValueError(\"This function uses the TensorRT model which requires a GPU device\")\n", + "\n", + " example_inputs = (torch.randn((50, 8), dtype=torch.float).to(\"cuda\"),)\n", + " exp_program = torch.export.load(model_path)\n", + " trt_gm = trt.dynamo.compile(exp_program,\n", + " tuple(example_inputs),\n", + " enabled_precisions={torch.float},\n", + " timing_cache_path=timing_cache,\n", + " workspace_size=1<<30)\n", + "\n", + " print(\"Model compiled.\")\n", + " \n", + " def predict(inputs):\n", + " stream = torch.cuda.Stream()\n", + " with torch.no_grad(), torch.cuda.stream(stream), trt.logging.errors():\n", + " print(f\"Predict {inputs.shape}\")\n", + " torch_inputs = torch.from_numpy(inputs).to(device)\n", + " outputs = trt_gm(torch_inputs) # .flatten()\n", + " return outputs.detach().cpu().numpy()\n", + "\n", + " return predict" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "220a00a4-e842-4f5d-a4b3-7693d09e2d31", + "metadata": {}, + "outputs": [], + "source": [ + "regress = predict_batch_udf(predict_batch_fn,\n", + " return_type=FloatType(),\n", + " input_tensor_shapes=[[8]],\n", + " batch_size=50)" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "0f3bf287-8ffc-4456-8772-e97c418d6aee", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 7:=======> (1 + 7) / 8]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 29.2 ms, sys: 4.38 ms, total: 33.6 ms\n", + "Wall time: 8.23 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "%%time\n", + "preds = df.withColumn(\"preds\", regress(struct(*columns)))\n", + "results = preds.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "6cd23b71-296d-4ce7-b56c-567cc2eec79c", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 30.7 ms, sys: 6.15 ms, total: 36.8 ms\n", + "Wall time: 309 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "preds = df.withColumn(\"preds\", regress(array(*columns)))\n", + "results = preds.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "75d16bd5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 27.3 ms, sys: 4.6 ms, total: 31.9 ms\n", + "Wall time: 276 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "preds = df.withColumn(\"preds\", regress(array(*columns)))\n", + "results = preds.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "764a40d8-25f7-425c-ba03-fe8c45f4b063", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+------------+------------+-----------+------------+-----------+------------+----------+------------+---------+\n", + "| MedInc| HouseAge| AveRooms| AveBedrms| Population| AveOccup| Latitude| Longitude| preds|\n", + "+------------+------------+-----------+------------+-----------+------------+----------+------------+---------+\n", + "| 0.20909257| -1.1632254| 0.38946992| 0.04609274| -0.9806099| -0.07099328|0.61245227|-0.020113053|1.4441836|\n", + "|-0.098627955| 0.34647804| 0.27216315| -0.0129226| -0.6953838| -0.05380849| 1.0665938| -1.2479742|1.7245855|\n", + "| -0.66006273| 1.0616008|-0.55292207| -0.48945764|-0.13641118| 0.028952759| 1.1040496| -1.3827378|1.3524103|\n", + "| 0.08218294| 0.5848523|-0.13912922| -0.14707813|-0.19116047| -0.07136432|0.96827507| -1.3028787|2.3009148|\n", + "| 0.0784456| -1.4810578| 0.57265776| 0.32067496| 1.0345173|-0.024157424| 1.4411427| -0.52423614| 1.272077|\n", + "| -0.82318723| -0.36864465| 0.07829511| -0.1808107|-0.67242444|-0.061470542| 1.9374212| -1.0083897|0.6968852|\n", + "| 0.59671736| 0.5848523| 0.19346413| -0.1371872|-0.19645879| 0.009964322|0.96827507| -1.2928978|2.6057932|\n", + "| -0.9612035| -1.5605159|-0.56329846| 0.027148023|-0.71127874| -0.08471591| 0.5328614| -0.13990337|1.0139045|\n", + "| -0.74344087| -1.2426835| 0.27282518| 0.4037246| -0.9841421| -0.05610115| 1.2257773| -0.42940006|0.9931088|\n", + "| 0.9784464| -0.2891866| 0.24374022| -0.24670053| 0.28922042| -0.01102468| 1.1087307| -1.2280084| 2.64501|\n", + "| -0.5070446| -1.0043093|-0.78254056|0.0122275995| 2.8465424|-0.060435444| 0.8980464| -1.2080427|2.1324868|\n", + "| -0.18690155| 1.2205169|0.015323491| 0.12183313|-0.41015765| 0.04452552| 1.010412| -1.3228445|1.9459988|\n", + "| -1.2551856| 1.6178073| -0.3341509|-0.060125165| -0.7554314| -0.08777025| 1.0291398| -1.3477987|1.2558049|\n", + "| 4.9607058| -1.9578062| 1.4854684| -0.03948475| 2.1833694|0.0029250523| 1.024457| -1.1581304|5.8959594|\n", + "| 0.73652315| -1.6399739| 0.7913185| -0.05238397| 1.67738| 0.01944797| 1.0993668| -1.1331724|2.0677836|\n", + "| -0.505834| 0.18756187|-0.47093546| -0.24297306|-0.60619545| -0.10791535| 0.977639| -1.2879055|1.7652202|\n", + "| -0.88477343|-0.050812364| -0.6318951| -0.15244243| -0.5258376| -0.15618815| 0.9823201| -1.2879055|1.5843444|\n", + "| -0.42840376| 0.9821427| -0.2266495| -0.36083496| -0.6883194| -0.08552282| 0.5328614| -0.12493005|0.9752624|\n", + "| 0.9369153| -1.4810578| 0.6722208|-0.121177554| 0.3996021| 0.01291408| 1.1040496| -1.1082181|2.2553492|\n", + "| -0.80702734| -0.92485124|-0.26602685| -0.1560743| 1.4398388| -0.09314839|0.55627036| -0.09498342|1.0328484|\n", + "+------------+------------+-----------+------------+-----------+------------+----------+------------+---------+\n", + "only showing top 20 rows\n", + "\n" + ] + } + ], + "source": [ + "preds.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "0aa85f81", + "metadata": {}, + "outputs": [], + "source": [ + "# This will clear the engine cache (containing previously compiled TensorRT engines) and reset the CUDA Context.\n", + "torch._dynamo.reset()" + ] + }, + { + "cell_type": "markdown", + "id": "53536808", + "metadata": {}, + "source": [ + "## Using Triton Inference Server\n", + "In this section, we demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. \n", + "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server. \n", + "\n", + "The process looks like this:\n", + "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n", + "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n", + "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n", + "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n", + "\n", + "\"drawing\"" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "id": "a9ab4cdf-8103-447e-9ac8-944e2e527239", + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial" + ] + }, + { + "cell_type": "markdown", + "id": "1b77dc96", + "metadata": {}, + "source": [ + "Import the utility functions from pytriton_utils.py:" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "1ac83062", + "metadata": {}, + "outputs": [], + "source": [ + "sc.addPyFile(\"pytriton_utils.py\")\n", + "\n", + "from pytriton_utils import (\n", + " use_stage_level_scheduling,\n", + " find_ports,\n", + " start_triton,\n", + " stop_triton\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "a4cc5d81", + "metadata": {}, + "source": [ + "Define the Triton Server function:" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "6632636e-67a3-406c-832c-758aac4245fd", + "metadata": {}, + "outputs": [], + "source": [ + "def triton_server(ports, model_path):\n", + " import time\n", + " import signal\n", + " import numpy as np\n", + " import torch\n", + " import torch_tensorrt as trt\n", + " from pytriton.decorators import batch\n", + " from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n", + " from pytriton.triton import Triton, TritonConfig\n", + " from pyspark import TaskContext\n", + "\n", + " print(f\"SERVER: Initializing model on worker {TaskContext.get().partitionId()}.\")\n", + " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + " \n", + " exp_program = torch.export.load(model_path)\n", + " example_inputs = (torch.randn((50, 8), dtype=torch.float).to(\"cuda\"),)\n", + " trt_gm = trt.dynamo.compile(exp_program,\n", + " tuple(example_inputs),\n", + " enabled_precisions={torch.float},\n", + " workspace_size=1<<30)\n", + "\n", + " print(\"SERVER: Compiled model.\")\n", + "\n", + " @batch\n", + " def _infer_fn(**inputs):\n", + " features = inputs[\"features\"]\n", + " if len(inputs[\"features\"]) != 1:\n", + " features = np.squeeze(features)\n", + " stream = torch.cuda.Stream()\n", + " with torch.no_grad(), torch.cuda.stream(stream):\n", + " torch_inputs = torch.from_numpy(features).to(device)\n", + " outputs = trt_gm(torch_inputs)\n", + " return {\n", + " \"preds\": outputs.cpu().numpy(),\n", + " }\n", + "\n", + " workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n", + " triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n", + " with Triton(config=triton_conf, workspace=workspace_path) as triton:\n", + " triton.bind(\n", + " model_name=\"HousingModel\",\n", + " infer_func=_infer_fn,\n", + " inputs=[\n", + " Tensor(name=\"features\", dtype=np.float32, shape=(-1,)),\n", + " ],\n", + " outputs=[\n", + " Tensor(name=\"preds\", dtype=np.float32, shape=(-1,)),\n", + " ],\n", + " config=ModelConfig(\n", + " max_batch_size=50,\n", + " batcher=DynamicBatcher(max_queue_delay_microseconds=5000), # 5ms\n", + " ),\n", + " strict=True,\n", + " )\n", + "\n", + " def _stop_triton(signum, frame):\n", + " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", + " triton.stop()\n", + "\n", + " signal.signal(signal.SIGTERM, _stop_triton)\n", + "\n", + " print(\"SERVER: Serving inference\")\n", + " triton.serve()" + ] + }, + { + "cell_type": "markdown", + "id": "74121cd7", + "metadata": {}, + "source": [ + "#### Start Triton servers" + ] + }, + { + "cell_type": "markdown", + "id": "bbcca988", + "metadata": {}, + "source": [ + "**Specify the number of nodes in the cluster.** \n", + "Following the README, the example standalone cluster uses 1 node. The example Databricks/Dataproc cluster scripts use 4 nodes by default. " + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "160a0460", + "metadata": {}, + "outputs": [], + "source": [ + "# Change based on cluster setup\n", + "num_nodes = 1 if on_standalone else 4" + ] + }, + { + "cell_type": "markdown", + "id": "8bba4f54", + "metadata": {}, + "source": [ + "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "id": "bca2f712", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + } + ], + "source": [ + "sc = spark.sparkContext\n", + "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" + ] + }, + { + "cell_type": "markdown", + "id": "fd76c554", + "metadata": {}, + "source": [ + "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "id": "ba954d7d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using ports [7000, 7001, 7002]\n" + ] + } + ], + "source": [ + "model_name = \"HousingModel\"\n", + "ports = find_ports()\n", + "assert len(ports) == 3\n", + "print(f\"Using ports {ports}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "id": "5358d5b6", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 11:> (0 + 1) / 1]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Triton Server PIDs:\n", + " {\n", + " \"cb4ae00-lcedt\": 2964321\n", + "}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", + " ports=ports,\n", + " model_name=model_name,\n", + " model_path=model_path)).collectAsMap()\n", + "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + ] + }, + { + "cell_type": "markdown", + "id": "9a1ac038", + "metadata": {}, + "source": [ + "#### Define client function" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "id": "3812e5ae", + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{ports[0]}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "id": "1ae91c54", + "metadata": {}, + "outputs": [], + "source": [ + "def triton_fn(url, model_name):\n", + " from pytriton.client import ModelClient\n", + "\n", + " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", + "\n", + " def infer_batch(inputs):\n", + " with ModelClient(url, model_name, inference_timeout_s=240) as client:\n", + " result_data = client.infer_batch(inputs)\n", + " return result_data[\"preds\"]\n", + " \n", + " return infer_batch" + ] + }, + { + "cell_type": "markdown", + "id": "20b8514e-01de-481f-86aa-75afd99bcc7c", + "metadata": {}, + "source": [ + "### Run Inference" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "id": "5eae04bc-75ca-421a-87c8-ac507ce1f2f5", + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.parquet(data_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "id": "b350bd8e-9b8f-4511-9ddf-76d917b21b5f", + "metadata": {}, + "outputs": [], + "source": [ + "columns = df.columns" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "id": "d3e64fda-117b-4810-a9a2-dd498239496f", + "metadata": {}, + "outputs": [], + "source": [ + "regress = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", + " input_tensor_shapes=[[8]],\n", + " return_type=FloatType(),\n", + " batch_size=50)" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "id": "a24149a5-3adc-4089-8769-13cf1e44547a", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 13:====================================> (5 + 3) / 8]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 21.1 ms, sys: 8.1 ms, total: 29.2 ms\n", + "Wall time: 1.01 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "%%time\n", + "# first pass caches model/fn\n", + "predictions = df.withColumn(\"preds\", regress(struct(*columns)))\n", + "preds = predictions.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "id": "df2ce39f-30af-491a-8472-800fb1ce8458", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 14:> (0 + 8) / 8]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 23.7 ms, sys: 4.15 ms, total: 27.8 ms\n", + "Wall time: 973 ms\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "%%time\n", + "predictions = df.withColumn(\"preds\", regress(array(*columns)))\n", + "preds = predictions.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "id": "ca6f3eaa-9569-45d0-88bf-9aa0757e1ecb", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 15:> (0 + 8) / 8]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 34.3 ms, sys: 6.47 ms, total: 40.7 ms\n", + "Wall time: 1.49 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "%%time\n", + "predictions = df.withColumn(\"preds\", regress(array(*columns)))\n", + "preds = predictions.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "b79c62c8-e1e8-4467-8aef-8939c31833b8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", + "| MedInc| HouseAge| AveRooms| AveBedrms| Population| AveOccup| Latitude| Longitude| preds|\n", + "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", + "| 0.20909257| -1.1632254| 0.38946992| 0.04609274| -0.9806099| -0.07099328|0.61245227|-0.020113053| 1.4441836|\n", + "|-0.098627955| 0.34647804| 0.27216315| -0.0129226| -0.6953838| -0.05380849| 1.0665938| -1.2479742| 1.7245854|\n", + "| -0.66006273| 1.0616008|-0.55292207| -0.48945764|-0.13641118| 0.028952759| 1.1040496| -1.3827378| 1.3524104|\n", + "| 0.08218294| 0.5848523|-0.13912922| -0.14707813|-0.19116047| -0.07136432|0.96827507| -1.3028787| 2.300915|\n", + "| 0.0784456| -1.4810578| 0.57265776| 0.32067496| 1.0345173|-0.024157424| 1.4411427| -0.52423614| 1.2720771|\n", + "| -0.82318723| -0.36864465| 0.07829511| -0.1808107|-0.67242444|-0.061470542| 1.9374212| -1.0083897| 0.6968852|\n", + "| 0.59671736| 0.5848523| 0.19346413| -0.1371872|-0.19645879| 0.009964322|0.96827507| -1.2928978| 2.6057932|\n", + "| -0.9612035| -1.5605159|-0.56329846| 0.027148023|-0.71127874| -0.08471591| 0.5328614| -0.13990337| 1.0139045|\n", + "| -0.74344087| -1.2426835| 0.27282518| 0.4037246| -0.9841421| -0.05610115| 1.2257773| -0.42940006|0.99310875|\n", + "| 0.9784464| -0.2891866| 0.24374022| -0.24670053| 0.28922042| -0.01102468| 1.1087307| -1.2280084| 2.6450102|\n", + "| -0.5070446| -1.0043093|-0.78254056|0.0122275995| 2.8465424|-0.060435444| 0.8980464| -1.2080427| 2.1324868|\n", + "| -0.18690155| 1.2205169|0.015323491| 0.12183313|-0.41015765| 0.04452552| 1.010412| -1.3228445| 1.945999|\n", + "| -1.2551856| 1.6178073| -0.3341509|-0.060125165| -0.7554314| -0.08777025| 1.0291398| -1.3477987| 1.255805|\n", + "| 4.9607058| -1.9578062| 1.4854684| -0.03948475| 2.1833694|0.0029250523| 1.024457| -1.1581304| 5.8959594|\n", + "| 0.73652315| -1.6399739| 0.7913185| -0.05238397| 1.67738| 0.01944797| 1.0993668| -1.1331724| 2.0677836|\n", + "| -0.505834| 0.18756187|-0.47093546| -0.24297306|-0.60619545| -0.10791535| 0.977639| -1.2879055| 1.7652202|\n", + "| -0.88477343|-0.050812364| -0.6318951| -0.15244243| -0.5258376| -0.15618815| 0.9823201| -1.2879055| 1.5843443|\n", + "| -0.42840376| 0.9821427| -0.2266495| -0.36083496| -0.6883194| -0.08552282| 0.5328614| -0.12493005|0.97526246|\n", + "| 0.9369153| -1.4810578| 0.6722208|-0.121177554| 0.3996021| 0.01291408| 1.1040496| -1.1082181| 2.2553492|\n", + "| -0.80702734| -0.92485124|-0.26602685| -0.1560743| 1.4398388| -0.09314839|0.55627036| -0.09498342| 1.0328484|\n", + "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", + "only showing top 20 rows\n", + "\n" + ] + } + ], + "source": [ + "predictions.show()" + ] + }, + { + "cell_type": "markdown", + "id": "3fec23b0-eaf2-4b6a-aa38-7a09873ed6eb", + "metadata": { + "tags": [] + }, + "source": [ + "#### Stop Triton Server on each executor" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "id": "8084bdef", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/plain": [ + "[True]" + ] + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", + "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "id": "0138a029-87c5-497f-ac5c-3eed0e11b0f6", + "metadata": {}, + "outputs": [], + "source": [ + "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n", + " spark.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d24147e7-5695-44a0-9961-b94bfba1cfff", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "spark-dl-torch", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/image_classification_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/image_classification_torch.ipynb index b6d739a3..e22ad389 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/image_classification_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/image_classification_torch.ipynb @@ -5,12 +5,17 @@ "id": "9e87c927", "metadata": {}, "source": [ + "\n", + "\n", "# PySpark PyTorch Inference\n", "\n", "### Image Classification\n", + "\n", + "In this notebook, we will train an MLP to perform image classification on FashionMNIST, and load it for distributed inference with Spark.\n", + "\n", "Based on: https://pytorch.org/tutorials/beginner/basics/quickstart_tutorial.html \n", "\n", - "Also demonstrates accelerated inference on GPU with Torch-TensorRT. " + "We also demonstrate accelerated inference via Torch-TensorRT model compilation. " ] }, { @@ -21,7 +26,8 @@ "outputs": [], "source": [ "import torch\n", - "\n", + "import os\n", + "import shutil\n", "from torch import nn\n", "from torch.utils.data import DataLoader\n", "from torchvision import datasets\n", @@ -31,6 +37,16 @@ { "cell_type": "code", "execution_count": 2, + "id": "f71f801d", + "metadata": {}, + "outputs": [], + "source": [ + "os.mkdir('models') if not os.path.exists('models') else None" + ] + }, + { + "cell_type": "code", + "execution_count": 3, "id": "d714f40d", "metadata": {}, "outputs": [ @@ -40,7 +56,7 @@ "'2.4.1+cu121'" ] }, - "execution_count": 2, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -49,16 +65,106 @@ "torch.__version__" ] }, + { + "cell_type": "markdown", + "id": "d0f6fb37", + "metadata": {}, + "source": [ + "### Load Dataset" + ] + }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "1c942a46", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz\n", + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz to datasets/data/FashionMNIST/raw/train-images-idx3-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 26421880/26421880 [00:02<00:00, 12502879.39it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting datasets/data/FashionMNIST/raw/train-images-idx3-ubyte.gz to datasets/data/FashionMNIST/raw\n", + "\n", + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz\n", + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz to datasets/data/FashionMNIST/raw/train-labels-idx1-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 29515/29515 [00:00<00:00, 195601.60it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting datasets/data/FashionMNIST/raw/train-labels-idx1-ubyte.gz to datasets/data/FashionMNIST/raw\n", + "\n", + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz\n", + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz to datasets/data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 4422102/4422102 [00:01<00:00, 3727194.18it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting datasets/data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz to datasets/data/FashionMNIST/raw\n", + "\n", + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz\n", + "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz to datasets/data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 5148/5148 [00:00<00:00, 8599074.87it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting datasets/data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz to datasets/data/FashionMNIST/raw\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], "source": [ "# Download training data from open datasets.\n", "training_data = datasets.FashionMNIST(\n", - " root=\"data\",\n", + " root=\"datasets/data\",\n", " train=True,\n", " download=True,\n", " transform=ToTensor(),\n", @@ -66,7 +172,7 @@ "\n", "# Download test data from open datasets.\n", "test_data = datasets.FashionMNIST(\n", - " root=\"data\",\n", + " root=\"datasets/data\",\n", " train=False,\n", " download=True,\n", " transform=ToTensor(),\n", @@ -75,7 +181,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "4a89aa8e-ef62-4aac-8260-4b004f2c1b55", "metadata": {}, "outputs": [], @@ -96,7 +202,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "10a97111", "metadata": {}, "outputs": [ @@ -132,7 +238,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "512d0bc7", "metadata": {}, "outputs": [ @@ -188,7 +294,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "4d4f5538", "metadata": {}, "outputs": [], @@ -199,7 +305,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "92d9076a", "metadata": {}, "outputs": [], @@ -229,7 +335,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "id": "11c5650d", "metadata": {}, "outputs": [], @@ -253,7 +359,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "id": "854608e6", "metadata": {}, "outputs": [ @@ -262,79 +368,85 @@ "output_type": "stream", "text": [ "Epoch 1\n", - "-------------------------------\n", - "loss: 2.301038 [ 64/60000]\n", - "loss: 2.289769 [ 6464/60000]\n", - "loss: 2.268618 [12864/60000]\n", - "loss: 2.264085 [19264/60000]\n", - "loss: 2.244277 [25664/60000]\n", - "loss: 2.209504 [32064/60000]\n", - "loss: 2.220515 [38464/60000]\n", - "loss: 2.185288 [44864/60000]\n", - "loss: 2.186121 [51264/60000]\n", - "loss: 2.149065 [57664/60000]\n", + "-------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loss: 2.303572 [ 64/60000]\n", + "loss: 2.283972 [ 6464/60000]\n", + "loss: 2.266706 [12864/60000]\n", + "loss: 2.268926 [19264/60000]\n", + "loss: 2.241029 [25664/60000]\n", + "loss: 2.226260 [32064/60000]\n", + "loss: 2.230959 [38464/60000]\n", + "loss: 2.197126 [44864/60000]\n", + "loss: 2.198262 [51264/60000]\n", + "loss: 2.169759 [57664/60000]\n", "Test Error: \n", - " Accuracy: 37.8%, Avg loss: 2.151644 \n", + " Accuracy: 48.6%, Avg loss: 2.155098 \n", "\n", "Epoch 2\n", "-------------------------------\n", - "loss: 2.164946 [ 64/60000]\n", - "loss: 2.157853 [ 6464/60000]\n", - "loss: 2.100765 [12864/60000]\n", - "loss: 2.117897 [19264/60000]\n", - "loss: 2.058581 [25664/60000]\n", - "loss: 1.995217 [32064/60000]\n", - "loss: 2.026708 [38464/60000]\n", - "loss: 1.948186 [44864/60000]\n", - "loss: 1.959582 [51264/60000]\n", - "loss: 1.881658 [57664/60000]\n", + "loss: 2.164836 [ 64/60000]\n", + "loss: 2.146720 [ 6464/60000]\n", + "loss: 2.086537 [12864/60000]\n", + "loss: 2.111843 [19264/60000]\n", + "loss: 2.043706 [25664/60000]\n", + "loss: 1.994988 [32064/60000]\n", + "loss: 2.016762 [38464/60000]\n", + "loss: 1.935367 [44864/60000]\n", + "loss: 1.949560 [51264/60000]\n", + "loss: 1.869589 [57664/60000]\n", "Test Error: \n", - " Accuracy: 52.5%, Avg loss: 1.886264 \n", + " Accuracy: 51.8%, Avg loss: 1.868578 \n", "\n", "Epoch 3\n", "-------------------------------\n", - "loss: 1.922469 [ 64/60000]\n", - "loss: 1.893279 [ 6464/60000]\n", - "loss: 1.780482 [12864/60000]\n", - "loss: 1.822908 [19264/60000]\n", - "loss: 1.696129 [25664/60000]\n", - "loss: 1.653140 [32064/60000]\n", - "loss: 1.675662 [38464/60000]\n", - "loss: 1.584822 [44864/60000]\n", - "loss: 1.609127 [51264/60000]\n", - "loss: 1.500899 [57664/60000]\n", + "loss: 1.898998 [ 64/60000]\n", + "loss: 1.865363 [ 6464/60000]\n", + "loss: 1.748555 [12864/60000]\n", + "loss: 1.797765 [19264/60000]\n", + "loss: 1.677429 [25664/60000]\n", + "loss: 1.638624 [32064/60000]\n", + "loss: 1.653928 [38464/60000]\n", + "loss: 1.564719 [44864/60000]\n", + "loss: 1.596518 [51264/60000]\n", + "loss: 1.483102 [57664/60000]\n", "Test Error: \n", - " Accuracy: 60.3%, Avg loss: 1.521902 \n", + " Accuracy: 61.1%, Avg loss: 1.506650 \n", "\n", "Epoch 4\n", "-------------------------------\n", - "loss: 1.593910 [ 64/60000]\n", - "loss: 1.555975 [ 6464/60000]\n", - "loss: 1.412051 [12864/60000]\n", - "loss: 1.480928 [19264/60000]\n", - "loss: 1.348195 [25664/60000]\n", - "loss: 1.352939 [32064/60000]\n", - "loss: 1.361179 [38464/60000]\n", - "loss: 1.298819 [44864/60000]\n", - "loss: 1.325064 [51264/60000]\n", - "loss: 1.226879 [57664/60000]\n", + "loss: 1.572357 [ 64/60000]\n", + "loss: 1.540674 [ 6464/60000]\n", + "loss: 1.396118 [12864/60000]\n", + "loss: 1.464891 [19264/60000]\n", + "loss: 1.345048 [25664/60000]\n", + "loss: 1.346302 [32064/60000]\n", + "loss: 1.351714 [38464/60000]\n", + "loss: 1.288438 [44864/60000]\n", + "loss: 1.324335 [51264/60000]\n", + "loss: 1.220271 [57664/60000]\n", "Test Error: \n", - " Accuracy: 63.2%, Avg loss: 1.254962 \n", + " Accuracy: 63.2%, Avg loss: 1.246688 \n", "\n", "Epoch 5\n", "-------------------------------\n", - "loss: 1.337471 [ 64/60000]\n", - "loss: 1.314826 [ 6464/60000]\n", - "loss: 1.155245 [12864/60000]\n", - "loss: 1.257553 [19264/60000]\n", - "loss: 1.123370 [25664/60000]\n", - "loss: 1.155071 [32064/60000]\n", - "loss: 1.168100 [38464/60000]\n", - "loss: 1.119365 [44864/60000]\n", - "loss: 1.149572 [51264/60000]\n", - "loss: 1.067573 [57664/60000]\n", + "loss: 1.320306 [ 64/60000]\n", + "loss: 1.310512 [ 6464/60000]\n", + "loss: 1.143846 [12864/60000]\n", + "loss: 1.247822 [19264/60000]\n", + "loss: 1.120661 [25664/60000]\n", + "loss: 1.147298 [32064/60000]\n", + "loss: 1.163092 [38464/60000]\n", + "loss: 1.110569 [44864/60000]\n", + "loss: 1.149649 [51264/60000]\n", + "loss: 1.061627 [57664/60000]\n", "Test Error: \n", - " Accuracy: 64.4%, Avg loss: 1.090368 \n", + " Accuracy: 64.8%, Avg loss: 1.082042 \n", "\n", "Done!\n" ] @@ -360,7 +472,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "id": "5d5d24de", "metadata": {}, "outputs": [ @@ -368,13 +480,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Saved PyTorch Model State to model.pt\n" + "Saved PyTorch Model State to models/model.pt\n" ] } ], "source": [ - "torch.save(model.state_dict(), \"model.pt\")\n", - "print(\"Saved PyTorch Model State to model.pt\")" + "torch.save(model.state_dict(), \"models/model.pt\")\n", + "print(\"Saved PyTorch Model State to models/model.pt\")" ] }, { @@ -388,7 +500,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "6d9b3a45-7618-43e4-8bd3-8bb317a484d3", "metadata": {}, "outputs": [ @@ -396,14 +508,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "Saved TorchScript Model to ts_model.pt\n" + "Saved TorchScript Model to models/ts_model.pt\n" ] } ], "source": [ "scripted = torch.jit.script(model)\n", - "scripted.save(\"ts_model.pt\")\n", - "print(\"Saved TorchScript Model to ts_model.pt\")" + "scripted.save(\"models/ts_model.pt\")\n", + "print(\"Saved TorchScript Model to models/ts_model.pt\")" ] }, { @@ -416,7 +528,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "id": "8fe3b5d1", "metadata": {}, "outputs": [ @@ -426,19 +538,19 @@ "" ] }, - "execution_count": 13, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model_from_state = NeuralNetwork().to(device)\n", - "model_from_state.load_state_dict(torch.load(\"model.pt\", weights_only=True))" + "model_from_state.load_state_dict(torch.load(\"models/model.pt\", weights_only=True))" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "id": "0c405bd0", "metadata": {}, "outputs": [ @@ -470,18 +582,17 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "id": "ef3c419e-d384-446c-b07b-1af93e07d6c0", "metadata": {}, "outputs": [], "source": [ - "# Load model to original device (GPU) and move to CPU. \n", - "ts_model = torch.jit.load(\"ts_model.pt\")" + "ts_model = torch.jit.load(\"models/ts_model.pt\")" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "id": "c92d6cdb", "metadata": {}, "outputs": [], @@ -491,7 +602,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "038af830-a360-45eb-ab4e-b1adab0af164", "metadata": {}, "outputs": [ @@ -518,7 +629,7 @@ "### Compile using the Torch JIT Compiler\n", "This leverages the [Torch-TensorRT inference compiler](https://pytorch.org/TensorRT/) for accelerated inference on GPUs using the `torch.compile` JIT interface under the hood. The compiler stack returns a [boxed-function](http://blog.ezyang.com/2020/09/lets-talk-about-the-pytorch-dispatcher/) that triggers compilation on the first call. \n", "\n", - "Modules compiled in this fashion are [not serializable with pickle](https://github.com/pytorch/pytorch/issues/101107#issuecomment-1542688089), so we cannot send the compiled model directly to Spark. Instead, we will recompile and cache the model on the executor. " + "Modules compiled in this fashion are [not serializable with pickle](https://github.com/pytorch/pytorch/issues/101107#issuecomment-1542688089), so we cannot send the compiled model directly to Spark. " ] }, { @@ -531,10 +642,18 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 19, "id": "362b266b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:torch_tensorrt.dynamo.conversion.aten_ops_converters:Unable to import quantization op. Please install modelopt library (https://github.com/NVIDIA/TensorRT-Model-Optimizer?tab=readme-ov-file#installation) to add support for compiling quantized models\n" + ] + } + ], "source": [ "import torch_tensorrt as trt\n", "import time" @@ -542,7 +661,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "id": "f0ac1362", "metadata": {}, "outputs": [], @@ -556,9 +675,28 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "id": "f3e3bdc4", "metadata": {}, + "outputs": [], + "source": [ + "inputs_bs1 = torch.randn((10, 784), dtype=torch.float).to(\"cuda\")\n", + "# This indicates dimension 0 of inputs_bs1 is dynamic whose range of values is [1, 50]. \n", + "torch._dynamo.mark_dynamic(inputs_bs1, 0, min=1, max=64)\n", + "trt_model = trt.compile(\n", + " model,\n", + " ir=\"torch_compile\",\n", + " inputs=inputs_bs1,\n", + " enabled_precisions={torch.float},\n", + " timing_cache_path=timing_cache,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "66f61302", + "metadata": {}, "outputs": [ { "name": "stderr", @@ -566,30 +704,36 @@ "text": [ "INFO:torch_tensorrt.dynamo.utils:Using Default Torch-TRT Runtime (as requested by user)\n", "INFO:torch_tensorrt.dynamo.utils:Device not specified, using Torch default current device - cuda:0. If this is incorrect, please specify an input device, via the device keyword.\n", - "INFO:torch_tensorrt.dynamo.utils:Compilation Settings: CompilationSettings(enabled_precisions={}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, refit=False, engine_capability=, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/timing_cache-1729187850.4862776.bin')\n", - "\n", + "INFO:torch_tensorrt.dynamo.utils:Compilation Settings: CompilationSettings(enabled_precisions={}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, refit=False, engine_capability=, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/timing_cache-1738007742.9371898.bin')\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ "WARNING:torch_tensorrt.dynamo._compiler:Node _param_constant1 of op type get_attr does not have metadata. This could sometimes lead to undefined behavior.\n", "WARNING:torch_tensorrt.dynamo._compiler:Some nodes do not have metadata (shape and dtype information). This could lead to problems sometimes if the graph has PyTorch and TensorRT segments.\n", "INFO:torch_tensorrt.dynamo._compiler:Partitioning the graph via the fast partitioner\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init CUDA: CPU +2, GPU +0, now: CPU 457, GPU 713 (MiB)\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init builder kernel library: CPU +1634, GPU +288, now: CPU 2238, GPU 1001 (MiB)\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init CUDA: CPU +2, GPU +0, now: CPU 457, GPU 2889 (MiB)\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init builder kernel library: CPU +1634, GPU +288, now: CPU 2238, GPU 3177 (MiB)\n", "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/torch_tensorrt/dynamo/conversion/impl/activation/base.py:40: DeprecationWarning: Use Deprecated in TensorRT 10.1. Superseded by explicit quantization. instead.\n", " if input_val.dynamic_range is not None and dyn_range_fn is not None:\n", "\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT INetwork construction elapsed time: 0:00:00.005662\n", + "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT INetwork construction elapsed time: 0:00:00.005228\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:Global timing cache in use. Profiling results in this builder pass will be stored.\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:Detected 1 inputs and 1 output network tensors.\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Host Persistent Memory: 21984\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Device Persistent Memory: 0\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Scratch Memory: 0\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Started assigning block shifts. This will take 4 steps to complete.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Algorithm ShiftNTopDown took 0.115746ms to assign 2 blocks to 4 nodes requiring 4096 bytes.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Algorithm ShiftNTopDown took 0.127126ms to assign 2 blocks to 4 nodes requiring 4096 bytes.\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Activation Memory: 4096\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Weights Memory: 2678824\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Engine generation completed in 1.58824 seconds.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Engine generation completed in 1.59804 seconds.\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage of TRT CPU/GPU memory allocators: CPU 1 MiB, GPU 5 MiB\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage during Engine building and serialization: CPU: 3950 MiB\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:Build TRT engine elapsed time: 0:00:01.591865\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage during Engine building and serialization: CPU: 3951 MiB\n", + "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:Build TRT engine elapsed time: 0:00:01.601778\n", "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT Engine uses: 2832188 bytes of Memory\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 26 bytes of code generator cache.\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 43 timing cache entries\n" @@ -604,17 +748,6 @@ } ], "source": [ - "inputs_bs1 = torch.randn((1, 784), dtype=torch.float).to(\"cuda\")\n", - "# This indicates dimension 0 of inputs_bs1 is dynamic whose range of values is [1, 50]. No recompilation will happen when the batch size changes.\n", - "torch._dynamo.mark_dynamic(inputs_bs1, 0, min=1, max=64)\n", - "trt_model = trt.compile(\n", - " model,\n", - " ir=\"torch_compile\",\n", - " inputs=inputs_bs1,\n", - " enabled_precisions={torch.float},\n", - " timing_cache_path=timing_cache,\n", - ")\n", - "\n", "stream = torch.cuda.Stream()\n", "with torch.no_grad(), torch.cuda.stream(stream):\n", " pred = trt_model(torch.flatten(x.to(device), start_dim=1, end_dim=-1))\n", @@ -635,41 +768,79 @@ }, { "cell_type": "code", - "execution_count": 21, - "id": "6b8f1b45", + "execution_count": 23, + "id": "3e7e7689", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "INFO:torch_tensorrt.dynamo._compiler:Compilation Settings: CompilationSettings(enabled_precisions={}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=True, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, refit=False, engine_capability=, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/timing_cache-1729187850.4862776.bin')\n", + "INFO:torch_tensorrt.dynamo._compiler:Compilation Settings: CompilationSettings(enabled_precisions={}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, refit=False, engine_capability=, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/timing_cache-1738007742.9371898.bin')\n", "\n", "INFO:torch_tensorrt.dynamo._compiler:Partitioning the graph via the fast partitioner\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init CUDA: CPU +0, GPU +0, now: CPU 758, GPU 715 (MiB)\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init builder kernel library: CPU +1633, GPU +286, now: CPU 2391, GPU 1001 (MiB)\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init CUDA: CPU +0, GPU +0, now: CPU 759, GPU 2891 (MiB)\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init builder kernel library: CPU +1633, GPU +286, now: CPU 2392, GPU 3177 (MiB)\n", "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/torch_tensorrt/dynamo/conversion/impl/activation/base.py:40: DeprecationWarning: Use Deprecated in TensorRT 10.1. Superseded by explicit quantization. instead.\n", " if input_val.dynamic_range is not None and dyn_range_fn is not None:\n", "\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT INetwork construction elapsed time: 0:00:00.004664\n", + "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT INetwork construction elapsed time: 0:00:00.004842\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:Global timing cache in use. Profiling results in this builder pass will be stored.\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:Detected 1 inputs and 1 output network tensors.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Host Persistent Memory: 21984\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Host Persistent Memory: 22944\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Device Persistent Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Scratch Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Started assigning block shifts. This will take 4 steps to complete.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Algorithm ShiftNTopDown took 0.113766ms to assign 2 blocks to 4 nodes requiring 4096 bytes.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Activation Memory: 4096\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Scratch Memory: 25088\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Started assigning block shifts. This will take 7 steps to complete.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Algorithm ShiftNTopDown took 0.253198ms to assign 4 blocks to 7 nodes requiring 263168 bytes.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Activation Memory: 262144\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Weights Memory: 2678824\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Engine generation completed in 0.022595 seconds.\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:Engine generation completed in 1.20766 seconds.\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage of TRT CPU/GPU memory allocators: CPU 1 MiB, GPU 5 MiB\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage during Engine building and serialization: CPU: 3968 MiB\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:Build TRT engine elapsed time: 0:00:00.025016\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT Engine uses: 2833124 bytes of Memory\n", + "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage during Engine building and serialization: CPU: 3978 MiB\n", + "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:Build TRT engine elapsed time: 0:00:01.210199\n", + "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT Engine uses: 2885916 bytes of Memory\n", "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 26 bytes of code generator cache.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 43 timing cache entries\n" + "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 76 timing cache entries\n" ] - }, + } + ], + "source": [ + "example_inputs = (torch.randn((10, 784), dtype=torch.float).to(\"cuda\"),)\n", + "\n", + "# Mark dim 1 (batch size) as dynamic\n", + "batch = torch.export.Dim(\"batch\", min=1, max=64)\n", + "# Produce traced graph in ExportedProgram format\n", + "exp_program = torch.export.export(model_from_state, args=example_inputs, dynamic_shapes={\"x\": {0: batch}})\n", + "# Compile the traced graph to produce an optimized module\n", + "trt_gm = trt.dynamo.compile(exp_program, tuple(example_inputs), enabled_precisions={torch.float}, timing_cache_path=timing_cache)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "6fda0c0e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + ".GraphModuleImpl'>\n" + ] + } + ], + "source": [ + "print(type(exp_program))\n", + "print(type(trt_gm))" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "5ed9e4c5", + "metadata": {}, + "outputs": [ { "name": "stdout", "output_type": "stream", @@ -679,16 +850,6 @@ } ], "source": [ - "# Preparing the inputs for batch_size = 1. \n", - "inputs = (torch.randn((1, 784), dtype=torch.float).cuda(),)\n", - "\n", - "# Produce traced graph in the ExportedProgram format\n", - "exp_program = trt.dynamo.trace(model_from_state, inputs)\n", - "# Compile the traced graph to produce an optimized module\n", - "trt_gm = trt.dynamo.compile(exp_program, \n", - " inputs=inputs, \n", - " timing_cache_path=timing_cache)\n", - "\n", "stream = torch.cuda.Stream()\n", "with torch.no_grad(), torch.cuda.stream(stream):\n", " trt_gm(torch.flatten(x.to(device), start_dim=1, end_dim=-1))\n", @@ -698,86 +859,226 @@ }, { "cell_type": "markdown", - "id": "6f2bbfe1", + "id": "38697a06", "metadata": {}, "source": [ - "We can save the compiled model using `torch_tensorrt.save`. Unfortunately, serializing the model to be reloaded at a later date currently only supports *static inputs*." + "We can run the optimized module with a few different batch sizes (without recompilation!):" ] }, { "cell_type": "code", - "execution_count": 23, - "id": "d87e4b20", + "execution_count": null, + "id": "27871156", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Saved AOT compiled TensorRT model to trt_model_aot.ep\n" + "Output shapes:\n", + "torch.Size([10, 10])\n", + "torch.Size([1, 10])\n", + "torch.Size([50, 10])\n" ] } ], "source": [ - "with torch.cuda.stream(stream):\n", - " trt.save(trt_gm, \"trt_model_aot.ep\", inputs=[torch.randn((1, 784), dtype=torch.float).to(\"cuda\")])\n", - " print(\"Saved AOT compiled TensorRT model to trt_model_aot.ep\")" + "inputs = (torch.randn((10, 784), dtype=torch.float).cuda(),)\n", + "inputs_bs1 = (torch.randn((1, 784), dtype=torch.float).cuda(),)\n", + "inputs_bs50 = (torch.randn((50, 784), dtype=torch.float).cuda(),)\n", + "\n", + "stream = torch.cuda.Stream()\n", + "with torch.no_grad(), torch.cuda.stream(stream):\n", + " print(\"Output shapes:\")\n", + " print(trt_gm(*inputs).shape)\n", + " print(trt_gm(*inputs_bs1).shape)\n", + " print(trt_gm(*inputs_bs50).shape)" ] }, { "cell_type": "markdown", - "id": "ad918393", + "id": "ab974244", "metadata": {}, "source": [ - "## PySpark" + "We can serialize the ExportedProgram (a traced graph representing the model's forward function) using `torch.export.save` to be recompiled at a later date." ] }, { - "cell_type": "markdown", - "id": "fd1daec3", + "cell_type": "code", + "execution_count": null, + "id": "d87e4b20", "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved ExportedProgram to models/trt_model.ep\n" + ] + } + ], "source": [ - "### Convert numpy dataset to Pandas DataFrame" + "torch.export.save(exp_program, \"models/trt_model.ep\")\n", + "print(\"Saved ExportedProgram to models/trt_model.ep\")" ] }, { - "cell_type": "code", - "execution_count": 21, - "id": "42c5feba", + "cell_type": "markdown", + "id": "ad918393", "metadata": {}, - "outputs": [], "source": [ - "import pandas as pd\n", - "from pyspark import SparkConf\n", - "from pyspark.sql import SparkSession\n", - "from pyspark.sql.types import StructType, StructField, ArrayType, FloatType" + "## PySpark" ] }, { "cell_type": "code", - "execution_count": 22, - "id": "f063cbe7", + "execution_count": 28, + "id": "42c5feba", "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "((10000, 28, 28), dtype('uint8'))" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/pyspark/broadcast.py:38: DeprecationWarning: typing.io is deprecated, import directly from typing instead. typing.io will be removed in Python 3.12.\n", + " from typing.io import BinaryIO # type: ignore[import]\n", + "\n" + ] } ], "source": [ - "data = test_data.data.numpy()\n", - "data.shape, data.dtype" + "from pyspark.sql.functions import col, struct, pandas_udf, array\n", + "from pyspark.ml.functions import predict_batch_udf\n", + "from pyspark.sql.types import *\n", + "from pyspark.sql import SparkSession\n", + "from pyspark import SparkConf" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 29, + "id": "ef97321d", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os" + ] + }, + { + "cell_type": "markdown", + "id": "ece094d6", + "metadata": {}, + "source": [ + "Check the cluster environment to handle any platform-specific Spark configurations." + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "10eb841f", + "metadata": {}, + "outputs": [], + "source": [ + "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n", + "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n", + "on_standalone = not (on_databricks or on_dataproc)" + ] + }, + { + "cell_type": "markdown", + "id": "425e94ac", + "metadata": {}, + "source": [ + "#### Create Spark Session\n", + "\n", + "For local standalone clusters, we'll connect to the cluster and create the Spark Session. \n", + "For CSP environments, Spark will either be preconfigured (Databricks) or we'll need to create the Spark Session (Dataproc)." + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "60ba6e74", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/27 19:55:48 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/01/27 19:55:48 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", + "25/01/27 19:55:48 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "25/01/27 19:55:49 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + ] + } + ], + "source": [ + "conf = SparkConf()\n", + "\n", + "if 'spark' not in globals():\n", + " if on_standalone:\n", + " import socket\n", + " conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + " hostname = socket.gethostname()\n", + " conf.setMaster(f\"spark://{hostname}:7077\")\n", + " conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", + " conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_env}/lib:{conda_env}/lib/python3.11/site-packages/nvidia_pytriton.libs:$LD_LIBRARY_PATH\")\n", + " elif on_dataproc:\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conda_lib_path=\"/opt/conda/miniconda3/lib\"\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_lib_path}:$LD_LIBRARY_PATH\") \n", + " conf.set(\"spark.executor.instances\", \"4\") # dataproc defaults to 2\n", + "\n", + " conf.set(\"spark.executor.cores\", \"8\")\n", + " conf.set(\"spark.task.resource.gpu.amount\", \"0.125\")\n", + " conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n", + " conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", + " conf.set(\"spark.python.worker.reuse\", \"true\")\n", + " \n", + "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", + "sc = spark.sparkContext" + ] + }, + { + "cell_type": "markdown", + "id": "2cd11476", + "metadata": {}, + "source": [ + "#### Create Spark DataFrame from Pandas DataFrame" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "f063cbe7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "((10000, 28, 28), dtype('uint8'))" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data = test_data.data.numpy()\n", + "data.shape, data.dtype" + ] + }, + { + "cell_type": "code", + "execution_count": null, "id": "8c828393", "metadata": {}, "outputs": [ @@ -787,7 +1088,7 @@ "((10000, 784), dtype('float64'))" ] }, - "execution_count": 23, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -799,7 +1100,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "7760bdbe", "metadata": {}, "outputs": [ @@ -1160,7 +1461,7 @@ "[10000 rows x 784 columns]" ] }, - "execution_count": 24, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -1172,18 +1473,10 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 35, "id": "f7d2bc0d", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 87.6 ms, sys: 56.2 ms, total: 144 ms\n", - "Wall time: 143 ms\n" - ] - }, { "data": { "text/html": [ @@ -1275,72 +1568,29 @@ "[10000 rows x 1 columns]" ] }, - "execution_count": 25, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "%%time\n", "# 1 column of array\n", "pdf1 = pd.DataFrame()\n", "pdf1['data'] = pdf784.values.tolist()\n", "pdf1" ] }, - { - "cell_type": "code", - "execution_count": 26, - "id": "a5d7ccf1", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "24/10/08 00:30:18 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "24/10/08 00:30:18 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", - "Setting default log level to \"WARN\".\n", - "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "24/10/08 00:30:18 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" - ] - } - ], - "source": [ - "import os\n", - "conda_env = os.environ.get(\"CONDA_PREFIX\")\n", - "\n", - "conf = SparkConf()\n", - "if 'spark' not in globals():\n", - " # If Spark is not already started with Jupyter, attach to Spark Standalone\n", - " import socket\n", - " hostname = socket.gethostname()\n", - " conf.setMaster(f\"spark://{hostname}:7077\") # assuming Master is on default port 7077\n", - "conf.set(\"spark.task.maxFailures\", \"1\")\n", - "conf.set(\"spark.driver.memory\", \"8g\")\n", - "conf.set(\"spark.executor.memory\", \"8g\")\n", - "conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled\", \"false\")\n", - "conf.set(\"spark.sql.pyspark.jvmStacktrace.enabled\", \"true\")\n", - "conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", - "conf.set(\"spark.python.worker.reuse\", \"true\")\n", - "# Create Spark Session\n", - "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", - "sc = spark.sparkContext" - ] - }, { "cell_type": "markdown", - "id": "320760db", + "id": "07b2a70b", "metadata": {}, "source": [ - "#### Create Spark DataFrame from Pandas DataFrame" + "Create dataframes with a single column of 784 floats and 784 separate columns." ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 36, "id": "4863d5ff", "metadata": {}, "outputs": [ @@ -1348,8 +1598,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/pyspark/sql/pandas/serializers.py:224: DeprecationWarning: is_categorical_dtype is deprecated and will be removed in a future version. Use isinstance(dtype, pd.CategoricalDtype) instead\n", - " if is_categorical_dtype(series.dtype):\n", + "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/pyspark/sql/pandas/serializers.py:229: DeprecationWarning: is_categorical_dtype is deprecated and will be removed in a future version. Use isinstance(dtype, pd.CategoricalDtype) instead\n", + " elif is_categorical_dtype(s.dtype):\n", "\n" ] }, @@ -1357,42 +1607,32 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 389 ms, sys: 63.2 ms, total: 452 ms\n", - "Wall time: 1.44 s\n" + "CPU times: user 151 ms, sys: 36.7 ms, total: 187 ms\n", + "Wall time: 1.49 s\n" ] - } - ], - "source": [ - "%%time\n", - "# force FloatType since Spark defaults to DoubleType\n", - "schema = StructType([StructField(\"data\",ArrayType(FloatType()), True)])\n", - "df = spark.createDataFrame(pdf1, schema).repartition(8)" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "406edba5", - "metadata": {}, - "outputs": [ + }, { "data": { "text/plain": [ "StructType([StructField('data', ArrayType(FloatType(), True), True)])" ] }, - "execution_count": 28, + "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "%%time\n", + "# force FloatType since Spark defaults to DoubleType\n", + "schema = StructType([StructField(\"data\",ArrayType(FloatType()), True)])\n", + "df = spark.createDataFrame(pdf1, schema).repartition(8)\n", "df.schema" ] }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 37, "id": "831f4a01-3a49-4114-b9a0-2ae54526d72d", "metadata": {}, "outputs": [ @@ -1400,29 +1640,32 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 61 ms, sys: 21.6 ms, total: 82.6 ms\n", - "Wall time: 854 ms\n" + "CPU times: user 199 ms, sys: 28 ms, total: 227 ms\n", + "Wall time: 995 ms\n" ] + }, + { + "data": { + "text/plain": [ + "StructType([StructField('data', ArrayType(FloatType(), True), True)])" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ "%%time\n", "# force FloatType since Spark defaults to DoubleType\n", "schema = StructType([StructField(str(x), FloatType(), True) for x in range(784)])\n", - "df784 = spark.createDataFrame(pdf784, schema).repartition(8)" - ] - }, - { - "cell_type": "markdown", - "id": "ac4c7448", - "metadata": {}, - "source": [ - "### Save the test dataset as parquet files" + "df784 = spark.createDataFrame(pdf784, schema).repartition(8)\n", + "df.schema" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 38, "id": "e8ebae46", "metadata": {}, "outputs": [ @@ -1430,7 +1673,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "24/10/08 00:30:21 WARN TaskSetManager: Stage 0 contains a task of very large size (4032 KiB). The maximum recommended task size is 1000 KiB.\n", + "25/01/27 19:55:52 WARN TaskSetManager: Stage 0 contains a task of very large size (4030 KiB). The maximum recommended task size is 1000 KiB.\n", "[Stage 0:> (0 + 8) / 8]\r" ] }, @@ -1438,8 +1681,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.09 ms, sys: 2.57 ms, total: 4.66 ms\n", - "Wall time: 1.78 s\n" + "CPU times: user 0 ns, sys: 4.56 ms, total: 4.56 ms\n", + "Wall time: 1.74 s\n" ] }, { @@ -1452,12 +1695,17 @@ ], "source": [ "%%time\n", - "df.write.mode(\"overwrite\").parquet(\"fashion_mnist_1\")" + "data_path_1 = \"spark-dl-datasets/fashion_mnist_1\"\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " data_path_1 = \"dbfs:/FileStore/\" + data_path_1\n", + "\n", + "df.write.mode(\"overwrite\").parquet(data_path_1)" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 39, "id": "922314ce-2996-4666-9fc9-bcd98d16bb56", "metadata": {}, "outputs": [ @@ -1465,50 +1713,40 @@ "name": "stderr", "output_type": "stream", "text": [ - "24/10/08 00:30:23 WARN SparkStringUtils: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n", - "24/10/08 00:30:23 WARN TaskSetManager: Stage 3 contains a task of very large size (7849 KiB). The maximum recommended task size is 1000 KiB.\n" + "25/01/27 19:55:53 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n", + "25/01/27 19:55:54 WARN TaskSetManager: Stage 3 contains a task of very large size (7847 KiB). The maximum recommended task size is 1000 KiB.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.01 ms, sys: 22 μs, total: 3.03 ms\n", - "Wall time: 734 ms\n" + "CPU times: user 2.32 ms, sys: 287 μs, total: 2.61 ms\n", + "Wall time: 864 ms\n" ] } ], "source": [ "%%time\n", - "df784.write.mode(\"overwrite\").parquet(\"fashion_mnist_784\")" - ] - }, - { - "cell_type": "markdown", - "id": "8688429e", - "metadata": {}, - "source": [ - "### Check arrow memory configuration" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "088cb37f", - "metadata": {}, - "outputs": [], - "source": [ - "spark.conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"128\")\n", - "# This line will fail if the vectorized reader runs out of memory\n", - "assert len(df.head()) > 0, \"`df` should not be empty\"" + "data_path_784 = \"spark-dl-datasets/fashion_mnist_784\"\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " data_path_784 = \"dbfs:/FileStore/\" + data_path_784\n", + "\n", + "df784.write.mode(\"overwrite\").parquet(data_path_784)" ] }, { "cell_type": "markdown", - "id": "d7c77eb4-7bd6-40c7-9a35-ee899a66ece3", + "id": "fce89cb0", "metadata": {}, "source": [ - "## Inference using Spark DL API" + "## Inference using Spark DL API\n", + "\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "\n", + "- predict_batch_fn uses PyTorch APIs to load the model and return a predict function which operates on numpy arrays \n", + "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" ] }, { @@ -1518,27 +1756,12 @@ "tags": [] }, "source": [ - "### 1 columns of 784 float" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "id": "133cc9a5-64c6-4820-807e-b87cf7e0b75a", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import numpy as np\n", - "\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import struct, col, array\n", - "from pyspark.sql.types import ArrayType, FloatType, Union, Dict" + "### 1 column of 784 float" ] }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 40, "id": "79b151d9-d112-43b6-a479-887e2fd0e2b1", "metadata": {}, "outputs": [ @@ -1548,59 +1771,58 @@ "1" ] }, - "execution_count": 34, + "execution_count": 40, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df = spark.read.parquet(\"fashion_mnist_1\")\n", + "df = spark.read.parquet(data_path_1)\n", "len(df.columns)" ] }, { "cell_type": "code", - "execution_count": 35, - "id": "cabcd546-2e8e-40d0-8b79-7598a7a83aae", + "execution_count": 41, + "id": "3e6a4dbb", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "StructType([StructField('data', ArrayType(FloatType(), True), True)])" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "df.schema" + "# A resource warning may occur due to unclosed file descriptors used by TensorRT across multiple PySpark daemon processes.\n", + "# These can be safely ignored as the resources will be cleaned up when the worker processes terminate.\n", + "\n", + "import warnings\n", + "warnings.simplefilter(\"ignore\", ResourceWarning)" ] }, { "cell_type": "code", - "execution_count": 36, - "id": "823c3825", + "execution_count": 42, + "id": "16e523c2", "metadata": {}, "outputs": [], "source": [ - "# Get absolute path to model\n", - "model_path = \"{}/model.pt\".format(os.getcwd())" - ] - }, - { - "cell_type": "markdown", - "id": "2d7c2bac", - "metadata": {}, - "source": [ - "For inference on Spark, we'll compile the model with the Torch-TensorRT AOT compiler and cache on the executor. We can specify dynamic batch sizes before compilation to [optimize across multiple input shapes](https://pytorch.org/TensorRT/user_guide/dynamic_shapes.html)." + "# get absolute path to model\n", + "model_path = \"{}/models/trt_model.ep\".format(os.getcwd())\n", + "\n", + "# For cloud environments, copy the model to the distributed file system.\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-models\")\n", + " dbfs_model_path = \"/dbfs/FileStore/spark-dl-models/model.pt\"\n", + " shutil.copy(model_path, dbfs_model_path)\n", + " model_path = dbfs_model_path\n", + "elif on_dataproc:\n", + " # GCS is mounted at /mnt/gcs by the init script\n", + " models_dir = \"/mnt/gcs/spark-dl/models\"\n", + " os.mkdir(models_dir) if not os.path.exists(models_dir) else None\n", + " gcs_model_path = models_dir + \"/trt_model.ep\"\n", + " shutil.copy(model_path, gcs_model_path)\n", + " model_path = gcs_model_path" ] }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 43, "id": "73dc73cb-25e3-4798-a019-e1abd684eaa1", "metadata": {}, "outputs": [], @@ -1613,49 +1835,27 @@ " if device != \"cuda\":\n", " raise ValueError(\"This function uses the TensorRT model which requires a GPU device\")\n", "\n", - " # Define model\n", - " class NeuralNetwork(nn.Module):\n", - " def __init__(self):\n", - " super(NeuralNetwork, self).__init__()\n", - " self.linear_relu_stack = nn.Sequential(\n", - " nn.Linear(28*28, 512),\n", - " nn.ReLU(),\n", - " nn.Linear(512, 512),\n", - " nn.ReLU(),\n", - " nn.Linear(512, 10)\n", - " )\n", - "\n", - " def forward(self, x):\n", - " logits = self.linear_relu_stack(x)\n", - " return logits\n", - "\n", - " model = NeuralNetwork().to(device)\n", - " model.load_state_dict(torch.load(model_path, weights_only=True))\n", - "\n", - " # Preparing the inputs for dynamic batch sizing.\n", - " inputs = [trt.Input(min_shape=(1, 784), \n", - " opt_shape=(50, 784), \n", - " max_shape=(64, 784), \n", - " dtype=torch.float32)]\n", - "\n", - " # Trace the computation graph and compile to produce an optimized module\n", - " trt_gm = trt.compile(model, ir=\"dynamo\", inputs=inputs, require_full_compilation=True)\n", + " example_inputs = (torch.randn((50, 784), dtype=torch.float).to(\"cuda\"),)\n", + " exp_program = torch.export.load(model_path)\n", + " trt_gm = trt.dynamo.compile(exp_program,\n", + " tuple(example_inputs),\n", + " enabled_precisions={torch.float},\n", + " workspace_size=1<<30)\n", "\n", " def predict(inputs: np.ndarray):\n", - " print(\"Predicting on process PID: {}\".format(os.getpid()))\n", " stream = torch.cuda.Stream()\n", " with torch.no_grad(), torch.cuda.stream(stream):\n", " # use array to combine columns into tensors\n", " torch_inputs = torch.from_numpy(inputs).to(device)\n", " outputs = trt_gm(torch_inputs)\n", " return outputs.detach().cpu().numpy()\n", - " \n", + "\n", " return predict" ] }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 44, "id": "df68cca1-2d47-4e88-8aad-9899402aee97", "metadata": {}, "outputs": [], @@ -1668,7 +1868,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 45, "id": "63555b3b-3673-4712-97aa-fd728c6c4979", "metadata": {}, "outputs": [ @@ -1683,20 +1883,20 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 189 ms, sys: 53.6 ms, total: 242 ms\n", - "Wall time: 9.73 s\n" + "CPU times: user 543 ms, sys: 1.13 s, total: 1.67 s\n", + "Wall time: 22.7 s\n" ] } ], "source": [ "%%time\n", - "# first pass caches model/fn\n", + "# first pass compiles and caches model/fn\n", "preds = df.withColumn(\"preds\", mnist(struct(df.columns))).collect()" ] }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 46, "id": "5dbf058a-70d6-4199-af9d-13843d078950", "metadata": {}, "outputs": [ @@ -1704,8 +1904,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 195 ms, sys: 75 ms, total: 270 ms\n", - "Wall time: 1.26 s\n" + "CPU times: user 211 ms, sys: 65.9 ms, total: 277 ms\n", + "Wall time: 715 ms\n" ] } ], @@ -1716,7 +1916,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 47, "id": "3f5ed801-6ca5-43a0-bf9c-2535a0dfe2e8", "metadata": {}, "outputs": [ @@ -1724,8 +1924,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 199 ms, sys: 54.9 ms, total: 254 ms\n", - "Wall time: 1.27 s\n" + "CPU times: user 192 ms, sys: 54.9 ms, total: 247 ms\n", + "Wall time: 628 ms\n" ] } ], @@ -1746,7 +1946,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 48, "id": "f1f1e5fd-5866-4b78-b9d3-709e6b383a0c", "metadata": {}, "outputs": [], @@ -1757,7 +1957,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 49, "id": "76b76502-adb7-45ec-a365-2e61cdd576fc", "metadata": {}, "outputs": [], @@ -1768,7 +1968,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 50, "id": "c163953a-1504-444f-b39f-86b61d34e440", "metadata": {}, "outputs": [], @@ -1778,7 +1978,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 51, "id": "bc0fad05-50ab-4ae5-b9fd-e50133c4c92a", "metadata": {}, "outputs": [ @@ -1801,7 +2001,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 52, "id": "56f36efb-e3a2-49f9-b9fb-1657bc25e5c5", "metadata": {}, "outputs": [ @@ -1809,7 +2009,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[-1.309907078742981, -3.8460376262664795, 0.845407247543335, -2.5534284114837646, 0.7116107940673828, 0.9341840147972107, 0.4921048879623413, 0.22850888967514038, 2.951157331466675, 1.0042279958724976]\n", + "[-1.3527451753616333, -3.264960765838623, 0.6525070071220398, -2.173452138900757, 0.7591032385826111, 1.0686758756637573, 0.4610092043876648, 0.4507816433906555, 3.1264138221740723, 1.1761378049850464]\n", "predicted label: Bag\n" ] } @@ -1829,7 +2029,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 53, "id": "e0ab0af6-b5c9-4b74-9dd6-baa7737cc986", "metadata": {}, "outputs": [ @@ -1839,19 +2039,19 @@ "784" ] }, - "execution_count": 48, + "execution_count": 53, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df = spark.read.parquet(\"fashion_mnist_784\")\n", + "df = spark.read.parquet(data_path_784)\n", "len(df.columns)" ] }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 54, "id": "13ae45dc-85a0-4864-8a58-9dc29ae4efd7", "metadata": {}, "outputs": [ @@ -1866,8 +2066,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 266 ms, sys: 69.9 ms, total: 336 ms\n", - "Wall time: 4.05 s\n" + "CPU times: user 231 ms, sys: 75.8 ms, total: 307 ms\n", + "Wall time: 3.22 s\n" ] } ], @@ -1878,7 +2078,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 55, "id": "0b3fb48b-f871-41f2-ac57-346899a6fe48", "metadata": {}, "outputs": [ @@ -1886,8 +2086,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 271 ms, sys: 66.2 ms, total: 337 ms\n", - "Wall time: 1.96 s\n" + "CPU times: user 292 ms, sys: 65.2 ms, total: 357 ms\n", + "Wall time: 1.48 s\n" ] } ], @@ -1897,18 +2097,38 @@ ] }, { - "cell_type": "markdown", - "id": "dc48ec42-0df6-4e6a-b019-1270ab71d2cf", - "metadata": { - "tags": [] - }, - "source": [ + "cell_type": "code", + "execution_count": 56, + "id": "b59114ad", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 235 ms, sys: 75.2 ms, total: 310 ms\n", + "Wall time: 1.2 s\n" + ] + } + ], + "source": [ + "%%time\n", + "preds = df.withColumn(\"preds\", mnist(array(*df.columns))).collect()" + ] + }, + { + "cell_type": "markdown", + "id": "dc48ec42-0df6-4e6a-b019-1270ab71d2cf", + "metadata": { + "tags": [] + }, + "source": [ "### Check predictions" ] }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 57, "id": "d815c701-9f5b-422c-b3f9-fbc30456953c", "metadata": {}, "outputs": [], @@ -1918,7 +2138,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 58, "id": "b571b742-5079-42b2-8524-9181a0dec2c7", "metadata": {}, "outputs": [], @@ -1930,7 +2150,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 59, "id": "d33d6a4e-e6b9-489d-ac21-c4eddc801784", "metadata": {}, "outputs": [], @@ -1941,7 +2161,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 60, "id": "6d10061e-aca6-4f81-bdfe-72e327ed7349", "metadata": {}, "outputs": [], @@ -1951,7 +2171,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 61, "id": "01f70e08-2c1d-419f-8676-3f6f4aba760f", "metadata": {}, "outputs": [ @@ -1974,7 +2194,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 62, "id": "8e1c07cc-b2bc-4902-a9a6-4ac7f02c5fe4", "metadata": {}, "outputs": [ @@ -1982,8 +2202,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "[ 2.5953586 3.9101725 0.65233815 3.2538052 1.270339 -3.0440047\n", - " 1.1500907 -3.7935097 -1.8807431 -3.321768 ]\n", + "[ 2.4792047 3.8394718 0.3413597 3.1128588 1.103995 -3.024009\n", + " 1.1684668 -3.6187618 -1.6347903 -3.1772547]\n", "predicted label: Trouser\n" ] } @@ -1993,263 +2213,375 @@ "print(\"predicted label:\", classes[np.argmax(predictions)])" ] }, + { + "cell_type": "code", + "execution_count": 63, + "id": "3d47a8ec", + "metadata": {}, + "outputs": [], + "source": [ + "# This will clear the engine cache (containing previously compiled TensorRT engines) and resets the CUDA Context.\n", + "torch._dynamo.reset()" + ] + }, { "cell_type": "markdown", - "id": "a937adc9-508d-4ccd-b92d-8ecaa27ee4e4", + "id": "281c7889", "metadata": {}, "source": [ - "### Using Triton Inference Server\n", + "## Using Triton Inference Server\n", + "In this section, we demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. \n", + "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server. \n", "\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment." + "The process looks like this:\n", + "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n", + "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n", + "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n", + "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n", + "\n", + "\"drawing\"" ] }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 64, "id": "53ca290a-ccc3-4923-a292-944921bab36d", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ - "import numpy as np\n", - "\n", - "from functools import partial\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import struct, col, array\n", - "from pyspark.sql.types import ArrayType, FloatType, Union, Dict" + "from functools import partial" + ] + }, + { + "cell_type": "markdown", + "id": "d8abea75", + "metadata": {}, + "source": [ + "Import the utility functions from pytriton_utils.py:" ] }, { "cell_type": "code", - "execution_count": 58, - "id": "8fa92fe4-2e04-4d82-a357-bfdfca38bd8c", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 65, + "id": "e616b207", + "metadata": {}, "outputs": [], "source": [ - "%%bash\n", - "# copy model to expected layout for Triton\n", - "rm -rf models\n", - "mkdir -p models/fashion_mnist/1\n", - "cp ts_model.pt models/fashion_mnist/1/model.pt\n", + "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "# add config.pbtxt\n", - "cp models_config/fashion_mnist/config.pbtxt models/fashion_mnist/config.pbtxt" + "from pytriton_utils import (\n", + " use_stage_level_scheduling,\n", + " find_ports,\n", + " start_triton,\n", + " stop_triton\n", + ")" ] }, { "cell_type": "markdown", - "id": "d42b329c-5921-436f-bfca-a382a6762da4", + "id": "606934ac", "metadata": {}, "source": [ - "#### Start Triton Server on each executor" + "Define the Triton Server function:" ] }, { "cell_type": "code", - "execution_count": null, - "id": "5e869730-3597-4074-bab0-f87768f8996a", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 66, + "id": "8fa92fe4-2e04-4d82-a357-bfdfca38bd8c", + "metadata": {}, "outputs": [], "source": [ - "num_executors = 1\n", - "triton_models_dir = \"{}/models\".format(os.getcwd())\n", - "nodeRDD = sc.parallelize(list(range(num_executors)), num_executors)\n", - "\n", - "def start_triton(it):\n", - " import docker\n", + "def triton_server(ports, model_path):\n", " import time\n", - " import tritonclient.grpc as grpcclient\n", + " import signal\n", + " import numpy as np\n", + " import torch\n", + " from torch import nn\n", + " import torch_tensorrt as trt\n", + " from pytriton.decorators import batch\n", + " from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n", + " from pytriton.triton import Triton, TritonConfig\n", + " from pyspark import TaskContext\n", + "\n", + " print(f\"SERVER: Initializing model on worker {TaskContext.get().partitionId()}.\")\n", + " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " if containers:\n", - " print(\">>>> containers: {}\".format([c.short_id for c in containers]))\n", - " else:\n", - " container=client.containers.run(\n", - " \"nvcr.io/nvidia/tritonserver:24.08-py3\", \"tritonserver --model-repository=/models\",\n", - " detach=True,\n", - " device_requests=[docker.types.DeviceRequest(device_ids=[\"0\"], capabilities=[['gpu']])],\n", - " name=\"spark-triton\",\n", - " network_mode=\"host\",\n", - " remove=True,\n", - " shm_size=\"64M\",\n", - " volumes={triton_models_dir: {\"bind\": \"/models\", \"mode\": \"ro\"}}\n", + " exp_program = torch.export.load(model_path)\n", + " example_inputs = (torch.randn((50, 784), dtype=torch.float).to(\"cuda\"),)\n", + " trt_gm = trt.dynamo.compile(exp_program,\n", + " tuple(example_inputs),\n", + " enabled_precisions={torch.float},\n", + " workspace_size=1<<30)\n", + "\n", + " print(\"SERVER: Compiled model.\")\n", + "\n", + " @batch\n", + " def _infer_fn(**inputs):\n", + " images = inputs[\"images\"]\n", + " if len(images) != 1:\n", + " images = np.squeeze(images)\n", + " stream = torch.cuda.Stream()\n", + " with torch.no_grad(), torch.cuda.stream(stream):\n", + " torch_inputs = torch.from_numpy(images).to(device)\n", + " outputs = trt_gm(torch_inputs)\n", + " return {\n", + " \"labels\": outputs.cpu().numpy(),\n", + " }\n", + " \n", + " workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n", + " triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n", + " with Triton(config=triton_conf, workspace=workspace_path) as triton:\n", + " triton.bind(\n", + " model_name=\"ImageClassifier\",\n", + " infer_func=_infer_fn,\n", + " inputs=[\n", + " Tensor(name=\"images\", dtype=np.float32, shape=(-1,)),\n", + " ],\n", + " outputs=[\n", + " Tensor(name=\"labels\", dtype=np.float32, shape=(-1,)),\n", + " ],\n", + " config=ModelConfig(\n", + " max_batch_size=64,\n", + " batcher=DynamicBatcher(max_queue_delay_microseconds=5000), # 5ms\n", + " ),\n", + " strict=True,\n", " )\n", - " print(\">>>> starting triton: {}\".format(container.short_id))\n", "\n", - " # wait for triton to be running\n", - " time.sleep(15)\n", - " client = grpcclient.InferenceServerClient(\"localhost:8001\")\n", - " ready = False\n", - " while not ready:\n", - " try:\n", - " ready = client.is_server_ready()\n", - " except Exception as e:\n", - " time.sleep(5)\n", - " \n", - " return [True]\n", + " def _stop_triton(signum, frame):\n", + " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", + " triton.stop()\n", + "\n", + " signal.signal(signal.SIGTERM, _stop_triton)\n", "\n", - "nodeRDD.barrier().mapPartitions(start_triton).collect()" + " print(\"SERVER: Serving inference\")\n", + " triton.serve()" ] }, { "cell_type": "markdown", - "id": "30a4362d-7514-4b84-b238-f704a97e1e72", + "id": "8fea6e5e", "metadata": {}, "source": [ - "#### Run inference" + "#### Start Triton servers " + ] + }, + { + "cell_type": "markdown", + "id": "2cbca940", + "metadata": {}, + "source": [ + "**Specify the number of nodes in the cluster.** \n", + "Following the README, the example standalone cluster uses 1 node. The example Databricks/Dataproc cluster scripts use 4 nodes by default. " ] }, { "cell_type": "code", - "execution_count": 60, - "id": "ab94d4d1-dac6-4474-9eb0-59478aa98f7d", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 67, + "id": "a63d19bc", + "metadata": {}, + "outputs": [], + "source": [ + "# Change based on cluster setup\n", + "num_nodes = 1 if on_standalone else 4" + ] + }, + { + "cell_type": "markdown", + "id": "ec34f9eb", + "metadata": {}, + "source": [ + "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU." + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "id": "06349836", + "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "1" - ] - }, - "execution_count": 60, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] } ], "source": [ - "df = spark.read.parquet(\"fashion_mnist_1\")\n", - "len(df.columns)" + "sc = spark.sparkContext\n", + "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" + ] + }, + { + "cell_type": "markdown", + "id": "508f9972", + "metadata": {}, + "source": [ + "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." ] }, { "cell_type": "code", - "execution_count": 61, - "id": "12b5f2fc-52e9-428a-b683-6ab1b639aa24", - "metadata": { - "scrolled": true, - "tags": [ - "TRITON" - ] - }, + "execution_count": 69, + "id": "33cd12f9", + "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "StructType([StructField('data', ArrayType(FloatType(), True), True)])" - ] - }, - "execution_count": 61, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "Using ports [7000, 7001, 7002]\n" + ] } ], "source": [ - "df.schema" + "model_name = \"ImageClassifier\"\n", + "ports = find_ports()\n", + "assert len(ports) == 3\n", + "print(f\"Using ports {ports}\")" ] }, { "cell_type": "code", - "execution_count": 62, - "id": "960657d0-31c9-4df6-8eb8-ac3d23137f7a", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 70, + "id": "6a0e8778", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 16:> (0 + 1) / 1]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Triton Server PIDs:\n", + " {\n", + " \"cb4ae00-lcedt\": 2968686\n", + "}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", + " ports=ports,\n", + " model_name=model_name,\n", + " model_path=model_path)).collectAsMap()\n", + "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + ] + }, + { + "cell_type": "markdown", + "id": "90ed191b", + "metadata": {}, + "source": [ + "#### Define client function" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "id": "549cda8b", + "metadata": {}, "outputs": [], "source": [ - "def triton_fn(triton_uri, model_name):\n", - " import numpy as np\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " np_types = {\n", - " \"BOOL\": np.dtype(np.bool_),\n", - " \"INT8\": np.dtype(np.int8),\n", - " \"INT16\": np.dtype(np.int16),\n", - " \"INT32\": np.dtype(np.int32),\n", - " \"INT64\": np.dtype(np.int64),\n", - " \"FP16\": np.dtype(np.float16),\n", - " \"FP32\": np.dtype(np.float32),\n", - " \"FP64\": np.dtype(np.float64),\n", - " \"FP64\": np.dtype(np.double),\n", - " \"BYTES\": np.dtype(object)\n", - " }\n", + "url = f\"http://localhost:{ports[0]}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "id": "cec9a48c", + "metadata": {}, + "outputs": [], + "source": [ + "def triton_fn(url, model_name):\n", + " from pytriton.client import ModelClient\n", "\n", - " client = grpcclient.InferenceServerClient(triton_uri)\n", - " model_meta = client.get_model_metadata(model_name)\n", - " \n", - " def predict(inputs):\n", - " if isinstance(inputs, np.ndarray):\n", - " # single ndarray input\n", - " request = [grpcclient.InferInput(model_meta.inputs[0].name, inputs.shape, model_meta.inputs[0].datatype)]\n", - " request[0].set_data_from_numpy(inputs.astype(np_types[model_meta.inputs[0].datatype]))\n", - " else:\n", - " # dict of multiple ndarray inputs\n", - " request = [grpcclient.InferInput(i.name, inputs[i.name].shape, i.datatype) for i in model_meta.inputs]\n", - " for i in request:\n", - " i.set_data_from_numpy(inputs[i.name()].astype(np_types[i.datatype()]))\n", - " \n", - " response = client.infer(model_name, inputs=request)\n", - " \n", - " if len(model_meta.outputs) > 1:\n", - " # return dictionary of numpy arrays\n", - " return {o.name: response.as_numpy(o.name) for o in model_meta.outputs}\n", - " else:\n", - " # return single numpy array\n", - " return response.as_numpy(model_meta.outputs[0].name)\n", + " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", + "\n", + " def infer_batch(inputs):\n", + " with ModelClient(url, model_name, inference_timeout_s=240) as client:\n", + " result_data = client.infer_batch(inputs)\n", + " return result_data[\"labels\"]\n", " \n", - " return predict" + " return infer_batch" + ] + }, + { + "cell_type": "markdown", + "id": "30a4362d-7514-4b84-b238-f704a97e1e72", + "metadata": {}, + "source": [ + "#### Run inference" ] }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 73, + "id": "ab94d4d1-dac6-4474-9eb0-59478aa98f7d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "StructType([StructField('data', ArrayType(FloatType(), True), True)])" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = spark.read.parquet(data_path_1)\n", + "df.schema" + ] + }, + { + "cell_type": "code", + "execution_count": 74, "id": "0262fd4a-9845-44b9-8c75-1c105e7deeca", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ - "mnist = predict_batch_udf(partial(triton_fn, triton_uri=\"localhost:8001\", model_name=\"fashion_mnist\"),\n", + "mnist = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", " input_tensor_shapes=[[784]],\n", " return_type=ArrayType(FloatType()),\n", - " batch_size=1024)" + " batch_size=50)" ] }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 75, "id": "fc5f6baa-052e-4b89-94b6-4821cf01952a", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 326 ms, sys: 39.6 ms, total: 365 ms\n", - "Wall time: 1.77 s\n" + "CPU times: user 159 ms, sys: 37.3 ms, total: 196 ms\n", + "Wall time: 1.53 s\n" ] } ], @@ -2260,20 +2592,16 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 76, "id": "a85dea35-e41d-482d-8a8f-52d3c108f038", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 199 ms, sys: 63.2 ms, total: 262 ms\n", - "Wall time: 1.21 s\n" + "CPU times: user 176 ms, sys: 68.6 ms, total: 245 ms\n", + "Wall time: 952 ms\n" ] } ], @@ -2284,20 +2612,23 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 77, "id": "bc3f0dbe-c52b-41d6-8097-8cebaa5ee5a8", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 195 ms, sys: 25.4 ms, total: 220 ms\n", - "Wall time: 1.21 s\n" + "CPU times: user 189 ms, sys: 43 ms, total: 232 ms\n", + "Wall time: 1.17 s\n" ] } ], @@ -2308,13 +2639,9 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 78, "id": "99fb5e8d", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -2359,39 +2686,50 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 79, "id": "ab2fe42f-a072-4370-bac2-52fd95363530", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/plain": [ + "[True]" + ] + }, + "execution_count": 79, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "def stop_triton(it):\n", - " import docker\n", - " import time\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " print(\">>>> stopping containers: {}\".format([c.short_id for c in containers]))\n", - " if containers:\n", - " container=containers[0]\n", - " container.stop(timeout=120)\n", - "\n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(stop_triton).collect()" + "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", + "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" ] }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 80, "id": "a0608fff-7cfb-489e-96c9-8e1d92e57562", "metadata": {}, "outputs": [], "source": [ - "spark.stop()" + "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n", + " spark.stop()" ] }, { diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/fashion_mnist/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/fashion_mnist/config.pbtxt deleted file mode 100644 index d98b2a11..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/fashion_mnist/config.pbtxt +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -platform: "pytorch_libtorch" -max_batch_size: 8192 -input [ - { - name: "input" - data_type: TYPE_FP32 - dims: [ 784 ] - } -] -output [ - { - name: "output" - data_type: TYPE_FP32 - dims: [ 10 ] - } -] diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/housing_model/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/housing_model/config.pbtxt deleted file mode 100644 index 64deefca..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/models_config/housing_model/config.pbtxt +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -platform: "pytorch_libtorch" -max_batch_size: 8192 -input [ - { - name: "input" - data_type: TYPE_FP32 - dims: [ 8 ] - } -] -output [ - { - name: "output" - data_type: TYPE_FP32 - dims: [ 1 ] - } -] diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/pytriton_utils.py b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/pytriton_utils.py new file mode 120000 index 00000000..330ea4a5 --- /dev/null +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/pytriton_utils.py @@ -0,0 +1 @@ +../pytriton_utils.py \ No newline at end of file diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/regression_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/regression_torch.ipynb deleted file mode 100644 index 3412f91a..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/regression_torch.ipynb +++ /dev/null @@ -1,2725 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "792d95f9", - "metadata": {}, - "source": [ - "# PySpark PyTorch Inference\n", - "\n", - "### Regression\n", - "Based on: https://github.com/christianversloot/machine-learning-articles/blob/main/how-to-create-a-neural-network-for-regression-with-pytorch.md \n", - "\n", - "For the first MLP (array inputs) we'll also demonstrate accelerated inference on GPU with Torch-TensorRT. " - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "75930360-c5ce-49ef-a69a-da88fa69a2ef", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "import torch\n", - "from torch import nn\n", - "from torch.utils.data import DataLoader\n", - "from sklearn.datasets import fetch_california_housing\n", - "from sklearn.preprocessing import StandardScaler" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "6d5bc0c7", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'2.4.1+cu121'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "cf02ba0a-8384-42b5-917c-53889b4a6471", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(42)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "bb5c10ab", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using cuda device\n" - ] - } - ], - "source": [ - "# Get cpu or gpu device for training.\n", - "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", - "print(f\"Using {device} device\")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "2bee64cf-a44a-4aff-82db-c64ee3a8b0e8", - "metadata": {}, - "outputs": [], - "source": [ - "X, y = fetch_california_housing(return_X_y=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "8644e508-5e4c-4cdd-9ed1-9235887d9659", - "metadata": {}, - "outputs": [], - "source": [ - "class HousingDataset(torch.utils.data.Dataset):\n", - " def __init__(self, X, y, scale_data=True):\n", - " if not torch.is_tensor(X) and not torch.is_tensor(y):\n", - " # Apply scaling if necessary\n", - " if scale_data:\n", - " X = StandardScaler().fit_transform(X)\n", - " self.X = torch.from_numpy(X.astype(np.float32))\n", - " self.y = torch.from_numpy(y.astype(np.float32))\n", - "\n", - " def __len__(self):\n", - " return len(self.X)\n", - "\n", - " def __getitem__(self, i):\n", - " return self.X[i], self.y[i]" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "cc6b55c3-dc7b-4831-9943-83efd48091bf", - "metadata": {}, - "outputs": [], - "source": [ - "dataset = HousingDataset(X, y)\n", - "trainloader = torch.utils.data.DataLoader(\n", - " dataset, batch_size=10, shuffle=True, num_workers=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "d868f39d-4695-4110-91d2-6f7a09d73b93", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[tensor([[ 0.4159, 0.3465, 0.2294, -0.1113, -0.3130, 0.0527, -0.5955, 0.3193],\n", - " [-0.3706, 0.5849, -0.1514, -0.2040, 0.3943, 0.0550, -0.8155, 0.6687],\n", - " [-0.2024, -0.8454, 0.1928, 0.0087, 0.5435, -0.0279, 0.9449, -1.2679],\n", - " [ 0.1064, -1.9578, 0.2968, 0.0363, 2.7556, -0.0217, -0.4925, 0.7685],\n", - " [ 0.1057, 1.0616, 0.1675, -0.0081, -0.3651, -0.0372, -0.6751, 0.7186],\n", - " [ 0.3343, -1.4811, -0.7187, 0.2041, 0.0967, 0.0529, -0.8483, 0.8234],\n", - " [-0.2691, 1.3000, -0.6491, -0.0872, 1.7074, -0.1372, -0.7312, 0.6088],\n", - " [-0.0891, -0.3686, 0.0260, -0.1563, 0.3996, 0.0449, 1.1790, -1.3378],\n", - " [ 0.1397, -0.2097, 0.1816, -0.1881, 0.4049, -0.0229, -0.7547, 1.2676],\n", - " [-0.3399, 0.3465, -0.4621, -0.0519, 0.6115, -0.0284, -0.6704, 0.5189]]),\n", - " tensor([1.8790, 1.1770, 3.3160, 1.5430, 2.3400, 1.5240, 2.8750, 1.2360, 1.2100,\n", - " 2.0530])]" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "next(iter(trainloader))" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "9a441b60-dca4-44d2-bc1c-aa7336d704bb", - "metadata": {}, - "outputs": [], - "source": [ - "class MLP(nn.Module):\n", - " def __init__(self):\n", - " super().__init__()\n", - " self.layers = nn.Sequential(\n", - " nn.Linear(8, 64),\n", - " nn.ReLU(),\n", - " nn.Linear(64, 32),\n", - " nn.ReLU(),\n", - " nn.Linear(32, 1)\n", - " )\n", - "\n", - " def forward(self, x):\n", - " return self.layers(x)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "15cff2b4-9d23-4d2b-808a-a5edb8eda135", - "metadata": { - "scrolled": true, - "tags": [] - }, - "outputs": [], - "source": [ - "# Initialize the MLP\n", - "mlp = MLP().to(device)\n", - "\n", - "# Define the loss function and optimizer\n", - "loss_function = nn.L1Loss()\n", - "optimizer = torch.optim.Adam(mlp.parameters(), lr=1e-4)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "5e2db3f9-5db8-4b42-89ad-e77f23c4c1fe", - "metadata": { - "scrolled": true, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Starting epoch 1\n", - "Loss after mini-batch 1: 0.004\n", - "Loss after mini-batch 201: 0.733\n", - "Loss after mini-batch 401: 0.534\n", - "Loss after mini-batch 601: 0.403\n", - "Loss after mini-batch 801: 0.330\n", - "Loss after mini-batch 1001: 0.269\n", - "Loss after mini-batch 1201: 0.232\n", - "Loss after mini-batch 1401: 0.226\n", - "Loss after mini-batch 1601: 0.223\n", - "Loss after mini-batch 1801: 0.214\n", - "Loss after mini-batch 2001: 0.214\n", - "Starting epoch 2\n", - "Loss after mini-batch 1: 0.002\n", - "Loss after mini-batch 201: 0.211\n", - "Loss after mini-batch 401: 0.205\n", - "Loss after mini-batch 601: 0.199\n", - "Loss after mini-batch 801: 0.192\n", - "Loss after mini-batch 1001: 0.194\n", - "Loss after mini-batch 1201: 0.196\n", - "Loss after mini-batch 1401: 0.193\n", - "Loss after mini-batch 1601: 0.194\n", - "Loss after mini-batch 1801: 0.187\n", - "Loss after mini-batch 2001: 0.197\n", - "Starting epoch 3\n", - "Loss after mini-batch 1: 0.001\n", - "Loss after mini-batch 201: 0.190\n", - "Loss after mini-batch 401: 0.183\n", - "Loss after mini-batch 601: 0.181\n", - "Loss after mini-batch 801: 0.190\n", - "Loss after mini-batch 1001: 0.188\n", - "Loss after mini-batch 1201: 0.186\n", - "Loss after mini-batch 1401: 0.193\n", - "Loss after mini-batch 1601: 0.182\n", - "Loss after mini-batch 1801: 0.184\n", - "Loss after mini-batch 2001: 0.188\n", - "Starting epoch 4\n", - "Loss after mini-batch 1: 0.001\n", - "Loss after mini-batch 201: 0.180\n", - "Loss after mini-batch 401: 0.179\n", - "Loss after mini-batch 601: 0.183\n", - "Loss after mini-batch 801: 0.180\n", - "Loss after mini-batch 1001: 0.176\n", - "Loss after mini-batch 1201: 0.189\n", - "Loss after mini-batch 1401: 0.176\n", - "Loss after mini-batch 1601: 0.185\n", - "Loss after mini-batch 1801: 0.177\n", - "Loss after mini-batch 2001: 0.185\n", - "Starting epoch 5\n", - "Loss after mini-batch 1: 0.001\n", - "Loss after mini-batch 201: 0.179\n", - "Loss after mini-batch 401: 0.177\n", - "Loss after mini-batch 601: 0.175\n", - "Loss after mini-batch 801: 0.178\n", - "Loss after mini-batch 1001: 0.173\n", - "Loss after mini-batch 1201: 0.178\n", - "Loss after mini-batch 1401: 0.176\n", - "Loss after mini-batch 1601: 0.174\n", - "Loss after mini-batch 1801: 0.179\n", - "Loss after mini-batch 2001: 0.180\n", - "Training process has finished.\n" - ] - } - ], - "source": [ - "# Run the training loop\n", - "for epoch in range(0, 5): # 5 epochs at maximum\n", - "\n", - " # Print epoch\n", - " print(f'Starting epoch {epoch+1}')\n", - "\n", - " # Set current loss value\n", - " current_loss = 0.0\n", - "\n", - " # Iterate over the DataLoader for training data\n", - " for i, data in enumerate(trainloader, 0):\n", - "\n", - " # Get and prepare inputs\n", - " inputs, targets = data\n", - " inputs, targets = inputs.to(device), targets.to(device)\n", - " targets = targets.reshape((targets.shape[0], 1))\n", - "\n", - " # Zero the gradients\n", - " optimizer.zero_grad()\n", - "\n", - " # Perform forward pass\n", - " outputs = mlp(inputs)\n", - "\n", - " # Compute loss\n", - " loss = loss_function(outputs, targets)\n", - "\n", - " # Perform backward pass\n", - " loss.backward()\n", - "\n", - " # Perform optimization\n", - " optimizer.step()\n", - "\n", - " # Print statistics\n", - " current_loss += loss.item()\n", - " if i % 200 == 0:\n", - " print('Loss after mini-batch %5d: %.3f' %\n", - " (i + 1, current_loss / 500))\n", - " current_loss = 0.0\n", - "\n", - "# Process is complete.\n", - "print('Training process has finished.')" - ] - }, - { - "cell_type": "markdown", - "id": "352539f5", - "metadata": {}, - "source": [ - "### Save Model State Dict\n", - "This saves the serialized object to disk using pickle." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "b950a3ed-ffe1-477f-a84f-f71c85dbf9ce", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saved PyTorch Model State to housing_model.pt\n" - ] - } - ], - "source": [ - "torch.save(mlp.state_dict(), \"housing_model.pt\")\n", - "print(\"Saved PyTorch Model State to housing_model.pt\")" - ] - }, - { - "cell_type": "markdown", - "id": "0060fcca", - "metadata": {}, - "source": [ - "### Save Model as TorchScript\n", - "This saves an [intermediate representation of the compute graph](https://pytorch.org/tutorials/beginner/saving_loading_models.html#export-load-model-in-torchscript-format), which does not require pickle (or even python). " - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "20fedb5d-c59e-4b0b-ba91-3dd15df1f09e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saved TorchScript Model to ts_housing_model.pt\n" - ] - } - ], - "source": [ - "scripted = torch.jit.script(mlp)\n", - "scripted.save(\"ts_housing_model.pt\")\n", - "print(\"Saved TorchScript Model to ts_housing_model.pt\")" - ] - }, - { - "cell_type": "markdown", - "id": "3101c0fe-65f1-411e-9192-e8a6b585ba0d", - "metadata": {}, - "source": [ - "### Load and Test from Model State" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "7411b00f-88d2-40f5-b716-a26733c968ff", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "loaded_mlp = MLP().to(device)\n", - "loaded_mlp.load_state_dict(torch.load(\"housing_model.pt\", weights_only=True))" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "e226f449-2931-4492-9003-503cdc61f061", - "metadata": {}, - "outputs": [], - "source": [ - "testX, testY = next(iter(trainloader))" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "d46af47e-db7e-42ee-9bd3-6e7d93850be3", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[1.7498],\n", - " [3.0116],\n", - " [1.1925],\n", - " [4.0598],\n", - " [2.0545],\n", - " [2.9072],\n", - " [2.0551],\n", - " [4.6094],\n", - " [1.0068],\n", - " [1.1174]], device='cuda:0', grad_fn=)" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "loaded_mlp(testX.to(device))" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "13ae2c0f-1da5-45a4-bf32-ed8b562d7907", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([3.5000, 2.9130, 0.7020, 5.0000, 1.7970, 2.7080, 2.1470, 5.0000, 0.6000,\n", - " 0.8480])" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "testY" - ] - }, - { - "cell_type": "markdown", - "id": "3bcd329d", - "metadata": {}, - "source": [ - "### Load and Test from TorchScript" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "422e317f-c9bd-4f76-9463-7af2935d401d", - "metadata": {}, - "outputs": [], - "source": [ - "scripted_mlp = torch.jit.load(\"ts_housing_model.pt\")" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "0cda8ec8-644e-4888-bfa0-b79425ece7c3", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([1.7498, 3.0116, 1.1925, 4.0598, 2.0545, 2.9072, 2.0551, 4.6094, 1.0068,\n", - " 1.1174], device='cuda:0', grad_fn=)" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "scripted_mlp(testX.to(device)).flatten()" - ] - }, - { - "cell_type": "markdown", - "id": "2a3b64e4", - "metadata": {}, - "source": [ - "### Compile using the Torch JIT Compiler\n", - "This leverages the [Torch-TensorRT inference compiler](https://pytorch.org/TensorRT/) for accelerated inference on GPUs using the `torch.compile` JIT interface under the hood. The compiler stack returns a [boxed-function](http://blog.ezyang.com/2020/09/lets-talk-about-the-pytorch-dispatcher/) that triggers compilation on the first call. \n", - "\n", - "Modules compiled in this fashion are [not serializable with pickle](https://github.com/pytorch/pytorch/issues/101107#issuecomment-1542688089), so we cannot send the compiled model directly to Spark. Instead, we will recompile and cache the model on the executor. " - ] - }, - { - "cell_type": "markdown", - "id": "c613f24e", - "metadata": {}, - "source": [ - "(You may see a warning about modelopt quantization. This is safe to ignore, as [implicit quantization](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#intro-quantization) is deprecated in the latest TensorRT. See [this link](https://pytorch.org/TensorRT/tutorials/_rendered_examples/dynamo/vgg16_fp8_ptq.html) for a guide to explicit quantization.)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9ffb27fc", - "metadata": {}, - "outputs": [], - "source": [ - "import torch_tensorrt as trt\n", - "import time" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e0c10f90", - "metadata": {}, - "outputs": [], - "source": [ - "# Optional: set the filename for the TensorRT timing cache\n", - "timestamp = time.time()\n", - "timing_cache = f\"/tmp/timing_cache-{timestamp}.bin\"\n", - "with open(timing_cache, \"wb\") as f:\n", - " pass" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "b4aa2523", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:torch_tensorrt.dynamo.conversion.aten_ops_converters:Unable to import quantization op. Please install modelopt library (https://github.com/NVIDIA/TensorRT-Model-Optimizer?tab=readme-ov-file#installation) to add support for compiling quantized models\n", - "INFO:torch_tensorrt.dynamo.utils:Using Default Torch-TRT Runtime (as requested by user)\n", - "INFO:torch_tensorrt.dynamo.utils:Device not specified, using Torch default current device - cuda:0. If this is incorrect, please specify an input device, via the device keyword.\n", - "INFO:torch_tensorrt.dynamo.utils:Compilation Settings: CompilationSettings(enabled_precisions={}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, refit=False, engine_capability=, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/timing_cache.bin')\n", - "\n", - "WARNING:torch_tensorrt.dynamo._compiler:Node _param_constant1 of op type get_attr does not have metadata. This could sometimes lead to undefined behavior.\n", - "WARNING:torch_tensorrt.dynamo._compiler:Some nodes do not have metadata (shape and dtype information). This could lead to problems sometimes if the graph has PyTorch and TensorRT segments.\n", - "INFO:torch_tensorrt.dynamo._compiler:Partitioning the graph via the fast partitioner\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init CUDA: CPU +1, GPU +0, now: CPU 586, GPU 1112 (MiB)\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init builder kernel library: CPU +1635, GPU +288, now: CPU 2368, GPU 1400 (MiB)\n", - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/torch_tensorrt/dynamo/conversion/impl/activation/base.py:40: DeprecationWarning: Use Deprecated in TensorRT 10.1. Superseded by explicit quantization. instead.\n", - " if input_val.dynamic_range is not None and dyn_range_fn is not None:\n", - "\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT INetwork construction elapsed time: 0:00:00.003844\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Global timing cache in use. Profiling results in this builder pass will be stored.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Detected 1 inputs and 1 output network tensors.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Host Persistent Memory: 22240\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Device Persistent Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Scratch Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Started assigning block shifts. This will take 10 steps to complete.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Algorithm ShiftNTopDown took 0.156445ms to assign 4 blocks to 10 nodes requiring 7168 bytes.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Activation Memory: 6656\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Weights Memory: 11648\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Engine generation completed in 0.0225783 seconds.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage of TRT CPU/GPU memory allocators: CPU 0 MiB, GPU 1 MiB\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage during Engine building and serialization: CPU: 3966 MiB\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:Build TRT engine elapsed time: 0:00:00.026248\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT Engine uses: 665972 bytes of Memory\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 26 bytes of code generator cache.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 176 timing cache entries\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "tensor([[1.7498],\n", - " [3.0116],\n", - " [1.1925],\n", - " [4.0598],\n", - " [2.0545],\n", - " [2.9072],\n", - " [2.0551],\n", - " [4.6094],\n", - " [1.0068],\n", - " [1.1174]], device='cuda:0')\n" - ] - } - ], - "source": [ - "inputs_bs1 = torch.randn((10, 8), dtype=torch.float).to(\"cuda\")\n", - "# This indicates dimension 0 of inputs_bs1 is dynamic whose range of values is [1, 50]. No recompilation will happen when the batch size changes.\n", - "torch._dynamo.mark_dynamic(inputs_bs1, 0, min=1, max=50)\n", - "trt_model = trt.compile(\n", - " loaded_mlp,\n", - " ir=\"torch_compile\",\n", - " inputs=inputs_bs1,\n", - " enabled_precisions={torch.float},\n", - " timing_cache_path=timing_cache,\n", - ")\n", - "\n", - "stream = torch.cuda.Stream()\n", - "with torch.no_grad(), torch.cuda.stream(stream):\n", - " testX = testX.to(device)\n", - " print(trt_model(testX))" - ] - }, - { - "cell_type": "markdown", - "id": "d2c55e07", - "metadata": {}, - "source": [ - "### Compile using the Torch-TensorRT AOT Compiler\n", - "Alternatively, use the Torch-TensorRT Dynamo backend for Ahead-of-Time (AOT) compilation to eagerly optimize the model in an explicit compilation phase. We first export the model to produce a traced graph representing the Tensor computation in an AOT fashion, which produces a `ExportedProgram` object which can be [serialized and reloaded](https://pytorch.org/TensorRT/user_guide/saving_models.html). We can then compile this IR using the Torch-TensorRT AOT compiler for inference. \n", - "\n", - "[Read the docs](https://pytorch.org/TensorRT/user_guide/torch_tensorrt_explained.html) for more information on JIT vs AOT compilation." - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "b6b5c112", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:torch_tensorrt.dynamo._compiler:Compilation Settings: CompilationSettings(enabled_precisions={}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, refit=False, engine_capability=, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/timing_cache.bin')\n", - "\n", - "INFO:torch_tensorrt.dynamo._compiler:Partitioning the graph via the fast partitioner\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init CUDA: CPU +0, GPU +0, now: CPU 762, GPU 1114 (MiB)\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init builder kernel library: CPU +1633, GPU +286, now: CPU 2395, GPU 1400 (MiB)\n", - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/torch_tensorrt/dynamo/conversion/impl/activation/base.py:40: DeprecationWarning: Use Deprecated in TensorRT 10.1. Superseded by explicit quantization. instead.\n", - " if input_val.dynamic_range is not None and dyn_range_fn is not None:\n", - "\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT INetwork construction elapsed time: 0:00:00.002832\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Global timing cache in use. Profiling results in this builder pass will be stored.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Detected 1 inputs and 1 output network tensors.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Host Persistent Memory: 22240\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Device Persistent Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Scratch Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Started assigning block shifts. This will take 10 steps to complete.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Algorithm ShiftNTopDown took 0.120805ms to assign 4 blocks to 10 nodes requiring 7168 bytes.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Activation Memory: 6656\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Weights Memory: 11648\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Engine generation completed in 0.014855 seconds.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage of TRT CPU/GPU memory allocators: CPU 0 MiB, GPU 1 MiB\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage during Engine building and serialization: CPU: 3989 MiB\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:Build TRT engine elapsed time: 0:00:00.016879\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT Engine uses: 666804 bytes of Memory\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 26 bytes of code generator cache.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 176 timing cache entries\n", - "INFO:torch_tensorrt.dynamo.utils:Using Default Torch-TRT Runtime (as requested by user)\n", - "INFO:torch_tensorrt.dynamo.utils:Device not specified, using Torch default current device - cuda:0. If this is incorrect, please specify an input device, via the device keyword.\n", - "INFO:torch_tensorrt.dynamo.utils:Compilation Settings: CompilationSettings(enabled_precisions={}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, refit=False, engine_capability=, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/timing_cache.bin')\n", - "\n", - "WARNING:torch_tensorrt.dynamo._compiler:Node _param_constant1 of op type get_attr does not have metadata. This could sometimes lead to undefined behavior.\n", - "WARNING:torch_tensorrt.dynamo._compiler:Some nodes do not have metadata (shape and dtype information). This could lead to problems sometimes if the graph has PyTorch and TensorRT segments.\n", - "INFO:torch_tensorrt.dynamo._compiler:Partitioning the graph via the fast partitioner\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:The logger passed into createInferBuilder differs from one already provided for an existing builder, runtime, or refitter. Uses of the global logger, returned by nvinfer1::getLogger(), will return the existing value.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init CUDA: CPU +0, GPU +0, now: CPU 764, GPU 1136 (MiB)\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init builder kernel library: CPU +1632, GPU +286, now: CPU 2396, GPU 1422 (MiB)\n", - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/torch_tensorrt/dynamo/conversion/impl/activation/base.py:40: DeprecationWarning: Use Deprecated in TensorRT 10.1. Superseded by explicit quantization. instead.\n", - " if input_val.dynamic_range is not None and dyn_range_fn is not None:\n", - "\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT INetwork construction elapsed time: 0:00:00.002990\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Global timing cache in use. Profiling results in this builder pass will be stored.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Detected 1 inputs and 1 output network tensors.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Host Persistent Memory: 22240\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Device Persistent Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Scratch Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Started assigning block shifts. This will take 10 steps to complete.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Algorithm ShiftNTopDown took 0.243798ms to assign 4 blocks to 10 nodes requiring 7168 bytes.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Activation Memory: 6656\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Weights Memory: 11648\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Engine generation completed in 0.0158591 seconds.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage of TRT CPU/GPU memory allocators: CPU 0 MiB, GPU 1 MiB\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage during Engine building and serialization: CPU: 3991 MiB\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:Build TRT engine elapsed time: 0:00:00.017873\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT Engine uses: 665972 bytes of Memory\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 26 bytes of code generator cache.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 176 timing cache entries\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "tensor([[1.7498],\n", - " [3.0116],\n", - " [1.1925],\n", - " [4.0598],\n", - " [2.0545],\n", - " [2.9072],\n", - " [2.0551],\n", - " [4.6094],\n", - " [1.0068],\n", - " [1.1174]], device='cuda:0')\n" - ] - } - ], - "source": [ - "# Preparing the inputs for batch_size = 50. \n", - "inputs = (torch.randn((10, 8), dtype=torch.float).cuda(),)\n", - "\n", - "# Produce traced graph in the ExportedProgram format\n", - "exp_program = trt.dynamo.trace(loaded_mlp, inputs)\n", - "# Compile the traced graph to produce an optimized module\n", - "trt_gm = trt.dynamo.compile(exp_program,\n", - " inputs=inputs,\n", - " timing_cache_path=timing_cache)\n", - "\n", - "stream = torch.cuda.Stream()\n", - "with torch.no_grad(), torch.cuda.stream(stream):\n", - " testX = testX.to(device)\n", - " print(trt_model(testX))" - ] - }, - { - "cell_type": "markdown", - "id": "b4fef57d", - "metadata": {}, - "source": [ - "We can save the compiled model using `torch_tensorrt.save`. Unfortunately, serializing the model to be reloaded at a later date currently only supports *static inputs*." - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "dabc91a4", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/torch_tensorrt/dynamo/_exporter.py:364: UserWarning: Attempted to insert a get_attr Node with no underlying reference in the owning GraphModule! Call GraphModule.add_submodule to add the necessary submodule, GraphModule.add_parameter to add the necessary Parameter, or nn.Module.register_buffer to add the necessary buffer\n", - " engine_node = gm.graph.get_attr(engine_name)\n", - "\n", - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/torch/fx/graph.py:1545: UserWarning: Node _run_on_acc_0_engine target _run_on_acc_0_engine _run_on_acc_0_engine of does not reference an nn.Module, nn.Parameter, or buffer, which is what 'get_attr' Nodes typically target\n", - " warnings.warn(f'Node {node} target {node.target} {atom} of {seen_qualname} does '\n", - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saved AOT compiled TensorRT model to trt_model_aot.ep\n" - ] - } - ], - "source": [ - "with torch.cuda.stream(stream):\n", - " trt.save(trt_gm, \"trt_model_aot.ep\", inputs=[torch.randn((10, 8), dtype=torch.float).to(\"cuda\")])\n", - " print(\"Saved AOT compiled TensorRT model to trt_model_aot.ep\")" - ] - }, - { - "cell_type": "markdown", - "id": "28ae694c-0127-4a61-8630-06004866cd14", - "metadata": {}, - "source": [ - "### Columns as separate input variables" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "32e11813-cf75-448e-a46e-f210cc7f52ba", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "import torch\n", - "\n", - "from inspect import signature\n", - "from torch import nn\n", - "from torch.utils.data import DataLoader\n", - "from sklearn.datasets import fetch_california_housing\n", - "from sklearn.preprocessing import StandardScaler" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "dc7da567-65df-4895-a867-0be05de27ee0", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.manual_seed(42)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "e3340e01-b3bc-4cce-bb21-890517e1bcd5", - "metadata": {}, - "outputs": [], - "source": [ - "housing = fetch_california_housing()" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "dea2ecd8-34e2-4ed5-8bd6-c9d56c951eeb", - "metadata": {}, - "outputs": [], - "source": [ - "class HousingDataset2(torch.utils.data.Dataset):\n", - " def __init__(self, X, y, scale_data=True):\n", - " if not torch.is_tensor(X) and not torch.is_tensor(y):\n", - " # Apply scaling if necessary\n", - " if scale_data:\n", - " X = StandardScaler().fit_transform(X)\n", - " self.X = torch.from_numpy(X.astype(np.float32))\n", - " self.y = torch.from_numpy(y.astype(np.float32))\n", - " \n", - " # Split dataset into separate variables\n", - " self.MedInc = self.X[:,0]\n", - " self.HouseAge = self.X[:,1]\n", - " self.AveRooms = self.X[:,2]\n", - " self.AveBedrms = self.X[:,3]\n", - " self.Population = self.X[:,4]\n", - " self.AveOccup = self.X[:,5]\n", - " self.Latitude = self.X[:,6]\n", - " self.Longitude = self.X[:,7]\n", - "\n", - " def __len__(self):\n", - " return len(self.MedInc)\n", - "\n", - " def __getitem__(self, i):\n", - " # Note: also returning combined X for ease of use later\n", - " return self.MedInc[i], self.HouseAge[i], self.AveRooms[i], self.AveBedrms[i], self.Population[i], self.AveOccup[i], self.Latitude[i], self.Longitude[i], self.y[i]" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "f52f4640-e190-413e-8e01-d67492408f97", - "metadata": {}, - "outputs": [], - "source": [ - "dataset2 = HousingDataset2(housing.data, housing.target)\n", - "trainloader2 = torch.utils.data.DataLoader(dataset2, batch_size=10, shuffle=True, num_workers=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "e2179934-6ae0-4d58-ae2c-d82f90d48074", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[tensor([ 0.4159, -0.3706, -0.2024, 0.1064, 0.1057, 0.3343, -0.2691, -0.0891,\n", - " 0.1397, -0.3399]),\n", - " tensor([ 0.3465, 0.5849, -0.8454, -1.9578, 1.0616, -1.4811, 1.3000, -0.3686,\n", - " -0.2097, 0.3465]),\n", - " tensor([ 0.2294, -0.1514, 0.1928, 0.2968, 0.1675, -0.7187, -0.6491, 0.0260,\n", - " 0.1816, -0.4621]),\n", - " tensor([-0.1113, -0.2040, 0.0087, 0.0363, -0.0081, 0.2041, -0.0872, -0.1563,\n", - " -0.1881, -0.0519]),\n", - " tensor([-0.3130, 0.3943, 0.5435, 2.7556, -0.3651, 0.0967, 1.7074, 0.3996,\n", - " 0.4049, 0.6115]),\n", - " tensor([ 0.0527, 0.0550, -0.0279, -0.0217, -0.0372, 0.0529, -0.1372, 0.0449,\n", - " -0.0229, -0.0284]),\n", - " tensor([-0.5955, -0.8155, 0.9449, -0.4925, -0.6751, -0.8483, -0.7312, 1.1790,\n", - " -0.7547, -0.6704]),\n", - " tensor([ 0.3193, 0.6687, -1.2679, 0.7685, 0.7186, 0.8234, 0.6088, -1.3378,\n", - " 1.2676, 0.5189]),\n", - " tensor([1.8790, 1.1770, 3.3160, 1.5430, 2.3400, 1.5240, 2.8750, 1.2360, 1.2100,\n", - " 2.0530])]" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "next(iter(trainloader2))" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "7fc3383b-fb1c-4daf-9844-88df7abf799d", - "metadata": {}, - "outputs": [], - "source": [ - "class MLP2(nn.Module):\n", - " def __init__(self):\n", - " super().__init__()\n", - " self.layers = nn.Sequential(\n", - " nn.Linear(8, 64),\n", - " nn.ReLU(),\n", - " nn.Linear(64, 32),\n", - " nn.ReLU(),\n", - " nn.Linear(32, 1)\n", - " )\n", - "\n", - " def forward(self, inc, age, rms, bdrms, pop, occup, lat, lon): \n", - " combined = torch.column_stack((inc, age, rms, bdrms, pop, occup, lat, lon))\n", - " return self.layers(combined)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "id": "25e9de54-a8da-46da-ba89-177a75227420", - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize the MLP\n", - "mlp2 = MLP2().to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "id": "631116aa-e496-4125-9970-14aeb816c106", - "metadata": {}, - "outputs": [], - "source": [ - "# Define the loss function and optimizer\n", - "loss_function = nn.L1Loss()\n", - "optimizer = torch.optim.Adam(mlp2.parameters(), lr=1e-4)" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "id": "509d0581-5911-4f21-b0c8-b94523f66dd2", - "metadata": { - "scrolled": true, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Starting epoch 1\n", - "Loss after mini-batch 1: 0.004\n", - "Loss after mini-batch 201: 0.733\n", - "Loss after mini-batch 401: 0.534\n", - "Loss after mini-batch 601: 0.403\n", - "Loss after mini-batch 801: 0.330\n", - "Loss after mini-batch 1001: 0.269\n", - "Loss after mini-batch 1201: 0.232\n", - "Loss after mini-batch 1401: 0.226\n", - "Loss after mini-batch 1601: 0.223\n", - "Loss after mini-batch 1801: 0.214\n", - "Loss after mini-batch 2001: 0.214\n", - "Starting epoch 2\n", - "Loss after mini-batch 1: 0.002\n", - "Loss after mini-batch 201: 0.211\n", - "Loss after mini-batch 401: 0.205\n", - "Loss after mini-batch 601: 0.199\n", - "Loss after mini-batch 801: 0.192\n", - "Loss after mini-batch 1001: 0.194\n", - "Loss after mini-batch 1201: 0.196\n", - "Loss after mini-batch 1401: 0.193\n", - "Loss after mini-batch 1601: 0.194\n", - "Loss after mini-batch 1801: 0.187\n", - "Loss after mini-batch 2001: 0.197\n", - "Starting epoch 3\n", - "Loss after mini-batch 1: 0.001\n", - "Loss after mini-batch 201: 0.190\n", - "Loss after mini-batch 401: 0.183\n", - "Loss after mini-batch 601: 0.181\n", - "Loss after mini-batch 801: 0.190\n", - "Loss after mini-batch 1001: 0.188\n", - "Loss after mini-batch 1201: 0.186\n", - "Loss after mini-batch 1401: 0.193\n", - "Loss after mini-batch 1601: 0.182\n", - "Loss after mini-batch 1801: 0.184\n", - "Loss after mini-batch 2001: 0.188\n", - "Starting epoch 4\n", - "Loss after mini-batch 1: 0.001\n", - "Loss after mini-batch 201: 0.180\n", - "Loss after mini-batch 401: 0.179\n", - "Loss after mini-batch 601: 0.183\n", - "Loss after mini-batch 801: 0.180\n", - "Loss after mini-batch 1001: 0.176\n", - "Loss after mini-batch 1201: 0.189\n", - "Loss after mini-batch 1401: 0.176\n", - "Loss after mini-batch 1601: 0.185\n", - "Loss after mini-batch 1801: 0.177\n", - "Loss after mini-batch 2001: 0.185\n", - "Starting epoch 5\n", - "Loss after mini-batch 1: 0.001\n", - "Loss after mini-batch 201: 0.179\n", - "Loss after mini-batch 401: 0.177\n", - "Loss after mini-batch 601: 0.175\n", - "Loss after mini-batch 801: 0.178\n", - "Loss after mini-batch 1001: 0.173\n", - "Loss after mini-batch 1201: 0.178\n", - "Loss after mini-batch 1401: 0.176\n", - "Loss after mini-batch 1601: 0.174\n", - "Loss after mini-batch 1801: 0.179\n", - "Loss after mini-batch 2001: 0.180\n", - "Training process has finished.\n" - ] - } - ], - "source": [ - "# Run the training loop\n", - "for epoch in range(0, 5): # 5 epochs at maximum\n", - "\n", - " # Print epoch\n", - " print(f'Starting epoch {epoch+1}')\n", - "\n", - " # Set current loss value\n", - " current_loss = 0.0\n", - "\n", - " # Iterate over the DataLoader for training data\n", - " for i, data in enumerate(trainloader2, 0):\n", - "\n", - " # Get and prepare inputs\n", - " a,b,c,d,e,f,g,h,targets = data\n", - " a,b,c,d,e,f,g,h,targets = a.to(device),b.to(device),c.to(device),d.to(device),e.to(device),f.to(device),g.to(device),h.to(device),targets.to(device)\n", - " targets = targets.reshape((targets.shape[0], 1))\n", - "\n", - " # Zero the gradients\n", - " optimizer.zero_grad()\n", - "\n", - " # Perform forward pass\n", - " outputs = mlp2(a,b,c,d,e,f,g,h)\n", - "\n", - " # Compute loss\n", - " loss = loss_function(outputs, targets)\n", - "\n", - " # Perform backward pass\n", - " loss.backward()\n", - "\n", - " # Perform optimization\n", - " optimizer.step()\n", - "\n", - " # Print statistics\n", - " current_loss += loss.item()\n", - " if i % 200 == 0:\n", - " print('Loss after mini-batch %5d: %.3f' %\n", - " (i + 1, current_loss / 500))\n", - " current_loss = 0.0\n", - "\n", - "# Process is complete.\n", - "print('Training process has finished.')" - ] - }, - { - "cell_type": "markdown", - "id": "5029a35d-8fbd-4a11-b3b0-55bcc0a072dd", - "metadata": {}, - "source": [ - "### Save Model State Dict" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "id": "ca720ac4-8b4e-489b-844f-d54dd0659755", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saved PyTorch Model State to housing_model2.pt\n" - ] - } - ], - "source": [ - "torch.save(mlp2.state_dict(), \"housing_model2.pt\")\n", - "print(\"Saved PyTorch Model State to housing_model2.pt\")" - ] - }, - { - "cell_type": "markdown", - "id": "7f677429", - "metadata": {}, - "source": [ - "### Save Model as TorchScript" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "id": "cdcced78-62ee-45fa-b334-6f73a2b21d32", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saved TorchScript Model to ts_housing_model2.pt\n" - ] - } - ], - "source": [ - "scripted = torch.jit.script(mlp2)\n", - "scripted.save(\"ts_housing_model2.pt\")\n", - "print(\"Saved TorchScript Model to ts_housing_model2.pt\")" - ] - }, - { - "cell_type": "markdown", - "id": "8ecb33c2-2e3f-487c-8b12-c4e8f13a67a2", - "metadata": {}, - "source": [ - "### Load and Test from Model State" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "id": "c54b12f2-9981-477b-8c21-a652a1736fc9", - "metadata": {}, - "outputs": [], - "source": [ - "a,b,c,d,e,f,g,h,targets = next(iter(trainloader2))\n", - "a,b,c,d,e,f,g,h,targets = a.to(device), b.to(device), c.to(device), d.to(device), e.to(device), f.to(device), g.to(device), h.to(device), targets.to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "id": "31b2fa69-9a8a-4409-8652-23c547536e50", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 36, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "loaded_mlp2 = MLP2().to(device)\n", - "loaded_mlp2.load_state_dict(torch.load(\"housing_model2.pt\", weights_only=True))" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "id": "80a21d52-ea98-4c74-98e8-1e088cdfa742", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[2.8778],\n", - " [0.6233],\n", - " [3.9021],\n", - " [2.4543],\n", - " [1.0209],\n", - " [1.8093],\n", - " [1.4593],\n", - " [3.2933],\n", - " [2.9263],\n", - " [1.4790]], device='cuda:0', grad_fn=)" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "loaded_mlp2(a,b,c,d,e,f,g,h)" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "id": "90d29fb0-5923-4684-8aa6-62618e8f1ef6", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(inc, age, rms, bdrms, pop, occup, lat, lon)\n" - ] - } - ], - "source": [ - "print(signature(loaded_mlp2.forward))" - ] - }, - { - "cell_type": "markdown", - "id": "de84bc12", - "metadata": {}, - "source": [ - "### Load and Test from TorchScript" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "id": "165772b2-8277-4b6e-a178-c99ea2a031fa", - "metadata": {}, - "outputs": [], - "source": [ - "scripted_mlp2 = torch.jit.load(\"ts_housing_model2.pt\")" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "id": "e53d927e-fa6d-419d-b570-8ca9b0756812", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([[2.8778],\n", - " [0.6233],\n", - " [3.9021],\n", - " [2.4543],\n", - " [1.0209],\n", - " [1.8093],\n", - " [1.4593],\n", - " [3.2933],\n", - " [2.9263],\n", - " [1.4790]], device='cuda:0', grad_fn=)" - ] - }, - "execution_count": 40, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "scripted_mlp2(a,b,c,d,e,f,g,h)" - ] - }, - { - "cell_type": "markdown", - "id": "13631d1f-2c71-4bee-afcb-bd3b55ec87c5", - "metadata": {}, - "source": [ - "## PySpark" - ] - }, - { - "cell_type": "markdown", - "id": "e3b9937e-2c70-4d67-b95f-4d9d5ab17c12", - "metadata": {}, - "source": [ - "### Convert dataset to Spark DataFrame" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "id": "cf35da14-61a3-4e7b-9d4f-086bf5e931b3", - "metadata": {}, - "outputs": [], - "source": [ - "housing = fetch_california_housing()" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "id": "95148019-ea95-40e5-a529-fcdb9a06f928", - "metadata": {}, - "outputs": [], - "source": [ - "X = StandardScaler().fit_transform(housing.data.astype(np.float32))" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "id": "f82d957c-6747-4408-aac8-45305afbfe5e", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
MedIncHouseAgeAveRoomsAveBedrmsPopulationAveOccupLatitudeLongitude
02.3447660.9821430.628559-0.153758-0.974429-0.0495971.052549-1.327837
12.332238-0.6070190.327041-0.2633360.861439-0.0925121.043185-1.322845
21.7826991.8561821.155620-0.049016-0.820777-0.0258431.038502-1.332825
30.9329671.8561820.156966-0.049833-0.766028-0.0503291.038502-1.337818
4-0.0128811.8561820.344711-0.032906-0.759847-0.0856161.038502-1.337818
...........................
20635-1.216128-0.289187-0.1550230.077354-0.512592-0.0491101.801647-0.758824
20636-0.691593-0.8453930.2768810.462365-0.9444050.0050211.806329-0.818721
20637-1.142593-0.924851-0.0903180.049414-0.369537-0.0717341.778238-0.823714
20638-1.054583-0.845393-0.0402110.158778-0.604429-0.0912251.778238-0.873626
20639-0.780129-1.004309-0.0704430.138403-0.033977-0.0436821.750146-0.833695
\n", - "

20640 rows × 8 columns

\n", - "
" - ], - "text/plain": [ - " MedInc HouseAge AveRooms AveBedrms Population AveOccup \\\n", - "0 2.344766 0.982143 0.628559 -0.153758 -0.974429 -0.049597 \n", - "1 2.332238 -0.607019 0.327041 -0.263336 0.861439 -0.092512 \n", - "2 1.782699 1.856182 1.155620 -0.049016 -0.820777 -0.025843 \n", - "3 0.932967 1.856182 0.156966 -0.049833 -0.766028 -0.050329 \n", - "4 -0.012881 1.856182 0.344711 -0.032906 -0.759847 -0.085616 \n", - "... ... ... ... ... ... ... \n", - "20635 -1.216128 -0.289187 -0.155023 0.077354 -0.512592 -0.049110 \n", - "20636 -0.691593 -0.845393 0.276881 0.462365 -0.944405 0.005021 \n", - "20637 -1.142593 -0.924851 -0.090318 0.049414 -0.369537 -0.071734 \n", - "20638 -1.054583 -0.845393 -0.040211 0.158778 -0.604429 -0.091225 \n", - "20639 -0.780129 -1.004309 -0.070443 0.138403 -0.033977 -0.043682 \n", - "\n", - " Latitude Longitude \n", - "0 1.052549 -1.327837 \n", - "1 1.043185 -1.322845 \n", - "2 1.038502 -1.332825 \n", - "3 1.038502 -1.337818 \n", - "4 1.038502 -1.337818 \n", - "... ... ... \n", - "20635 1.801647 -0.758824 \n", - "20636 1.806329 -0.818721 \n", - "20637 1.778238 -0.823714 \n", - "20638 1.778238 -0.873626 \n", - "20639 1.750146 -0.833695 \n", - "\n", - "[20640 rows x 8 columns]" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pdf = pd.DataFrame(X, columns=housing.feature_names)\n", - "pdf" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "id": "5ba338cd-76d2-46bd-baf5-7d18a339a449", - "metadata": {}, - "outputs": [], - "source": [ - "foo = pdf.to_dict('series')" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "id": "224b5036-d2ed-4edf-975f-66127862343d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keys(['MedInc', 'HouseAge', 'AveRooms', 'AveBedrms', 'Population', 'AveOccup', 'Latitude', 'Longitude'])" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "foo.keys()" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "id": "0b32ea98-a7f1-4011-a067-700377f1717f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "MedInc float32\n", - "HouseAge float32\n", - "AveRooms float32\n", - "AveBedrms float32\n", - "Population float32\n", - "AveOccup float32\n", - "Latitude float32\n", - "Longitude float32\n", - "dtype: object" - ] - }, - "execution_count": 46, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pdf.dtypes" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "id": "e630236c", - "metadata": {}, - "outputs": [], - "source": [ - "from pyspark.sql.types import *\n", - "from pyspark.sql import SparkSession\n", - "from pyspark import SparkConf" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "id": "6388cce9-6469-4f5a-898a-1a0b74eec438", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "24/10/08 00:36:22 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "24/10/08 00:36:22 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", - "Setting default log level to \"WARN\".\n", - "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "24/10/08 00:36:22 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" - ] - } - ], - "source": [ - "import os\n", - "conda_env = os.environ.get(\"CONDA_PREFIX\")\n", - "\n", - "conf = SparkConf()\n", - "if 'spark' not in globals():\n", - " # If Spark is not already started with Jupyter, attach to Spark Standalone\n", - " import socket\n", - " hostname = socket.gethostname()\n", - " conf.setMaster(f\"spark://{hostname}:7077\") # assuming Master is on default port 7077\n", - "conf.set(\"spark.task.maxFailures\", \"1\")\n", - "conf.set(\"spark.driver.memory\", \"8g\")\n", - "conf.set(\"spark.executor.memory\", \"8g\")\n", - "conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", - "conf.set(\"spark.python.worker.reuse\", \"true\")\n", - "# Create Spark Session\n", - "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", - "sc = spark.sparkContext" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "id": "881afee9", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/pyspark/sql/pandas/serializers.py:224: DeprecationWarning: is_categorical_dtype is deprecated and will be removed in a future version. Use isinstance(dtype, pd.CategoricalDtype) instead\n", - " if is_categorical_dtype(series.dtype):\n", - "\n", - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------+------------+-----------+------------+-----------+------------+----------+------------+\n", - "| MedInc| HouseAge| AveRooms| AveBedrms| Population| AveOccup| Latitude| Longitude|\n", - "+------------+------------+-----------+------------+-----------+------------+----------+------------+\n", - "| 0.20909257| -1.1632254| 0.38946992| 0.04609274| -0.9806099| -0.07099328|0.61245227|-0.020113053|\n", - "|-0.098627955| 0.34647804| 0.27216315| -0.0129226| -0.6953838| -0.05380849| 1.0665938| -1.2479742|\n", - "| -0.66006273| 1.0616008|-0.55292207| -0.48945764|-0.13641118| 0.028952759| 1.1040496| -1.3827378|\n", - "| 0.08218294| 0.5848523|-0.13912922| -0.14707813|-0.19116047| -0.07136432|0.96827507| -1.3028787|\n", - "| 0.0784456| -1.4810578| 0.57265776| 0.32067496| 1.0345173|-0.024157424| 1.4411427| -0.52423614|\n", - "| -0.82318723| -0.36864465| 0.07829511| -0.1808107|-0.67242444|-0.061470542| 1.9374212| -1.0083897|\n", - "| 0.59671736| 0.5848523| 0.19346413| -0.1371872|-0.19645879| 0.009964322|0.96827507| -1.2928978|\n", - "| -0.9612035| -1.5605159|-0.56329846| 0.027148023|-0.71127874| -0.08471591| 0.5328614| -0.13990337|\n", - "| -0.74344087| -1.2426835| 0.27282518| 0.4037246| -0.9841421| -0.05610115| 1.2257773| -0.42940006|\n", - "| 0.9784464| -0.2891866| 0.24374022| -0.24670053| 0.28922042| -0.01102468| 1.1087307| -1.2280084|\n", - "| -0.5070446| -1.0043093|-0.78254056|0.0122275995| 2.8465424|-0.060435444| 0.8980464| -1.2080427|\n", - "| -0.18690155| 1.2205169|0.015323491| 0.12183313|-0.41015765| 0.04452552| 1.010412| -1.3228445|\n", - "| -1.2551856| 1.6178073| -0.3341509|-0.060125165| -0.7554314| -0.08777025| 1.0291398| -1.3477987|\n", - "| 4.9607058| -1.9578062| 1.4854684| -0.03948475| 2.1833694|0.0029250523| 1.024457| -1.1581304|\n", - "| 0.73652315| -1.6399739| 0.7913185| -0.05238397| 1.67738| 0.01944797| 1.0993668| -1.1331724|\n", - "| -0.505834| 0.18756187|-0.47093546| -0.24297306|-0.60619545| -0.10791535| 0.977639| -1.2879055|\n", - "| -0.88477343|-0.050812364| -0.6318951| -0.15244243| -0.5258376| -0.15618815| 0.9823201| -1.2879055|\n", - "| -0.42840376| 0.9821427| -0.2266495| -0.36083496| -0.6883194| -0.08552282| 0.5328614| -0.12493005|\n", - "| 0.9369153| -1.4810578| 0.6722208|-0.121177554| 0.3996021| 0.01291408| 1.1040496| -1.1082181|\n", - "| -0.80702734| -0.92485124|-0.26602685| -0.1560743| 1.4398388| -0.09314839|0.55627036| -0.09498342|\n", - "+------------+------------+-----------+------------+-----------+------------+----------+------------+\n", - "only showing top 20 rows\n", - "\n" - ] - } - ], - "source": [ - "# Spark is somehow auto-converting Pandas float32 to DoubleType(), so forcing FloatType()\n", - "schema = StructType([\n", - "StructField(\"MedInc\",FloatType(),True),\n", - "StructField(\"HouseAge\",FloatType(),True),\n", - "StructField(\"AveRooms\",FloatType(),True),\n", - "StructField(\"AveBedrms\",FloatType(),True),\n", - "StructField(\"Population\",FloatType(),True),\n", - "StructField(\"AveOccup\",FloatType(),True),\n", - "StructField(\"Latitude\",FloatType(),True),\n", - "StructField(\"Longitude\",FloatType(),True)\n", - "])\n", - "\n", - "df = spark.createDataFrame(pdf, schema=schema).repartition(8)\n", - "df.show(truncate=12)" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "id": "7b33d367-fbf9-4918-b755-5447125547c4", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "StructType([StructField('MedInc', FloatType(), True), StructField('HouseAge', FloatType(), True), StructField('AveRooms', FloatType(), True), StructField('AveBedrms', FloatType(), True), StructField('Population', FloatType(), True), StructField('AveOccup', FloatType(), True), StructField('Latitude', FloatType(), True), StructField('Longitude', FloatType(), True)])" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.schema" - ] - }, - { - "cell_type": "markdown", - "id": "21c9932f-c21f-4eda-954e-26c38925ff84", - "metadata": {}, - "source": [ - "### Save DataFrame as parquet" - ] - }, - { - "cell_type": "code", - "execution_count": 51, - "id": "751bff7a-b687-4184-b3fa-b5f5b46ef5d1", - "metadata": {}, - "outputs": [], - "source": [ - "df.write.mode(\"overwrite\").parquet(\"california_housing\")" - ] - }, - { - "cell_type": "markdown", - "id": "a80bb9ee-27fd-4604-89f8-6b438af0b984", - "metadata": {}, - "source": [ - "## Inference using Spark DL API\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment." - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "id": "986d1a97-ea84-4707-b94a-78498780c47c", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import array, struct, col\n", - "from pyspark.sql.types import ArrayType, FloatType" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "id": "1e40c266-24de-454d-a776-f3716ba50e90", - "metadata": {}, - "outputs": [], - "source": [ - "df = spark.read.parquet(\"california_housing\")" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "id": "ac802fb6-f159-4776-b55d-b9c421e8c57e", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['MedInc',\n", - " 'HouseAge',\n", - " 'AveRooms',\n", - " 'AveBedrms',\n", - " 'Population',\n", - " 'AveOccup',\n", - " 'Latitude',\n", - " 'Longitude']" - ] - }, - "execution_count": 54, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "columns = df.columns\n", - "columns" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "id": "4b8de001-e791-4a91-bd6f-c80bdf1c4472", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------+------------+-----------+------------+-----------+------------+----------+------------+\n", - "| MedInc| HouseAge| AveRooms| AveBedrms| Population| AveOccup| Latitude| Longitude|\n", - "+------------+------------+-----------+------------+-----------+------------+----------+------------+\n", - "| 0.20909257| -1.1632254| 0.38946992| 0.04609274| -0.9806099| -0.07099328|0.61245227|-0.020113053|\n", - "|-0.098627955| 0.34647804| 0.27216315| -0.0129226| -0.6953838| -0.05380849| 1.0665938| -1.2479742|\n", - "| -0.66006273| 1.0616008|-0.55292207| -0.48945764|-0.13641118| 0.028952759| 1.1040496| -1.3827378|\n", - "| 0.08218294| 0.5848523|-0.13912922| -0.14707813|-0.19116047| -0.07136432|0.96827507| -1.3028787|\n", - "| 0.0784456| -1.4810578| 0.57265776| 0.32067496| 1.0345173|-0.024157424| 1.4411427| -0.52423614|\n", - "| -0.82318723| -0.36864465| 0.07829511| -0.1808107|-0.67242444|-0.061470542| 1.9374212| -1.0083897|\n", - "| 0.59671736| 0.5848523| 0.19346413| -0.1371872|-0.19645879| 0.009964322|0.96827507| -1.2928978|\n", - "| -0.9612035| -1.5605159|-0.56329846| 0.027148023|-0.71127874| -0.08471591| 0.5328614| -0.13990337|\n", - "| -0.74344087| -1.2426835| 0.27282518| 0.4037246| -0.9841421| -0.05610115| 1.2257773| -0.42940006|\n", - "| 0.9784464| -0.2891866| 0.24374022| -0.24670053| 0.28922042| -0.01102468| 1.1087307| -1.2280084|\n", - "| -0.5070446| -1.0043093|-0.78254056|0.0122275995| 2.8465424|-0.060435444| 0.8980464| -1.2080427|\n", - "| -0.18690155| 1.2205169|0.015323491| 0.12183313|-0.41015765| 0.04452552| 1.010412| -1.3228445|\n", - "| -1.2551856| 1.6178073| -0.3341509|-0.060125165| -0.7554314| -0.08777025| 1.0291398| -1.3477987|\n", - "| 4.9607058| -1.9578062| 1.4854684| -0.03948475| 2.1833694|0.0029250523| 1.024457| -1.1581304|\n", - "| 0.73652315| -1.6399739| 0.7913185| -0.05238397| 1.67738| 0.01944797| 1.0993668| -1.1331724|\n", - "| -0.505834| 0.18756187|-0.47093546| -0.24297306|-0.60619545| -0.10791535| 0.977639| -1.2879055|\n", - "| -0.88477343|-0.050812364| -0.6318951| -0.15244243| -0.5258376| -0.15618815| 0.9823201| -1.2879055|\n", - "| -0.42840376| 0.9821427| -0.2266495| -0.36083496| -0.6883194| -0.08552282| 0.5328614| -0.12493005|\n", - "| 0.9369153| -1.4810578| 0.6722208|-0.121177554| 0.3996021| 0.01291408| 1.1040496| -1.1082181|\n", - "| -0.80702734| -0.92485124|-0.26602685| -0.1560743| 1.4398388| -0.09314839|0.55627036| -0.09498342|\n", - "+------------+------------+-----------+------------+-----------+------------+----------+------------+\n", - "only showing top 20 rows\n", - "\n" - ] - } - ], - "source": [ - "df.show()" - ] - }, - { - "cell_type": "markdown", - "id": "650dda22-31b7-419b-96d4-387a036f3b07", - "metadata": {}, - "source": [ - "### Using TensorRT Model (single input)" - ] - }, - { - "cell_type": "code", - "execution_count": 56, - "id": "3d608e2f-66a8-44b5-9cde-5f7837bf4247", - "metadata": {}, - "outputs": [], - "source": [ - "# get absolute path to model\n", - "model_dir = \"{}/housing_model.pt\".format(os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "id": "a2f45f5d-c941-4197-a274-1eec2af3fca4", - "metadata": {}, - "outputs": [], - "source": [ - "def predict_batch_fn():\n", - " import torch\n", - " import torch_tensorrt as trt\n", - "\n", - " device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", - " if device != \"cuda\":\n", - " raise ValueError(\"This function uses the TensorRT model which requires a GPU device\")\n", - "\n", - " # Define model\n", - " class MLP(nn.Module):\n", - " def __init__(self):\n", - " super().__init__()\n", - " self.layers = nn.Sequential(\n", - " nn.Linear(8, 64),\n", - " nn.ReLU(),\n", - " nn.Linear(64, 32),\n", - " nn.ReLU(),\n", - " nn.Linear(32, 1)\n", - " )\n", - "\n", - " def forward(self, x):\n", - " return self.layers(x)\n", - "\n", - " model = MLP().to(device)\n", - " model.load_state_dict(torch.load(model_dir, weights_only=True))\n", - "\n", - " # Preparing the inputs for dynamic batch sizing.\n", - " inputs = [trt.Input(min_shape=(20, 8), \n", - " opt_shape=(50, 8), \n", - " max_shape=(64, 8), \n", - " dtype=torch.float32)]\n", - "\n", - " # Trace the computation graph and compile to produce an optimized module\n", - " trt_gm = trt.compile(model, ir=\"dynamo\", inputs=inputs)\n", - " \n", - " def predict(inputs):\n", - " stream = torch.cuda.Stream()\n", - " with torch.no_grad(), torch.cuda.stream(stream), trt.logging.errors():\n", - " torch_inputs = torch.from_numpy(inputs).to(device)\n", - " outputs = trt_gm(torch_inputs) # .flatten()\n", - " return outputs.detach().cpu().numpy()\n", - "\n", - " return predict" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "id": "220a00a4-e842-4f5d-a4b3-7693d09e2d31", - "metadata": {}, - "outputs": [], - "source": [ - "classify = predict_batch_udf(predict_batch_fn,\n", - " return_type=FloatType(),\n", - " input_tensor_shapes=[[8]],\n", - " batch_size=50)" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "id": "0f3bf287-8ffc-4456-8772-e97c418d6aee", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 148 ms, sys: 17.9 ms, total: 166 ms\n", - "Wall time: 8.28 s\n" - ] - } - ], - "source": [ - "%%time\n", - "preds = df.withColumn(\"preds\", classify(struct(*columns)))\n", - "results = preds.collect()" - ] - }, - { - "cell_type": "code", - "execution_count": 61, - "id": "6cd23b71-296d-4ce7-b56c-567cc2eec79c", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 31.2 ms, sys: 5.84 ms, total: 37.1 ms\n", - "Wall time: 257 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "preds = df.withColumn(\"preds\", classify(array(*columns)))\n", - "results = preds.collect()" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "id": "13c52980-fc55-4e81-ae54-b476b98f11b1", - "metadata": {}, - "outputs": [], - "source": [ - "# should raise ValueError\n", - "# preds = df.withColumn(\"preds\", classify(*columns))\n", - "# results = preds.collect()" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "id": "764a40d8-25f7-425c-ba03-fe8c45f4b063", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", - "| MedInc| HouseAge| AveRooms| AveBedrms| Population| AveOccup| Latitude| Longitude| preds|\n", - "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", - "| 0.20909257| -1.1632254| 0.38946992| 0.04609274| -0.9806099| -0.07099328|0.61245227|-0.020113053| 1.3980268|\n", - "|-0.098627955| 0.34647804| 0.27216315| -0.0129226| -0.6953838| -0.05380849| 1.0665938| -1.2479742| 1.7447104|\n", - "| -0.66006273| 1.0616008|-0.55292207| -0.48945764|-0.13641118| 0.028952759| 1.1040496| -1.3827378| 1.439564|\n", - "| 0.08218294| 0.5848523|-0.13912922| -0.14707813|-0.19116047| -0.07136432|0.96827507| -1.3028787| 2.4199378|\n", - "| 0.0784456| -1.4810578| 0.57265776| 0.32067496| 1.0345173|-0.024157424| 1.4411427| -0.52423614| 1.2448893|\n", - "| -0.82318723| -0.36864465| 0.07829511| -0.1808107|-0.67242444|-0.061470542| 1.9374212| -1.0083897|0.68910843|\n", - "| 0.59671736| 0.5848523| 0.19346413| -0.1371872|-0.19645879| 0.009964322|0.96827507| -1.2928978| 2.656445|\n", - "| -0.9612035| -1.5605159|-0.56329846| 0.027148023|-0.71127874| -0.08471591| 0.5328614| -0.13990337| 1.13419|\n", - "| -0.74344087| -1.2426835| 0.27282518| 0.4037246| -0.9841421| -0.05610115| 1.2257773| -0.42940006| 1.1380601|\n", - "| 0.9784464| -0.2891866| 0.24374022| -0.24670053| 0.28922042| -0.01102468| 1.1087307| -1.2280084| 2.5711632|\n", - "| -0.5070446| -1.0043093|-0.78254056|0.0122275995| 2.8465424|-0.060435444| 0.8980464| -1.2080427| 1.8561494|\n", - "| -0.18690155| 1.2205169|0.015323491| 0.12183313|-0.41015765| 0.04452552| 1.010412| -1.3228445| 1.8643656|\n", - "| -1.2551856| 1.6178073| -0.3341509|-0.060125165| -0.7554314| -0.08777025| 1.0291398| -1.3477987| 1.2487215|\n", - "| 4.9607058| -1.9578062| 1.4854684| -0.03948475| 2.1833694|0.0029250523| 1.024457| -1.1581304| 5.595224|\n", - "| 0.73652315| -1.6399739| 0.7913185| -0.05238397| 1.67738| 0.01944797| 1.0993668| -1.1331724| 2.069084|\n", - "| -0.505834| 0.18756187|-0.47093546| -0.24297306|-0.60619545| -0.10791535| 0.977639| -1.2879055| 1.7858529|\n", - "| -0.88477343|-0.050812364| -0.6318951| -0.15244243| -0.5258376| -0.15618815| 0.9823201| -1.2879055| 1.6675146|\n", - "| -0.42840376| 0.9821427| -0.2266495| -0.36083496| -0.6883194| -0.08552282| 0.5328614| -0.12493005| 1.01702|\n", - "| 0.9369153| -1.4810578| 0.6722208|-0.121177554| 0.3996021| 0.01291408| 1.1040496| -1.1082181| 2.1314554|\n", - "| -0.80702734| -0.92485124|-0.26602685| -0.1560743| 1.4398388| -0.09314839|0.55627036| -0.09498342| 0.8631196|\n", - "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", - "only showing top 20 rows\n", - "\n" - ] - } - ], - "source": [ - "preds.show()" - ] - }, - { - "cell_type": "markdown", - "id": "773953dd-e645-4848-8f33-ed82f8242a43", - "metadata": {}, - "source": [ - "### Using TorchScript Model (separate input variables)" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "id": "1a69a9d2-5c7f-4e71-bb65-ae51927ccacf", - "metadata": {}, - "outputs": [], - "source": [ - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import struct, col\n", - "from pyspark.sql.types import ArrayType, FloatType" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "id": "7214e2ac-fd2c-473e-a9c7-a65488570b5c", - "metadata": {}, - "outputs": [], - "source": [ - "df = spark.read.parquet(\"california_housing\")" - ] - }, - { - "cell_type": "code", - "execution_count": 66, - "id": "5ee170b9-8ba6-4681-a10c-4cea71c1be15", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['MedInc',\n", - " 'HouseAge',\n", - " 'AveRooms',\n", - " 'AveBedrms',\n", - " 'Population',\n", - " 'AveOccup',\n", - " 'Latitude',\n", - " 'Longitude']" - ] - }, - "execution_count": 66, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "columns = df.columns\n", - "columns" - ] - }, - { - "cell_type": "code", - "execution_count": 67, - "id": "0b7af043-5f20-49c7-bed6-39a9d13988e4", - "metadata": {}, - "outputs": [], - "source": [ - "# get absolute path to model\n", - "model2_dir = \"{}/ts_housing_model2.pt\".format(os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "id": "104b2378-e191-4560-9a2e-276b8dcf0f2b", - "metadata": {}, - "outputs": [], - "source": [ - "def predict_batch_fn():\n", - " import torch\n", - "\n", - " device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", - " scripted_mlp = torch.jit.load(model2_dir)\n", - " scripted_mlp.to(device)\n", - " \n", - " def predict(inc, age, rms, bdrms, pop, occ, lat, lon):\n", - " # print input shape\n", - " outputs = scripted_mlp(\n", - " torch.from_numpy(inc).to(device),\n", - " torch.from_numpy(age).to(device),\n", - " torch.from_numpy(rms).to(device),\n", - " torch.from_numpy(bdrms).to(device),\n", - " torch.from_numpy(pop).to(device),\n", - " torch.from_numpy(occ).to(device),\n", - " torch.from_numpy(lat).to(device),\n", - " torch.from_numpy(lon).to(device),\n", - " )\n", - " return outputs.detach().cpu().numpy()\n", - "\n", - " return predict" - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "id": "020056dc-f8b0-483a-88eb-7e1ff2a0fdcf", - "metadata": {}, - "outputs": [], - "source": [ - "classify = predict_batch_udf(predict_batch_fn,\n", - " return_type=FloatType(),\n", - " batch_size=50)" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "id": "1b73518e-04ec-49c7-bf1e-93520d94028e", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 12:==================================================> (7 + 1) / 8]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.7 ms, sys: 5.44 ms, total: 22.1 ms\n", - "Wall time: 1.13 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - } - ], - "source": [ - "%%time\n", - "# first pass caches model/fn\n", - "preds = df.withColumn(\"preds\", classify(struct(*columns)))\n", - "results = preds.collect()" - ] - }, - { - "cell_type": "code", - "execution_count": 71, - "id": "86b56805-a211-43cb-878d-78957b08f865", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 19.6 ms, sys: 8.68 ms, total: 28.3 ms\n", - "Wall time: 451 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "preds = df.withColumn(\"preds\", classify(*columns))\n", - "results = preds.collect()" - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "id": "5032b474-db92-4f04-b732-8b9d418cf211", - "metadata": { - "scrolled": true, - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", - "| MedInc| HouseAge| AveRooms| AveBedrms| Population| AveOccup| Latitude| Longitude| preds|\n", - "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", - "| 0.20909257| -1.1632254| 0.38946992| 0.04609274| -0.9806099| -0.07099328|0.61245227|-0.020113053| 1.3979516|\n", - "|-0.098627955| 0.34647804| 0.27216315| -0.0129226| -0.6953838| -0.05380849| 1.0665938| -1.2479742| 1.7442212|\n", - "| -0.66006273| 1.0616008|-0.55292207| -0.48945764|-0.13641118| 0.028952759| 1.1040496| -1.3827378| 1.4398992|\n", - "| 0.08218294| 0.5848523|-0.13912922| -0.14707813|-0.19116047| -0.07136432|0.96827507| -1.3028787| 2.4199052|\n", - "| 0.0784456| -1.4810578| 0.57265776| 0.32067496| 1.0345173|-0.024157424| 1.4411427| -0.52423614| 1.2446644|\n", - "| -0.82318723| -0.36864465| 0.07829511| -0.1808107|-0.67242444|-0.061470542| 1.9374212| -1.0083897| 0.6888372|\n", - "| 0.59671736| 0.5848523| 0.19346413| -0.1371872|-0.19645879| 0.009964322|0.96827507| -1.2928978| 2.6563153|\n", - "| -0.9612035| -1.5605159|-0.56329846| 0.027148023|-0.71127874| -0.08471591| 0.5328614| -0.13990337| 1.1341839|\n", - "| -0.74344087| -1.2426835| 0.27282518| 0.4037246| -0.9841421| -0.05610115| 1.2257773| -0.42940006| 1.1378745|\n", - "| 0.9784464| -0.2891866| 0.24374022| -0.24670053| 0.28922042| -0.01102468| 1.1087307| -1.2280084| 2.5710382|\n", - "| -0.5070446| -1.0043093|-0.78254056|0.0122275995| 2.8465424|-0.060435444| 0.8980464| -1.2080427| 1.8561647|\n", - "| -0.18690155| 1.2205169|0.015323491| 0.12183313|-0.41015765| 0.04452552| 1.010412| -1.3228445| 1.8639375|\n", - "| -1.2551856| 1.6178073| -0.3341509|-0.060125165| -0.7554314| -0.08777025| 1.0291398| -1.3477987| 1.24879|\n", - "| 4.9607058| -1.9578062| 1.4854684| -0.03948475| 2.1833694|0.0029250523| 1.024457| -1.1581304| 5.5946765|\n", - "| 0.73652315| -1.6399739| 0.7913185| -0.05238397| 1.67738| 0.01944797| 1.0993668| -1.1331724| 2.0694952|\n", - "| -0.505834| 0.18756187|-0.47093546| -0.24297306|-0.60619545| -0.10791535| 0.977639| -1.2879055| 1.7852836|\n", - "| -0.88477343|-0.050812364| -0.6318951| -0.15244243| -0.5258376| -0.15618815| 0.9823201| -1.2879055| 1.6675682|\n", - "| -0.42840376| 0.9821427| -0.2266495| -0.36083496| -0.6883194| -0.08552282| 0.5328614| -0.12493005| 1.0169572|\n", - "| 0.9369153| -1.4810578| 0.6722208|-0.121177554| 0.3996021| 0.01291408| 1.1040496| -1.1082181| 2.1317377|\n", - "| -0.80702734| -0.92485124|-0.26602685| -0.1560743| 1.4398388| -0.09314839|0.55627036| -0.09498342|0.86303747|\n", - "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", - "only showing top 20 rows\n", - "\n" - ] - } - ], - "source": [ - "preds.show()" - ] - }, - { - "cell_type": "markdown", - "id": "c1445875-fa94-4318-9bb8-0b9bfab0a795", - "metadata": {}, - "source": [ - "### Using Triton Inference Server\n", - "\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment." - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "id": "a9ab4cdf-8103-447e-9ac8-944e2e527239", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "from functools import partial\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import struct, col, array\n", - "from pyspark.sql.types import ArrayType, FloatType, Union, Dict" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "id": "6632636e-67a3-406c-832c-758aac4245fd", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "%%bash\n", - "# copy custom model to expected layout for Triton\n", - "rm -rf models\n", - "mkdir models\n", - "cp -r models_config/housing_model models\n", - "mkdir -p models/housing_model/1\n", - "cp ts_housing_model.pt models/housing_model/1/model.pt" - ] - }, - { - "cell_type": "markdown", - "id": "a99f8022-d9a4-4f60-bfa0-c37241d24292", - "metadata": {}, - "source": [ - "#### Start Triton Server on each executor" - ] - }, - { - "cell_type": "code", - "execution_count": 75, - "id": "c6fd1612-de6a-461c-a2ad-1a3fcd277d66", - "metadata": { - "scrolled": true, - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "data": { - "text/plain": [ - "[True]" - ] - }, - "execution_count": 75, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "num_executors = 1\n", - "triton_models_dir = \"{}/models\".format(os.getcwd())\n", - "nodeRDD = sc.parallelize(list(range(num_executors)), num_executors)\n", - "\n", - "def start_triton(it):\n", - " import docker\n", - " import time\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " if containers:\n", - " print(\">>>> containers: {}\".format([c.short_id for c in containers]))\n", - " else:\n", - " container=client.containers.run(\n", - " \"nvcr.io/nvidia/tritonserver:24.08-py3\", \"tritonserver --model-repository=/models\",\n", - " detach=True,\n", - " device_requests=[docker.types.DeviceRequest(device_ids=[\"0\"], capabilities=[['gpu']])],\n", - " name=\"spark-triton\",\n", - " network_mode=\"host\",\n", - " remove=True,\n", - " shm_size=\"64M\",\n", - " volumes={triton_models_dir: {\"bind\": \"/models\", \"mode\": \"ro\"}}\n", - " )\n", - " print(\">>>> starting triton: {}\".format(container.short_id))\n", - "\n", - " # wait for triton to be running\n", - " time.sleep(15)\n", - " client = grpcclient.InferenceServerClient(\"localhost:8001\")\n", - " ready = False\n", - " while not ready:\n", - " try:\n", - " ready = client.is_server_ready()\n", - " except Exception as e:\n", - " time.sleep(5)\n", - " \n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(start_triton).collect()" - ] - }, - { - "cell_type": "markdown", - "id": "20b8514e-01de-481f-86aa-75afd99bcc7c", - "metadata": {}, - "source": [ - "### Run Inference" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "id": "5eae04bc-75ca-421a-87c8-ac507ce1f2f5", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "df = spark.read.parquet(\"california_housing\")" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "id": "b350bd8e-9b8f-4511-9ddf-76d917b21b5f", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "['MedInc',\n", - " 'HouseAge',\n", - " 'AveRooms',\n", - " 'AveBedrms',\n", - " 'Population',\n", - " 'AveOccup',\n", - " 'Latitude',\n", - " 'Longitude']" - ] - }, - "execution_count": 77, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "columns = df.columns\n", - "columns" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "id": "69b343ec-688d-4e4d-985e-db72beaaf00c", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "def triton_fn(triton_uri, model_name):\n", - " import numpy as np\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " np_types = {\n", - " \"BOOL\": np.dtype(np.bool_),\n", - " \"INT8\": np.dtype(np.int8),\n", - " \"INT16\": np.dtype(np.int16),\n", - " \"INT32\": np.dtype(np.int32),\n", - " \"INT64\": np.dtype(np.int64),\n", - " \"FP16\": np.dtype(np.float16),\n", - " \"FP32\": np.dtype(np.float32),\n", - " \"FP64\": np.dtype(np.float64),\n", - " \"FP64\": np.dtype(np.double),\n", - " \"BYTES\": np.dtype(object)\n", - " }\n", - "\n", - " client = grpcclient.InferenceServerClient(triton_uri)\n", - " model_meta = client.get_model_metadata(model_name)\n", - " \n", - " def predict(inputs):\n", - " if isinstance(inputs, np.ndarray):\n", - " # single ndarray input\n", - " request = [grpcclient.InferInput(model_meta.inputs[0].name, inputs.shape, model_meta.inputs[0].datatype)]\n", - " request[0].set_data_from_numpy(inputs.astype(np_types[model_meta.inputs[0].datatype]))\n", - " else:\n", - " # dict of multiple ndarray inputs\n", - " request = [grpcclient.InferInput(i.name, inputs[i.name].shape, i.datatype) for i in model_meta.inputs]\n", - " for i in request:\n", - " i.set_data_from_numpy(inputs[i.name()].astype(np_types[i.datatype()]))\n", - " \n", - " response = client.infer(model_name, inputs=request)\n", - " \n", - " if len(model_meta.outputs) > 1:\n", - " # return dictionary of numpy arrays\n", - " return {o.name: response.as_numpy(o.name) for o in model_meta.outputs}\n", - " else:\n", - " # return single numpy array\n", - " return response.as_numpy(model_meta.outputs[0].name)\n", - " \n", - " return predict" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "id": "d3e64fda-117b-4810-a9a2-dd498239496f", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "classify = predict_batch_udf(partial(triton_fn, triton_uri=\"localhost:8001\", model_name=\"housing_model\"),\n", - " return_type=FloatType(),\n", - " input_tensor_shapes=[[8]],\n", - " batch_size=500)" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "id": "a24149a5-3adc-4089-8769-13cf1e44547a", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 15.3 ms, sys: 4.33 ms, total: 19.6 ms\n", - "Wall time: 461 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "# first pass caches model/fn\n", - "predictions = df.withColumn(\"preds\", classify(struct(*columns)))\n", - "preds = predictions.collect()" - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "id": "df2ce39f-30af-491a-8472-800fb1ce8458", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 30.2 ms, sys: 5.78 ms, total: 36 ms\n", - "Wall time: 200 ms\n" - ] - } - ], - "source": [ - "%%time\n", - "predictions = df.withColumn(\"preds\", classify(array(*columns)))\n", - "preds = predictions.collect()" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "id": "ca6f3eaa-9569-45d0-88bf-9aa0757e1ecb", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "# should raise ValueError\n", - "# predictions = df.withColumn(\"preds\", classify(*columns))\n", - "# preds = predictions.collect()" - ] - }, - { - "cell_type": "code", - "execution_count": 83, - "id": "b79c62c8-e1e8-4467-8aef-8939c31833b8", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", - "| MedInc| HouseAge| AveRooms| AveBedrms| Population| AveOccup| Latitude| Longitude| preds|\n", - "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", - "| 0.20909257| -1.1632254| 0.38946992| 0.04609274| -0.9806099| -0.07099328|0.61245227|-0.020113053| 1.3979516|\n", - "|-0.098627955| 0.34647804| 0.27216315| -0.0129226| -0.6953838| -0.05380849| 1.0665938| -1.2479742| 1.7442212|\n", - "| -0.66006273| 1.0616008|-0.55292207| -0.48945764|-0.13641118| 0.028952759| 1.1040496| -1.3827378| 1.4398992|\n", - "| 0.08218294| 0.5848523|-0.13912922| -0.14707813|-0.19116047| -0.07136432|0.96827507| -1.3028787| 2.4199052|\n", - "| 0.0784456| -1.4810578| 0.57265776| 0.32067496| 1.0345173|-0.024157424| 1.4411427| -0.52423614| 1.2446644|\n", - "| -0.82318723| -0.36864465| 0.07829511| -0.1808107|-0.67242444|-0.061470542| 1.9374212| -1.0083897| 0.6888372|\n", - "| 0.59671736| 0.5848523| 0.19346413| -0.1371872|-0.19645879| 0.009964322|0.96827507| -1.2928978| 2.6563153|\n", - "| -0.9612035| -1.5605159|-0.56329846| 0.027148023|-0.71127874| -0.08471591| 0.5328614| -0.13990337| 1.1341839|\n", - "| -0.74344087| -1.2426835| 0.27282518| 0.4037246| -0.9841421| -0.05610115| 1.2257773| -0.42940006| 1.1378745|\n", - "| 0.9784464| -0.2891866| 0.24374022| -0.24670053| 0.28922042| -0.01102468| 1.1087307| -1.2280084| 2.5710382|\n", - "| -0.5070446| -1.0043093|-0.78254056|0.0122275995| 2.8465424|-0.060435444| 0.8980464| -1.2080427| 1.8561647|\n", - "| -0.18690155| 1.2205169|0.015323491| 0.12183313|-0.41015765| 0.04452552| 1.010412| -1.3228445| 1.8639375|\n", - "| -1.2551856| 1.6178073| -0.3341509|-0.060125165| -0.7554314| -0.08777025| 1.0291398| -1.3477987| 1.24879|\n", - "| 4.9607058| -1.9578062| 1.4854684| -0.03948475| 2.1833694|0.0029250523| 1.024457| -1.1581304| 5.5946765|\n", - "| 0.73652315| -1.6399739| 0.7913185| -0.05238397| 1.67738| 0.01944797| 1.0993668| -1.1331724| 2.0694952|\n", - "| -0.505834| 0.18756187|-0.47093546| -0.24297306|-0.60619545| -0.10791535| 0.977639| -1.2879055| 1.7852836|\n", - "| -0.88477343|-0.050812364| -0.6318951| -0.15244243| -0.5258376| -0.15618815| 0.9823201| -1.2879055| 1.6675682|\n", - "| -0.42840376| 0.9821427| -0.2266495| -0.36083496| -0.6883194| -0.08552282| 0.5328614| -0.12493005| 1.0169572|\n", - "| 0.9369153| -1.4810578| 0.6722208|-0.121177554| 0.3996021| 0.01291408| 1.1040496| -1.1082181| 2.1317377|\n", - "| -0.80702734| -0.92485124|-0.26602685| -0.1560743| 1.4398388| -0.09314839|0.55627036| -0.09498342|0.86303747|\n", - "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", - "only showing top 20 rows\n", - "\n" - ] - } - ], - "source": [ - "predictions.show()" - ] - }, - { - "cell_type": "markdown", - "id": "3fec23b0-eaf2-4b6a-aa38-7a09873ed6eb", - "metadata": { - "tags": [] - }, - "source": [ - "#### Stop Triton Server on each executor" - ] - }, - { - "cell_type": "code", - "execution_count": 84, - "id": "15e9b3df-f3c9-46bb-bbeb-42496f7663de", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "data": { - "text/plain": [ - "[True]" - ] - }, - "execution_count": 84, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def stop_triton(it):\n", - " import docker\n", - " import time\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " print(\">>>> stopping containers: {}\".format([c.short_id for c in containers]))\n", - " if containers:\n", - " container=containers[0]\n", - " container.stop(timeout=120)\n", - "\n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(stop_triton).collect()" - ] - }, - { - "cell_type": "code", - "execution_count": 85, - "id": "0138a029-87c5-497f-ac5c-3eed0e11b0f6", - "metadata": {}, - "outputs": [], - "source": [ - "spark.stop()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d24147e7-5695-44a0-9961-b94bfba1cfff", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "spark-dl-torch", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytriton_utils.py b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytriton_utils.py new file mode 100644 index 00000000..74423cc4 --- /dev/null +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytriton_utils.py @@ -0,0 +1,130 @@ +# +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import inspect +import socket +import psutil +import signal +import time +from pyspark import RDD +from multiprocessing import Process +from pytriton.client import ModelClient +from typing import Dict, List, Optional + + +def start_triton(triton_server_fn: callable, ports: List[int], model_name: str, model_path: Optional[str] = None) -> List[tuple]: + """ + Start a Triton server in a separate process and wait for it to be ready. + Return the (hostname, PID) of the node and process. + """ + sig = inspect.signature(triton_server_fn) + params = sig.parameters + + if model_path is not None: + assert len(params) == 2, "To pass a model_path for the server to load, make sure it accepts two arguments: ports and model_path" + args = (ports, model_path) + else: + assert len(params) == 1, "To start a Triton server, the function must accept one argument: ports" + args = (ports,) + + hostname = socket.gethostname() + process = Process(target=triton_server_fn, args=args) + process.start() + + client = ModelClient(f"http://localhost:{ports[0]}", model_name) + patience = 10 + while patience > 0: + try: + client.wait_for_model(5) + return [(hostname, process.pid)] + except Exception: + print("Waiting for server to be ready...") + patience -= 1 + + emsg = "Failure: client waited too long for server startup. Check the executor logs for more info." + raise TimeoutError(emsg) + +def find_ports() -> List[int]: + """ + Find three available network ports starting from port 7000 for Triton's HTTP, gRPC, and metrics services. + """ + ports = [] + conns = {conn.laddr.port for conn in psutil.net_connections(kind="inet")} + i = 7000 + while len(ports) < 3: + if i not in conns: + ports.append(i) + i += 1 + + return ports + +def use_stage_level_scheduling(spark, rdd: RDD) -> RDD: + """ + From https://github.com/NVIDIA/spark-rapids-ml/blob/main/python/src/spark_rapids_ml/core.py + Used to ensure each Triton server instance requires a full GPU to create a 1:1 executor-server mapping. + """ + + executor_cores = spark.conf.get("spark.executor.cores") + assert executor_cores is not None, "spark.executor.cores is not set" + executor_gpus = spark.conf.get("spark.executor.resource.gpu.amount") + assert executor_gpus is not None and int(executor_gpus) == 1, "spark.executor.resource.gpu.amount must be set and = 1" + + from pyspark.resource.profile import ResourceProfileBuilder + from pyspark.resource.requests import TaskResourceRequests + + # each training task requires cpu cores > total executor cores/2 which can + # ensure each training task be sent to different executor. + # + # Please note that we can't set task_cores to the value which is smaller than total executor cores/2 + # because only task_gpus can't ensure the tasks be sent to different executor even task_gpus=1.0 + # + # If spark-rapids enabled. we don't allow other ETL task running alongside training task to avoid OOM + spark_plugins = spark.conf.get("spark.plugins", " ") + assert spark_plugins is not None + spark_rapids_sql_enabled = spark.conf.get("spark.rapids.sql.enabled", "true") + assert spark_rapids_sql_enabled is not None + + task_cores = ( + int(executor_cores) + if "com.nvidia.spark.SQLPlugin" in spark_plugins + and "true" == spark_rapids_sql_enabled.lower() + else (int(executor_cores) // 2) + 1 + ) + # task_gpus means how many slots per gpu address the task requires, + # it does mean how many gpus it would like to require, so it can be any value of (0, 0.5] or 1. + task_gpus = 1.0 + treqs = TaskResourceRequests().cpus(task_cores).resource("gpu", task_gpus) + rp = ResourceProfileBuilder().require(treqs).build + print(f"Requesting stage-level resources: (cores={task_cores}, gpu={task_gpus})") + + return rdd.withResources(rp) + +def stop_triton(pids: Dict[str, int]) -> List[bool]: + """ + Stop Triton server instances by sending a SIGTERM signal. + """ + hostname = socket.gethostname() + pid = pids.get(hostname, None) + assert pid is not None, f"Could not find pid for {hostname}" + + for _ in range(5): + try: + os.kill(pid, signal.SIGTERM) + except OSError: + return [True] + time.sleep(5) + + return [False] diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/requirements.txt b/examples/ML+DL-Examples/Spark-DL/dl_inference/requirements.txt index f30f172c..2520fb8e 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/requirements.txt +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/requirements.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,7 +24,8 @@ jupyterlab pyspark>=3.4.0 huggingface datasets -docker -tritonclient[grpc] transformers -ipywidgets \ No newline at end of file +ipywidgets +ipykernel +urllib3<2 +nvidia-pytriton diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/image_classification_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/image_classification_tf.ipynb index f7df9f51..379a4d91 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/image_classification_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/image_classification_tf.ipynb @@ -5,9 +5,12 @@ "id": "52d55e3f", "metadata": {}, "source": [ + "\n", + "\n", "# Pyspark TensorFlow Inference\n", "\n", "## Image classification\n", + "This notebook demonstrates training and distributed inference for image classification on MNIST. \n", "Based on: https://www.tensorflow.org/tutorials/keras/save_and_load" ] }, @@ -16,14 +19,12 @@ "id": "5233632d", "metadata": {}, "source": [ - "### Using TensorFlow\n", - "Note that cuFFT/cuDNN/cuBLAS registration errors are expected with `tf=2.17.0` and will not affect behavior, as noted in [this issue.](https://github.com/tensorflow/tensorflow/issues/62075) \n", - "This notebook does not demonstrate inference with TensorRT, as [TF-TRT](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/index.html#tensorrt-10) does not yet support `tf=2.17.0`. See the `pytorch` notebooks for TensorRT demos." + "Note that cuFFT/cuDNN/cuBLAS registration errors are expected (as of `tf=2.17.0`) and will not affect behavior, as noted in [this issue.](https://github.com/tensorflow/tensorflow/issues/62075) " ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "c8b28f02", "metadata": {}, "outputs": [ @@ -31,20 +32,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-03 17:40:20.324462: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2024-10-03 17:40:20.331437: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2024-10-03 17:40:20.339109: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2024-10-03 17:40:20.341362: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2024-10-03 17:40:20.347337: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "2025-01-27 12:09:45.634884: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2025-01-27 12:09:45.641535: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2025-01-27 12:09:45.649019: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2025-01-27 12:09:45.651240: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2025-01-27 12:09:45.657098: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2024-10-03 17:40:20.672391: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2.17.0\n" + "2025-01-27 12:09:46.013264: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" ] } ], @@ -52,21 +46,40 @@ "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import subprocess\n", - "import tensorflow as tf\n", + "import shutil\n", "import os\n", "\n", - "from tensorflow import keras\n", - "\n", - "print(tf.version.VERSION)" + "import tensorflow as tf\n", + "from tensorflow import keras" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "e2e67086", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2.17.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", + "I0000 00:00:1738008586.411773 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008586.433071 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008586.435784 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" + ] + } + ], "source": [ + "print(tf.version.VERSION)\n", + "\n", "# Enable GPU memory growth\n", "gpus = tf.config.experimental.list_physical_devices('GPU')\n", "if gpus:\n", @@ -82,49 +95,36 @@ "id": "7e0c7ad6", "metadata": {}, "source": [ - "### Load and preprocess dataset" + "### Load and preprocess dataset\n", + "\n", + "Load MNIST and create a train/test split." ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "5b007f7c", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[1m11490434/11490434\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 0us/step\n" - ] - }, { "data": { "text/plain": [ "((60000, 28, 28), (10000, 28, 28))" ] }, - "execution_count": 4, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# load dataset as numpy arrays\n", "(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()\n", "train_images.shape, test_images.shape" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "7b7cedd1", "metadata": {}, "outputs": [ @@ -134,7 +134,7 @@ "((1000, 784), (1000, 784))" ] }, - "execution_count": 5, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -159,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "746d94db", "metadata": {}, "outputs": [ @@ -169,7 +169,13 @@ "text": [ "/home/rishic/anaconda3/envs/spark-dl-tf/lib/python3.11/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", - "2024-10-03 17:40:21.624052: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 45743 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" + "I0000 00:00:1738008586.592974 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008586.595670 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008586.598309 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008586.702447 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008586.703486 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008586.704390 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2025-01-27 12:09:46.705298: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 40715 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" ] }, { @@ -284,6 +290,16 @@ "### Save checkpoints during training" ] }, + { + "cell_type": "code", + "execution_count": 6, + "id": "dde1a855", + "metadata": {}, + "outputs": [], + "source": [ + "os.mkdir(\"models\") if not os.path.exists(\"models\") else None" + ] + }, { "cell_type": "code", "execution_count": 7, @@ -302,38 +318,38 @@ "output_type": "stream", "text": [ "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", - "I0000 00:00:1727977222.161202 1835280 service.cc:146] XLA service 0x7ec778008e00 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n", - "I0000 00:00:1727977222.161216 1835280 service.cc:154] StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n", - "2024-10-03 17:40:22.168848: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", - "2024-10-03 17:40:22.206298: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8907\n" + "I0000 00:00:1738008587.286019 3021614 service.cc:146] XLA service 0x7e8e7c005520 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n", + "I0000 00:00:1738008587.286032 3021614 service.cc:154] StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n", + "2025-01-27 12:09:47.295640: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", + "2025-01-27 12:09:47.333525: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8907\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m24s\u001b[0m 778ms/step - loss: 2.3278 - sparse_categorical_accuracy: 0.1250" + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m25s\u001b[0m 821ms/step - loss: 2.3309 - sparse_categorical_accuracy: 0.1562" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "I0000 00:00:1727977222.715572 1835280 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n" + "I0000 00:00:1738008587.874862 3021614 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 17ms/step - loss: 1.5867 - sparse_categorical_accuracy: 0.5096 " + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - loss: 1.5918 - sparse_categorical_accuracy: 0.5187 " ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2024-10-03 17:40:23.780912: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_33', 4 bytes spill stores, 4 bytes spill loads\n", + "2025-01-27 12:09:49.034107: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_33', 4 bytes spill stores, 4 bytes spill loads\n", "\n" ] }, @@ -342,50 +358,50 @@ "output_type": "stream", "text": [ "\n", - "Epoch 1: val_sparse_categorical_accuracy improved from -inf to 0.78700, saving model to training_1/checkpoint.model.keras\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 44ms/step - loss: 1.5733 - sparse_categorical_accuracy: 0.5144 - val_loss: 0.7061 - val_sparse_categorical_accuracy: 0.7870\n", + "Epoch 1: val_sparse_categorical_accuracy improved from -inf to 0.79300, saving model to models/training_1/checkpoint.model.keras\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 49ms/step - loss: 1.5776 - sparse_categorical_accuracy: 0.5239 - val_loss: 0.6896 - val_sparse_categorical_accuracy: 0.7930\n", "Epoch 2/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.5514 - sparse_categorical_accuracy: 0.8438\n", - "Epoch 2: val_sparse_categorical_accuracy improved from 0.78700 to 0.83700, saving model to training_1/checkpoint.model.keras\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - loss: 0.4276 - sparse_categorical_accuracy: 0.8935 - val_loss: 0.5268 - val_sparse_categorical_accuracy: 0.8370\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 15ms/step - loss: 0.5264 - sparse_categorical_accuracy: 0.9062\n", + "Epoch 2: val_sparse_categorical_accuracy improved from 0.79300 to 0.82800, saving model to models/training_1/checkpoint.model.keras\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.4540 - sparse_categorical_accuracy: 0.8786 - val_loss: 0.5568 - val_sparse_categorical_accuracy: 0.8280\n", "Epoch 3/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 20ms/step - loss: 0.1458 - sparse_categorical_accuracy: 0.9688\n", - "Epoch 3: val_sparse_categorical_accuracy improved from 0.83700 to 0.85600, saving model to training_1/checkpoint.model.keras\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.2721 - sparse_categorical_accuracy: 0.9236 - val_loss: 0.4716 - val_sparse_categorical_accuracy: 0.8560\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.2200 - sparse_categorical_accuracy: 0.9375\n", + "Epoch 3: val_sparse_categorical_accuracy improved from 0.82800 to 0.84000, saving model to models/training_1/checkpoint.model.keras\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.2886 - sparse_categorical_accuracy: 0.9285 - val_loss: 0.5069 - val_sparse_categorical_accuracy: 0.8400\n", "Epoch 4/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.2223 - sparse_categorical_accuracy: 0.9375\n", - "Epoch 4: val_sparse_categorical_accuracy did not improve from 0.85600\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.2159 - sparse_categorical_accuracy: 0.9547 - val_loss: 0.4682 - val_sparse_categorical_accuracy: 0.8540\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - loss: 0.2431 - sparse_categorical_accuracy: 0.9375\n", + "Epoch 4: val_sparse_categorical_accuracy improved from 0.84000 to 0.84800, saving model to models/training_1/checkpoint.model.keras\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - loss: 0.2302 - sparse_categorical_accuracy: 0.9448 - val_loss: 0.4648 - val_sparse_categorical_accuracy: 0.8480\n", "Epoch 5/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - loss: 0.1483 - sparse_categorical_accuracy: 0.9688\n", - "Epoch 5: val_sparse_categorical_accuracy improved from 0.85600 to 0.86900, saving model to training_1/checkpoint.model.keras\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - loss: 0.1457 - sparse_categorical_accuracy: 0.9716 - val_loss: 0.4285 - val_sparse_categorical_accuracy: 0.8690\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.1749 - sparse_categorical_accuracy: 0.9375\n", + "Epoch 5: val_sparse_categorical_accuracy improved from 0.84800 to 0.87300, saving model to models/training_1/checkpoint.model.keras\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - loss: 0.1602 - sparse_categorical_accuracy: 0.9631 - val_loss: 0.4069 - val_sparse_categorical_accuracy: 0.8730\n", "Epoch 6/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.0836 - sparse_categorical_accuracy: 0.9688\n", - "Epoch 6: val_sparse_categorical_accuracy did not improve from 0.86900\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.1292 - sparse_categorical_accuracy: 0.9712 - val_loss: 0.4551 - val_sparse_categorical_accuracy: 0.8580\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.0735 - sparse_categorical_accuracy: 1.0000\n", + "Epoch 6: val_sparse_categorical_accuracy did not improve from 0.87300\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.1052 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.4261 - val_sparse_categorical_accuracy: 0.8620\n", "Epoch 7/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - loss: 0.0920 - sparse_categorical_accuracy: 0.9688\n", - "Epoch 7: val_sparse_categorical_accuracy did not improve from 0.86900\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0974 - sparse_categorical_accuracy: 0.9822 - val_loss: 0.4016 - val_sparse_categorical_accuracy: 0.8670\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - loss: 0.0753 - sparse_categorical_accuracy: 1.0000\n", + "Epoch 7: val_sparse_categorical_accuracy did not improve from 0.87300\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0881 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.4703 - val_sparse_categorical_accuracy: 0.8580\n", "Epoch 8/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - loss: 0.0993 - sparse_categorical_accuracy: 0.9688\n", - "Epoch 8: val_sparse_categorical_accuracy did not improve from 0.86900\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0702 - sparse_categorical_accuracy: 0.9920 - val_loss: 0.3999 - val_sparse_categorical_accuracy: 0.8650\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - loss: 0.0561 - sparse_categorical_accuracy: 1.0000\n", + "Epoch 8: val_sparse_categorical_accuracy did not improve from 0.87300\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0663 - sparse_categorical_accuracy: 0.9902 - val_loss: 0.4121 - val_sparse_categorical_accuracy: 0.8670\n", "Epoch 9/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - loss: 0.0599 - sparse_categorical_accuracy: 1.0000\n", - "Epoch 9: val_sparse_categorical_accuracy improved from 0.86900 to 0.87800, saving model to training_1/checkpoint.model.keras\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0457 - sparse_categorical_accuracy: 0.9974 - val_loss: 0.4145 - val_sparse_categorical_accuracy: 0.8780\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.0313 - sparse_categorical_accuracy: 1.0000\n", + "Epoch 9: val_sparse_categorical_accuracy did not improve from 0.87300\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0493 - sparse_categorical_accuracy: 0.9959 - val_loss: 0.4398 - val_sparse_categorical_accuracy: 0.8570\n", "Epoch 10/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.0286 - sparse_categorical_accuracy: 1.0000\n", - "Epoch 10: val_sparse_categorical_accuracy did not improve from 0.87800\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0351 - sparse_categorical_accuracy: 0.9987 - val_loss: 0.4200 - val_sparse_categorical_accuracy: 0.8720\n" + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.0590 - sparse_categorical_accuracy: 1.0000\n", + "Epoch 10: val_sparse_categorical_accuracy did not improve from 0.87300\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0476 - sparse_categorical_accuracy: 0.9981 - val_loss: 0.4099 - val_sparse_categorical_accuracy: 0.8640\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 7, @@ -394,7 +410,7 @@ } ], "source": [ - "checkpoint_path = \"training_1/checkpoint.model.keras\"\n", + "checkpoint_path = \"models/training_1/checkpoint.model.keras\"\n", "checkpoint_dir = os.path.dirname(checkpoint_path)\n", "\n", "# Create a callback that saves the model's weights\n", @@ -447,37 +463,37 @@ "name": "stdout", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: mnist_model/assets\n" + "INFO:tensorflow:Assets written to: models/mnist_model/assets\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:tensorflow:Assets written to: mnist_model/assets\n" + "INFO:tensorflow:Assets written to: models/mnist_model/assets\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Saved artifact at 'mnist_model'. The following endpoints are available:\n", + "Saved artifact at 'models/mnist_model'. The following endpoints are available:\n", "\n", "* Endpoint 'serve'\n", " args_0 (POSITIONAL_ONLY): TensorSpec(shape=(None, 784), dtype=tf.float32, name='keras_tensor')\n", "Output Type:\n", " TensorSpec(shape=(None, 10), dtype=tf.float32, name=None)\n", "Captures:\n", - " 139403584120848: TensorSpec(shape=(), dtype=tf.resource, name=None)\n", - " 139403240100240: TensorSpec(shape=(), dtype=tf.resource, name=None)\n", - " 139403240100048: TensorSpec(shape=(), dtype=tf.resource, name=None)\n", - " 139403240099856: TensorSpec(shape=(), dtype=tf.resource, name=None)\n" + " 139159539831760: TensorSpec(shape=(), dtype=tf.resource, name=None)\n", + " 139159206493264: TensorSpec(shape=(), dtype=tf.resource, name=None)\n", + " 139163873256720: TensorSpec(shape=(), dtype=tf.resource, name=None)\n", + " 139159206493456: TensorSpec(shape=(), dtype=tf.resource, name=None)\n" ] } ], "source": [ "# Export model in saved_model format\n", - "model.export(\"mnist_model\")" + "model.export(\"models/mnist_model\")" ] }, { @@ -498,8 +514,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "32/32 - 0s - 10ms/step - loss: 2.4196 - sparse_categorical_accuracy: 0.0590\n", - "Untrained model, accuracy: 5.90%\n" + "32/32 - 0s - 12ms/step - loss: 2.3042 - sparse_categorical_accuracy: 0.1720\n", + "Untrained model, accuracy: 17.20%\n" ] } ], @@ -522,8 +538,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "32/32 - 0s - 713us/step - loss: 0.4145 - sparse_categorical_accuracy: 0.8780\n", - "Restored model, accuracy: 87.80%\n" + "32/32 - 0s - 776us/step - loss: 0.4069 - sparse_categorical_accuracy: 0.8730\n", + "Restored model, accuracy: 87.30%\n" ] }, { @@ -559,8 +575,7 @@ "metadata": {}, "outputs": [], "source": [ - "!rm -rf training_2\n", - "!mkdir training_2" + "os.mkdir(\"models/training_2\") if not os.path.exists(\"models/training_2\") else None" ] }, { @@ -574,31 +589,31 @@ "output_type": "stream", "text": [ "\n", - "Epoch 5: saving model to training_2/cp-0005.weights.h5\n", + "Epoch 5: saving model to models/training_2/cp-0005.weights.h5\n", "\n", - "Epoch 10: saving model to training_2/cp-0010.weights.h5\n", + "Epoch 10: saving model to models/training_2/cp-0010.weights.h5\n", "\n", - "Epoch 15: saving model to training_2/cp-0015.weights.h5\n", + "Epoch 15: saving model to models/training_2/cp-0015.weights.h5\n", "\n", - "Epoch 20: saving model to training_2/cp-0020.weights.h5\n", + "Epoch 20: saving model to models/training_2/cp-0020.weights.h5\n", "\n", - "Epoch 25: saving model to training_2/cp-0025.weights.h5\n", + "Epoch 25: saving model to models/training_2/cp-0025.weights.h5\n", "\n", - "Epoch 30: saving model to training_2/cp-0030.weights.h5\n", + "Epoch 30: saving model to models/training_2/cp-0030.weights.h5\n", "\n", - "Epoch 35: saving model to training_2/cp-0035.weights.h5\n", + "Epoch 35: saving model to models/training_2/cp-0035.weights.h5\n", "\n", - "Epoch 40: saving model to training_2/cp-0040.weights.h5\n", + "Epoch 40: saving model to models/training_2/cp-0040.weights.h5\n", "\n", - "Epoch 45: saving model to training_2/cp-0045.weights.h5\n", + "Epoch 45: saving model to models/training_2/cp-0045.weights.h5\n", "\n", - "Epoch 50: saving model to training_2/cp-0050.weights.h5\n" + "Epoch 50: saving model to models/training_2/cp-0050.weights.h5\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 13, @@ -608,7 +623,7 @@ ], "source": [ "# Include the epoch in the file name (uses `str.format`)\n", - "checkpoint_path = \"training_2/cp-{epoch:04d}.weights.h5\"\n", + "checkpoint_path = \"models/training_2/cp-{epoch:04d}.weights.h5\"\n", "checkpoint_dir = os.path.dirname(checkpoint_path)\n", "\n", "batch_size = 32\n", @@ -679,7 +694,7 @@ "metadata": {}, "outputs": [], "source": [ - "latest = \"training_2/cp-0030.weights.h5\"" + "latest = \"models/training_2/cp-0030.weights.h5\"" ] }, { @@ -692,8 +707,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "32/32 - 0s - 9ms/step - loss: 0.4501 - sparse_categorical_accuracy: 0.8720\n", - "Restored model, accuracy: 87.20%\n" + "32/32 - 0s - 8ms/step - loss: 0.4725 - sparse_categorical_accuracy: 0.8700\n", + "Restored model, accuracy: 87.00%\n" ] } ], @@ -724,37 +739,101 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", + "from pyspark.ml.functions import predict_batch_udf\n", + "from pyspark.sql.functions import struct, col, array, pandas_udf\n", + "from pyspark.sql.types import *\n", "from pyspark.sql import SparkSession\n", - "from pyspark import SparkConf" + "from pyspark import SparkConf\n", + "import pandas as pd\n", + "import json" + ] + }, + { + "cell_type": "markdown", + "id": "50f02919", + "metadata": {}, + "source": [ + "Check the cluster environment to handle any platform-specific Spark configurations." ] }, { "cell_type": "code", - "execution_count": null, - "id": "2c022c24", + "execution_count": 18, + "id": "4c81d510", "metadata": {}, "outputs": [], "source": [ - "import os\n", - "conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n", + "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n", + "on_standalone = not (on_databricks or on_dataproc)" + ] + }, + { + "cell_type": "markdown", + "id": "c58f4df7", + "metadata": {}, + "source": [ + "#### Create Spark Session\n", "\n", + "For local standalone clusters, we'll connect to the cluster and create the Spark Session. \n", + "For CSP environments, Spark will either be preconfigured (Databricks) or we'll need to create the Spark Session (Dataproc)." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "2c022c24", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/27 20:09:55 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/01/27 20:09:55 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", + "25/01/27 20:09:55 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "25/01/27 20:09:55 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + ] + } + ], + "source": [ "conf = SparkConf()\n", + "\n", "if 'spark' not in globals():\n", - " # If Spark is not already started with Jupyter, attach to Spark Standalone\n", - " import socket\n", - " hostname = socket.gethostname()\n", - " conf.setMaster(f\"spark://{hostname}:7077\") # assuming Master is on default port 7077\n", - "conf.set(\"spark.task.maxFailures\", \"1\")\n", - "conf.set(\"spark.driver.memory\", \"8g\")\n", - "conf.set(\"spark.executor.memory\", \"8g\")\n", - "conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled\", \"false\")\n", - "conf.set(\"spark.sql.pyspark.jvmStacktrace.enabled\", \"true\")\n", - "conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", - "conf.set(\"spark.python.worker.reuse\", \"true\")\n", - "# Create Spark Session\n", + " if on_standalone:\n", + " import socket\n", + " \n", + " conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + " hostname = socket.gethostname()\n", + " conf.setMaster(f\"spark://{hostname}:7077\")\n", + " conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", + " conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_env}/lib:{conda_env}/lib/python3.11/site-packages/nvidia_pytriton.libs:$LD_LIBRARY_PATH\")\n", + " source = \"/usr/lib/x86_64-linux-gnu/libstdc++.so.6\"\n", + " target = f\"{conda_env}/lib/libstdc++.so.6\"\n", + " try:\n", + " if os.path.islink(target) or os.path.exists(target):\n", + " os.remove(target)\n", + " os.symlink(source, target)\n", + " except OSError as e:\n", + " print(f\"Error creating symlink: {e}\")\n", + " elif on_dataproc:\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conda_lib_path=\"/opt/conda/miniconda3/lib\"\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_lib_path}:$LD_LIBRARY_PATH\")\n", + " conf.set(\"spark.executorEnv.TF_GPU_ALLOCATOR\", \"cuda_malloc_async\")\n", + " conf.set(\"spark.executor.instances\", \"4\") # dataproc defaults to 2\n", + "\n", + " conf.set(\"spark.executor.cores\", \"8\")\n", + " conf.set(\"spark.task.resource.gpu.amount\", \"0.125\")\n", + " conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n", + " conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", + " conf.set(\"spark.python.worker.reuse\", \"true\")\n", + "\n", + "conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"1000\")\n", "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", "sc = spark.sparkContext" ] @@ -764,12 +843,12 @@ "id": "c81d0b1b", "metadata": {}, "source": [ - "### Convert numpy array to Spark DataFrame (via Pandas DataFrame)" + "### Create Spark Dataframe" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "id": "49ff5203", "metadata": {}, "outputs": [ @@ -779,7 +858,7 @@ "(1000, 784)" ] }, - "execution_count": 19, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -792,21 +871,11 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "id": "182ee0c7", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 134 ms, sys: 15.5 ms, total: 149 ms\n", - "Wall time: 1.36 s\n" - ] - } - ], + "outputs": [], "source": [ - "%%time\n", "df = spark.createDataFrame(test_pdf).repartition(8)" ] }, @@ -820,7 +889,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 22, "id": "0061c39a-0871-429e-a4ff-751d26bf4b04", "metadata": {}, "outputs": [ @@ -828,7 +897,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "24/10/03 17:40:32 WARN SparkStringUtils: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n", + "25/01/27 20:09:58 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n", "[Stage 0:> (0 + 8) / 8]\r" ] }, @@ -836,8 +905,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.49 ms, sys: 1.65 ms, total: 4.13 ms\n", - "Wall time: 1.66 s\n" + "CPU times: user 3.02 ms, sys: 1.31 ms, total: 4.33 ms\n", + "Wall time: 1.92 s\n" ] }, { @@ -850,7 +919,12 @@ ], "source": [ "%%time\n", - "df.write.mode(\"overwrite\").parquet(\"mnist_784\")" + "data_path_784 = \"spark-dl-datasets/mnist_784\"\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " data_path_784 = \"dbfs:/FileStore/\" + data_path_784\n", + "\n", + "df.write.mode(\"overwrite\").parquet(data_path_784)" ] }, { @@ -863,31 +937,22 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 23, "id": "302c73ec", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 6.71 ms, sys: 4.92 ms, total: 11.6 ms\n", - "Wall time: 11.4 ms\n" - ] - }, { "data": { "text/plain": [ "(1000, 1)" ] }, - "execution_count": 22, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "%%time\n", "test_pdf['data'] = test_pdf.values.tolist()\n", "pdf = test_pdf[['data']]\n", "pdf.shape" @@ -895,27 +960,17 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 24, "id": "5495901b", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 46.6 ms, sys: 4.71 ms, total: 51.3 ms\n", - "Wall time: 91.7 ms\n" - ] - } - ], + "outputs": [], "source": [ - "%%time\n", "df = spark.createDataFrame(pdf)" ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 25, "id": "5fa7faa8-c6bd-41b0-b5f7-fb121f0332e6", "metadata": {}, "outputs": [ @@ -923,42 +978,32 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 807 μs, sys: 724 μs, total: 1.53 ms\n", - "Wall time: 211 ms\n" + "CPU times: user 16 μs, sys: 1.01 ms, total: 1.02 ms\n", + "Wall time: 218 ms\n" ] } ], "source": [ "%%time\n", - "df.write.mode(\"overwrite\").parquet(\"mnist_1\")" - ] - }, - { - "cell_type": "markdown", - "id": "c87b444e", - "metadata": {}, - "source": [ - "### Check arrow memory configuration" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "3d4ca414", - "metadata": {}, - "outputs": [], - "source": [ - "spark.conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"128\")\n", - "# This line will fail if the vectorized reader runs out of memory\n", - "assert len(df.head()) > 0, \"`df` should not be empty\" " + "data_path_1 = \"spark-dl-datasets/mnist_1\"\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " data_path_1 = \"dbfs:/FileStore/\" + data_path_1\n", + "\n", + "df.write.mode(\"overwrite\").parquet(data_path_1)" ] }, { "cell_type": "markdown", - "id": "9b6dde30-98a9-45db-ab3a-d4546f9bed99", + "id": "b366aaeb", "metadata": {}, "source": [ - "## Inference using Spark DL API" + "## Inference using Spark DL API\n", + "\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "\n", + "- predict_batch_fn uses Tensorflow APIs to load the model and return a predict function which operates on numpy arrays \n", + "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" ] }, { @@ -972,33 +1017,30 @@ { "cell_type": "code", "execution_count": 26, - "id": "db30fba6-24d0-4c00-8502-04f9b10e7e16", + "id": "b9cf62f8-96b2-4716-80bd-bb93d5f939bd", "metadata": {}, "outputs": [], "source": [ - "import numpy as np\n", - "import os\n", - "import pandas as pd\n", + "model_path = \"{}/models/training_1/checkpoint.model.keras\".format(os.getcwd())\n", "\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import array, col, struct\n", - "from pyspark.sql.types import ArrayType, FloatType, Union, Dict" + "# For cloud environments, copy the model to the distributed file system.\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-models\")\n", + " dbfs_model_path = \"/dbfs/FileStore/spark-dl-models/checkpoint.model.keras\"\n", + " shutil.copy(model_path, dbfs_model_path)\n", + " model_path = dbfs_model_path\n", + "elif on_dataproc:\n", + " # GCS is mounted at /mnt/gcs by the init script\n", + " models_dir = \"/mnt/gcs/spark-dl/models\"\n", + " os.mkdir(models_dir) if not os.path.exists(models_dir) else None\n", + " gcs_model_path = models_dir + \"/checkpoint.model.keras\"\n", + " shutil.copy(model_path, gcs_model_path)\n", + " model_path = gcs_model_path" ] }, { "cell_type": "code", "execution_count": 27, - "id": "b9cf62f8-96b2-4716-80bd-bb93d5f939bd", - "metadata": {}, - "outputs": [], - "source": [ - "# get absolute path to model\n", - "model_dir = \"{}/training_1/checkpoint.model.keras\".format(os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": 28, "id": "b81fa297-d9d0-4600-880d-dbdcdf8bccc6", "metadata": {}, "outputs": [], @@ -1015,7 +1057,7 @@ " except RuntimeError as e:\n", " print(e)\n", "\n", - " model = tf.keras.models.load_model(model_dir)\n", + " model = tf.keras.models.load_model(model_path)\n", " def predict(inputs: np.ndarray) -> np.ndarray:\n", " return model.predict(inputs)\n", " \n", @@ -1024,20 +1066,20 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 28, "id": "72a689bd-dd82-492e-8740-1738a215325f", "metadata": {}, "outputs": [], "source": [ "mnist = predict_batch_udf(predict_batch_fn,\n", " return_type=ArrayType(FloatType()),\n", - " batch_size=1024,\n", + " batch_size=128,\n", " input_tensor_shapes=[[784]])" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 29, "id": "60a70150-26b1-4145-9e7d-6e17389216b7", "metadata": {}, "outputs": [ @@ -1047,19 +1089,19 @@ "1" ] }, - "execution_count": 30, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df = spark.read.parquet(\"mnist_1\")\n", + "df = spark.read.parquet(data_path_1)\n", "len(df.columns)" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 30, "id": "e027f0d2-0f65-47b7-a562-2f0965faceec", "metadata": {}, "outputs": [ @@ -1087,7 +1129,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 31, "id": "f0c3fb2e-469e-47bc-b948-8f6b0d7f6513", "metadata": {}, "outputs": [ @@ -1095,22 +1137,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 4:===================================================> (7 + 1) / 8]\r" + " \r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 18.5 ms, sys: 13.3 ms, total: 31.8 ms\n", - "Wall time: 5.03 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" + "CPU times: user 22.4 ms, sys: 14.1 ms, total: 36.5 ms\n", + "Wall time: 5.89 s\n" ] } ], @@ -1122,7 +1157,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 32, "id": "cdfa229a-f4a9-4c11-a410-de4a21c02c82", "metadata": {}, "outputs": [ @@ -1130,7 +1165,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 37.3 ms, sys: 12.4 ms, total: 49.8 ms\n", + "CPU times: user 27.2 ms, sys: 12.8 ms, total: 40 ms\n", "Wall time: 259 ms\n" ] } @@ -1142,7 +1177,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 33, "id": "5586ce49-6f93-4343-9b66-0dbb64972179", "metadata": {}, "outputs": [ @@ -1150,8 +1185,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 22.9 ms, sys: 5.96 ms, total: 28.8 ms\n", - "Wall time: 237 ms\n" + "CPU times: user 40.1 ms, sys: 8.81 ms, total: 48.9 ms\n", + "Wall time: 231 ms\n" ] } ], @@ -1172,7 +1207,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 34, "id": "4f947dc0-6b18-4605-810b-e83250a161db", "metadata": {}, "outputs": [ @@ -1212,52 +1247,52 @@ " \n", " 0\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-5.88436, -3.1058547, 0.10873719, 12.67319, -...\n", + " [-2.982547, -2.495611, 1.2573445, 12.891551, -...\n", " \n", " \n", " 1\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-3.273286, -8.362554, 1.8936121, -3.8881433, ...\n", + " [-0.95476156, -5.622215, 2.241567, -2.0017645,...\n", " \n", " \n", " 2\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-3.3856308, 0.6785604, 1.3146863, 0.9275978, ...\n", + " [-0.9178914, 0.43411195, 1.9357576, 1.7975042,...\n", " \n", " \n", " 3\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-2.7754688, -7.3659225, 11.768427, 1.3434286,...\n", + " [0.21153986, -5.5822043, 11.524151, 2.8064005,...\n", " \n", " \n", " 4\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-4.9426627, 4.0774136, -0.4529277, -0.9312789...\n", + " [-3.3954644, 3.7420855, -0.2531985, 0.1679054,...\n", " \n", " \n", " 5\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-5.226616, -3.1389174, 2.6100307, 3.695045, -...\n", + " [-3.1469436, -2.4765797, 3.0782855, 4.3063755,...\n", " \n", " \n", " 6\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-4.3006196, 5.1169925, 0.5850615, -0.76248693...\n", + " [-2.9109762, 4.0204973, 1.3597142, -0.00311854...\n", " \n", " \n", " 7\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-2.3985956, -1.4814724, -4.884057, -0.2391600...\n", + " [-0.3474851, -0.7892854, -4.275904, 0.51054317...\n", " \n", " \n", " 8\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [0.82160115, -2.8640625, -1.6951559, -4.489290...\n", + " [2.0500484, -2.2071126, -0.07787762, -3.207581...\n", " \n", " \n", " 9\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-1.2338604, -2.151981, -4.171742, 1.6106845, ...\n", + " [-0.0029632095, -1.7498987, -2.1225448, 2.1881...\n", " \n", " \n", "\n", @@ -1277,19 +1312,19 @@ "9 [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ... \n", "\n", " preds \n", - "0 [-5.88436, -3.1058547, 0.10873719, 12.67319, -... \n", - "1 [-3.273286, -8.362554, 1.8936121, -3.8881433, ... \n", - "2 [-3.3856308, 0.6785604, 1.3146863, 0.9275978, ... \n", - "3 [-2.7754688, -7.3659225, 11.768427, 1.3434286,... \n", - "4 [-4.9426627, 4.0774136, -0.4529277, -0.9312789... \n", - "5 [-5.226616, -3.1389174, 2.6100307, 3.695045, -... \n", - "6 [-4.3006196, 5.1169925, 0.5850615, -0.76248693... \n", - "7 [-2.3985956, -1.4814724, -4.884057, -0.2391600... \n", - "8 [0.82160115, -2.8640625, -1.6951559, -4.489290... \n", - "9 [-1.2338604, -2.151981, -4.171742, 1.6106845, ... " + "0 [-2.982547, -2.495611, 1.2573445, 12.891551, -... \n", + "1 [-0.95476156, -5.622215, 2.241567, -2.0017645,... \n", + "2 [-0.9178914, 0.43411195, 1.9357576, 1.7975042,... \n", + "3 [0.21153986, -5.5822043, 11.524151, 2.8064005,... \n", + "4 [-3.3954644, 3.7420855, -0.2531985, 0.1679054,... \n", + "5 [-3.1469436, -2.4765797, 3.0782855, 4.3063755,... \n", + "6 [-2.9109762, 4.0204973, 1.3597142, -0.00311854... \n", + "7 [-0.3474851, -0.7892854, -4.275904, 0.51054317... \n", + "8 [2.0500484, -2.2071126, -0.07787762, -3.207581... \n", + "9 [-0.0029632095, -1.7498987, -2.1225448, 2.1881... " ] }, - "execution_count": 35, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -1301,19 +1336,19 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 35, "id": "de4964e0-d1f8-4753-afa1-a8f95ca3f151", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "array([ -5.88436 , -3.1058547 , 0.10873719, 12.67319 ,\n", - " -5.143787 , 4.0859914 , -10.203137 , -1.4333997 ,\n", - " -3.3865087 , -3.8473575 ], dtype=float32)" + "array([-2.982547 , -2.495611 , 1.2573445 , 12.891551 , -5.40115 ,\n", + " 2.7227228 , -6.900943 , -1.4402989 , 0.59647846, -3.8927622 ],\n", + " dtype=float32)" ] }, - "execution_count": 36, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } @@ -1325,7 +1360,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 36, "id": "44e9a874-e301-4b72-8df7-bf1c5133c287", "metadata": {}, "outputs": [], @@ -1336,7 +1371,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 37, "id": "c60e5af4-fc1e-4575-a717-f304664235be", "metadata": {}, "outputs": [], @@ -1347,7 +1382,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 38, "id": "eb45ecc9-d376-40c4-ad7b-2bd08ca5aaf6", "metadata": {}, "outputs": [ @@ -1379,21 +1414,7 @@ }, { "cell_type": "code", - "execution_count": 40, - "id": "f1285e8b-1b96-437b-973a-eb868e33afb7", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import array, col, struct\n", - "from pyspark.sql.types import ArrayType, FloatType, Union, Dict" - ] - }, - { - "cell_type": "code", - "execution_count": 41, + "execution_count": 39, "id": "6bea332e-f6de-494f-a0db-795d9fe3e134", "metadata": {}, "outputs": [], @@ -1409,7 +1430,7 @@ " except RuntimeError as e:\n", " print(e)\n", " \n", - " model = tf.keras.models.load_model(model_dir)\n", + " model = tf.keras.models.load_model(model_path)\n", " def predict(inputs: np.ndarray) -> np.ndarray:\n", " return model.predict(inputs)\n", " \n", @@ -1418,20 +1439,20 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 40, "id": "731d234c-549f-4df3-8a2b-312e63195396", "metadata": {}, "outputs": [], "source": [ "mnist = predict_batch_udf(predict_batch_fn,\n", " return_type=ArrayType(FloatType()),\n", - " batch_size=1024,\n", + " batch_size=128,\n", " input_tensor_shapes=[[784]])" ] }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 41, "id": "a40fe207-6246-4b0e-abde-823979878d97", "metadata": {}, "outputs": [ @@ -1441,19 +1462,19 @@ "784" ] }, - "execution_count": 43, + "execution_count": 41, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df = spark.read.parquet(\"mnist_784\")\n", + "df = spark.read.parquet(data_path_784)\n", "len(df.columns)" ] }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 42, "id": "10904f12-03e7-4518-8f12-2aa11989ddf5", "metadata": {}, "outputs": [ @@ -1461,22 +1482,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 10:=======> (1 + 7) / 8]\r" + " \r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 45.6 ms, sys: 26 ms, total: 71.6 ms\n", - "Wall time: 5.51 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" + "CPU times: user 53.2 ms, sys: 31.2 ms, total: 84.4 ms\n", + "Wall time: 6.21 s\n" ] } ], @@ -1487,16 +1501,23 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 43, "id": "671128df-f0f4-4f54-b35c-d63a78c7f89a", "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 46.5 ms, sys: 34 ms, total: 80.5 ms\n", - "Wall time: 884 ms\n" + "CPU times: user 54.5 ms, sys: 20.1 ms, total: 74.6 ms\n", + "Wall time: 1.95 s\n" ] } ], @@ -1507,7 +1528,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 44, "id": "ce35deaf-7d49-4f34-9bf9-b4e6fc5761f4", "metadata": {}, "outputs": [], @@ -1526,7 +1547,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 45, "id": "f9119632-b284-45d7-a262-c262e034c15c", "metadata": {}, "outputs": [ @@ -1604,7 +1625,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-5.88436, -3.1058552, 0.108737305, 12.67319, ...\n", + " [-6.554102, 2.085586, 1.3187729, 0.7378275, -4...\n", " \n", " \n", " 1\n", @@ -1628,7 +1649,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-3.2732859, -8.362555, 1.893612, -3.888143, 0...\n", + " [-3.8565233, 4.518864, -1.2931982, -0.78954405...\n", " \n", " \n", " 2\n", @@ -1652,7 +1673,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-3.3856308, 0.6785604, 1.3146865, 0.9275978, ...\n", + " [-3.7759259, -2.682865, -1.2959752, -5.4488983...\n", " \n", " \n", " 3\n", @@ -1676,7 +1697,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-2.775469, -7.3659234, 11.768431, 1.3434289, ...\n", + " [0.15489274, -1.4305159, -1.5703316, 1.2637339...\n", " \n", " \n", " 4\n", @@ -1700,7 +1721,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-4.942663, 4.0774136, -0.45292768, -0.9312788...\n", + " [-3.9276226, 4.4247217, 1.0965542, 0.3403727, ...\n", " \n", " \n", " 5\n", @@ -1724,7 +1745,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-5.226616, -3.1389174, 2.6100307, 3.695045, -...\n", + " [8.987603, -3.3934922, 2.4643059, -0.84833026,...\n", " \n", " \n", " 6\n", @@ -1748,7 +1769,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-4.3006196, 5.116993, 0.5850617, -0.7624871, ...\n", + " [7.024207, -3.5837536, 1.645798, 0.86984664, -...\n", " \n", " \n", " 7\n", @@ -1772,7 +1793,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-2.398596, -1.4814726, -4.8840575, -0.2391601...\n", + " [3.0095522, -4.574903, 0.7152054, -5.211629, 4...\n", " \n", " \n", " 8\n", @@ -1796,7 +1817,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [0.82160157, -2.8640628, -1.6951559, -4.489291...\n", + " [-1.548282, -3.065774, 10.493741, -1.5243877, ...\n", " \n", " \n", " 9\n", @@ -1820,7 +1841,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-1.2338604, -2.151981, -4.1717424, 1.6106843,...\n", + " [-1.5818219, -3.821816, -3.4366767, 7.7891817,...\n", " \n", " \n", "\n", @@ -1841,21 +1862,21 @@ "9 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 \n", "\n", " 779 780 781 782 783 preds \n", - "0 0.0 0.0 0.0 0.0 0.0 [-5.88436, -3.1058552, 0.108737305, 12.67319, ... \n", - "1 0.0 0.0 0.0 0.0 0.0 [-3.2732859, -8.362555, 1.893612, -3.888143, 0... \n", - "2 0.0 0.0 0.0 0.0 0.0 [-3.3856308, 0.6785604, 1.3146865, 0.9275978, ... \n", - "3 0.0 0.0 0.0 0.0 0.0 [-2.775469, -7.3659234, 11.768431, 1.3434289, ... \n", - "4 0.0 0.0 0.0 0.0 0.0 [-4.942663, 4.0774136, -0.45292768, -0.9312788... \n", - "5 0.0 0.0 0.0 0.0 0.0 [-5.226616, -3.1389174, 2.6100307, 3.695045, -... \n", - "6 0.0 0.0 0.0 0.0 0.0 [-4.3006196, 5.116993, 0.5850617, -0.7624871, ... \n", - "7 0.0 0.0 0.0 0.0 0.0 [-2.398596, -1.4814726, -4.8840575, -0.2391601... \n", - "8 0.0 0.0 0.0 0.0 0.0 [0.82160157, -2.8640628, -1.6951559, -4.489291... \n", - "9 0.0 0.0 0.0 0.0 0.0 [-1.2338604, -2.151981, -4.1717424, 1.6106843,... \n", + "0 0.0 0.0 0.0 0.0 0.0 [-6.554102, 2.085586, 1.3187729, 0.7378275, -4... \n", + "1 0.0 0.0 0.0 0.0 0.0 [-3.8565233, 4.518864, -1.2931982, -0.78954405... \n", + "2 0.0 0.0 0.0 0.0 0.0 [-3.7759259, -2.682865, -1.2959752, -5.4488983... \n", + "3 0.0 0.0 0.0 0.0 0.0 [0.15489274, -1.4305159, -1.5703316, 1.2637339... \n", + "4 0.0 0.0 0.0 0.0 0.0 [-3.9276226, 4.4247217, 1.0965542, 0.3403727, ... \n", + "5 0.0 0.0 0.0 0.0 0.0 [8.987603, -3.3934922, 2.4643059, -0.84833026,... \n", + "6 0.0 0.0 0.0 0.0 0.0 [7.024207, -3.5837536, 1.645798, 0.86984664, -... \n", + "7 0.0 0.0 0.0 0.0 0.0 [3.0095522, -4.574903, 0.7152054, -5.211629, 4... \n", + "8 0.0 0.0 0.0 0.0 0.0 [-1.548282, -3.065774, 10.493741, -1.5243877, ... \n", + "9 0.0 0.0 0.0 0.0 0.0 [-1.5818219, -3.821816, -3.4366767, 7.7891817,... \n", "\n", "[10 rows x 785 columns]" ] }, - "execution_count": 47, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } @@ -1867,7 +1888,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 46, "id": "7c067c62-03a6-461e-a1ff-4653276fbea1", "metadata": {}, "outputs": [], @@ -1878,19 +1899,19 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 47, "id": "a7084ad0-c021-4296-bad0-7a238971f53b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "array([ -5.88436 , -3.1058552, 0.1087373, 12.67319 , -5.1437874,\n", - " 4.085992 , -10.203137 , -1.4333997, -3.3865087, -3.8473575],\n", + "array([-6.554102 , 2.085586 , 1.3187729, 0.7378275, -4.1074104,\n", + " -3.352563 , -4.2061253, 4.6558733, 1.0386009, 0.9432423],\n", " dtype=float32)" ] }, - "execution_count": 49, + "execution_count": 47, "metadata": {}, "output_type": "execute_result" } @@ -1902,7 +1923,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 48, "id": "8167c832-93ef-4f50-873b-07b67c19ef53", "metadata": {}, "outputs": [], @@ -1914,13 +1935,13 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 49, "id": "297811e1-aecb-4afd-9a6a-30c49e8881cc", "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGzCAYAAABpdMNsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAkfElEQVR4nO3dfXQUdZ7v8U/nqSEkaR7yLAFCFHRAcAYly/AgSiQEZUCYGUG9F7gziJiggI6KR0Udzsksrg7qIHjcHVhHEGWOyMoiDg9JGBRwwTCIM2QhJ0g4kIBcSYcAIaR/9w+uvbQkQDUdfkl4v86pc+iq37fqm6Lgk+qqrnYZY4wAALjKwmw3AAC4NhFAAAArCCAAgBUEEADACgIIAGAFAQQAsIIAAgBYQQABAKwggAAAVhBAgAPdunXTpEmT/K8LCwvlcrlUWFgYsm24XC698MILIVsf0FwRQGgxlixZIpfL5Z/atGmjHj16KC8vT5WVlbbbc2TNmjUtJmTefvtt3X777UpKSpLb7VZ6eromT56s/fv3224NLVyE7QYAp1566SWlp6fr9OnT2rx5sxYuXKg1a9Zo9+7dio6Ovqq9DBkyRKdOnVJUVJSjujVr1mjBggUNhtCpU6cUEdF8/mkWFxcrPT1dP/vZz9ShQweVlZXp7bff1urVq/W3v/1NqamptltEC9V8jnLgMuXk5OjWW2+VJP36179Wp06d9Oqrr2rVqlWaMGFCgzU1NTVq165dyHsJCwtTmzZtQrrOUK/vSr355psXzBszZoxuvfVWvfPOO3r66actdIXWgLfg0OLdeeedkqSysjJJ0qRJkxQTE6PS0lKNHDlSsbGxeuCBByRJPp9P8+fPV69evdSmTRslJSVp6tSp+u677wLWaYzR3Llz1blzZ0VHR+uOO+7Q119/fcG2G7sGtG3bNo0cOVIdOnRQu3bt1KdPH7322mv+/hYsWCBJAW8pfq+ha0DFxcXKyclRXFycYmJiNGzYMG3dujVgzPdvUX722WeaNWuWEhIS1K5dO9177706evRowNiqqirt2bNHVVVVl7OLL9CtWzdJ0vHjx4OqByTOgNAKlJaWSpI6derkn3f27FllZ2dr0KBB+pd/+Rf/W3NTp07VkiVLNHnyZD366KMqKyvTH/7wBxUXF+uzzz5TZGSkJOn555/X3LlzNXLkSI0cOVJffvmlhg8frjNnzlyyn3Xr1umee+5RSkqKHnvsMSUnJ+sf//iHVq9erccee0xTp07VoUOHtG7dOv3pT3+65Pq+/vprDR48WHFxcXryyScVGRmpt956S0OHDlVRUZEyMzMDxk+fPl0dOnTQnDlztH//fs2fP195eXl6//33/WNWrlypyZMna/HixQE3VVzMsWPHVF9frwMHDuill16SJA0bNuyyaoEGGaCFWLx4sZFk1q9fb44ePWrKy8vN8uXLTadOnUzbtm3NwYMHjTHGTJw40UgyTz/9dED9X//6VyPJLF26NGD+2rVrA+YfOXLEREVFmbvvvtv4fD7/uGeeecZIMhMnTvTPKygoMJJMQUGBMcaYs2fPmvT0dNO1a1fz3XffBWzn/HXl5uaaxv75STJz5szxvx4zZoyJiooypaWl/nmHDh0ysbGxZsiQIRfsn6ysrIBtzZw504SHh5vjx49fMHbx4sUN9tAQt9ttJBlJplOnTub111+/7FqgIbwFhxYnKytLCQkJSktL0/jx4xUTE6OVK1fquuuuCxg3bdq0gNcrVqyQx+PRXXfdpW+//dY/9evXTzExMSooKJAkrV+/XmfOnNH06dMD3hqbMWPGJXsrLi5WWVmZZsyYofbt2wcsO39dl6u+vl5/+ctfNGbMGHXv3t0/PyUlRffff782b94sr9cbUPPQQw8FbGvw4MGqr6/XN9984583adIkGWMu++xHkj755BOtWbNGr7zyirp06aKamhrHPw9wPt6CQ4uzYMEC9ejRQxEREUpKSlLPnj0VFhb4u1RERIQ6d+4cMG/v3r2qqqpSYmJig+s9cuSIJPn/o77hhhsClickJKhDhw4X7e37twN79+59+T/QRRw9elQnT55Uz549L1h20003yefzqby8XL169fLP79KlS8C473v+4XUup+644w5J524CGT16tHr37q2YmBjl5eVd0Xpx7SKA0OL079/ffxdcY9xu9wWh5PP5lJiYqKVLlzZYk5CQELIebQoPD29wvjEmZNvIyMjQj3/8Yy1dupQAQtAIIFwzMjIytH79eg0cOFBt27ZtdFzXrl0lnTtjOv9tr6NHj17yLCIjI0OStHv3bmVlZTU67nLfjktISFB0dLRKSkouWLZnzx6FhYUpLS3tstYVaqdOnVJtba2VbaN14BoQrhm//OUvVV9fr9/+9rcXLDt79qz/luKsrCxFRkbqjTfeCDhrmD9//iW38ZOf/ETp6emaP3/+Bbcon7+u7z+TdKnbmMPDwzV8+HCtWrUq4MkDlZWVWrZsmQYNGqS4uLhL9vVDl3sb9tmzZxsM3S+++EJfffXVJc9EgYvhDAjXjNtvv11Tp05Vfn6+du7cqeHDhysyMlJ79+7VihUr9Nprr+nnP/+5EhIS9MQTTyg/P1/33HOPRo4cqeLiYn3yySeKj4+/6DbCwsK0cOFCjRo1SrfccosmT56slJQU7dmzR19//bU+/fRTSVK/fv0kSY8++qiys7MVHh6u8ePHN7jOuXPnat26dRo0aJAeeeQRRURE6K233lJtba3mzZsX1L643NuwT5w4obS0NN13333q1auX2rVrp6+++kqLFy+Wx+PRc889F9T2AYkAwjVm0aJF6tevn9566y0988wzioiIULdu3fTggw9q4MCB/nFz585VmzZttGjRIhUUFCgzM1N/+ctfdPfdd19yG9nZ2SooKNCLL76oV155RT6fTxkZGZoyZYp/zNixYzV9+nQtX75c7777rowxjQZQr1699Ne//lWzZ89Wfn6+fD6fMjMz9e67717wGaBQi46O1q9//WsVFBToz3/+s06dOqXU1FRNmDBBzz77rP8DqUAwXCaUVyYBALhMXAMCAFhBAAEArCCAAABWEEAAACsIIACAFQQQAMCKZvc5IJ/Pp0OHDik2NjaopwcDAOwyxqi6ulqpqakXPJPxfM0ugA4dOmTt2VYAgNApLy+/4Kn052t2ARQbGytJGqSRilCk5W4AAE6dVZ02a43///PGNFkALViwQC+//LIqKirUt29fvfHGG+rfv/8l675/2y1CkYpwEUAA0OL8/+frXOoySpPchPD+++9r1qxZmjNnjr788kv17dtX2dnZ/i/8AgCgSQLo1Vdf1ZQpUzR58mT96Ec/0qJFixQdHa0//vGPTbE5AEALFPIAOnPmjHbs2BHwZVxhYWHKysrSli1bLhhfW1srr9cbMAEAWr+QB9C3336r+vp6JSUlBcxPSkpSRUXFBePz8/Pl8Xj8E3fAAcC1wfoHUWfPnq2qqir/VF5ebrslAMBVEPK74OLj4xUeHq7KysqA+ZWVlUpOTr5gvNvtltvtDnUbAIBmLuRnQFFRUerXr582bNjgn+fz+bRhwwYNGDAg1JsDALRQTfI5oFmzZmnixIm69dZb1b9/f82fP181NTWaPHlyU2wOANACNUkA3XfffTp69Kief/55VVRU6JZbbtHatWsvuDEBAHDtchljjO0mzuf1euXxeDRUo3kSAgC0QGdNnQq1SlVVVYqLi2t0nPW74AAA1yYCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWBFhuwHYZ37aN7i6cOe/v0RWeh3XlP7vRMc1vu6nHNdI0p7b/+i4JtzlfD9MPTjAcU3hp7c4run2nzWOayRJW3cFVwc4wBkQAMAKAggAYEXIA+iFF16Qy+UKmG688cZQbwYA0MI1yTWgXr16af369f+zkQguNQEAAjVJMkRERCg5ObkpVg0AaCWa5BrQ3r17lZqaqu7du+uBBx7QgQMHGh1bW1srr9cbMAEAWr+QB1BmZqaWLFmitWvXauHChSorK9PgwYNVXV3d4Pj8/Hx5PB7/lJaWFuqWAADNUMgDKCcnR7/4xS/Up08fZWdna82aNTp+/Lg++OCDBsfPnj1bVVVV/qm8vDzULQEAmqEmvzugffv26tGjh/bt29fgcrfbLbfb3dRtAACamSb/HNCJEydUWlqqlJSUpt4UAKAFCXkAPfHEEyoqKtL+/fv1+eef695771V4eLgmTJgQ6k0BAFqwkL8Fd/DgQU2YMEHHjh1TQkKCBg0apK1btyohISHUmwIAtGAuY4yx3cT5vF6vPB6Phmq0IlyRttuxqubnmY5rKm91flK7dsLLjmskqUtEW8c1/2v/XY5r/tRtneManFN8xhdU3eOP5zmuif5wW1DbQutz1tSpUKtUVVWluLi4RsfxLDgAgBUEEADACgIIAGAFAQQAsIIAAgBYQQABAKwggAAAVhBAAAArCCAAgBUEEADACgIIAGAFAQQAsIKHkV4lR/J+6rim8OlXHNdEu6Ic1zR339afclzTxhXc71Z1cv7PYcaBexzX/DLxvxzX3B1d5bgmWPvqah3XPDH4l45rzpYfdFyD5o+HkQIAmjUCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsiLDdwLXCF+68pjU+2frlYz9yXLNhxiDHNfVtg/vd6rvrnT+B/br/POy45s2EcY5r7v7zHx3XBGvsf011XNPt+P7QN4JWjTMgAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALCCh5FeJan/+jfHNR88kui4Jjv6gOOanDlPOK6RpLoYl+Oa6/7joOOaiP07nNc4rjgnOYia+iBqKu/5aRBVV8+uny5xXDMmiAes+qqrHdeg9eAMCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCs4GGkV4mvpsZxzTs90xzXvJ0z1nFNfEGx4xpJ8p0+7bjmbFBbunrCExIc13x3V4bjmscf/sBxDdDacAYEALCCAAIAWOE4gDZt2qRRo0YpNTVVLpdLH330UcByY4yef/55paSkqG3btsrKytLevXtD1S8AoJVwHEA1NTXq27evFixY0ODyefPm6fXXX9eiRYu0bds2tWvXTtnZ2TodxPUCAEDr5fgmhJycHOXk5DS4zBij+fPn69lnn9Xo0aMlSe+8846SkpL00Ucfafz48VfWLQCg1QjpNaCysjJVVFQoKyvLP8/j8SgzM1NbtmxpsKa2tlZerzdgAgC0fiENoIqKCklSUlJSwPykpCT/sh/Kz8+Xx+PxT2lpzm89BgC0PNbvgps9e7aqqqr8U3l5ue2WAABXQUgDKDk5WZJUWVkZML+ystK/7Ifcbrfi4uICJgBA6xfSAEpPT1dycrI2bNjgn+f1erVt2zYNGDAglJsCALRwju+CO3HihPbt2+d/XVZWpp07d6pjx47q0qWLZsyYoblz5+qGG25Qenq6nnvuOaWmpmrMmDGh7BsA0MI5DqDt27frjjvu8L+eNWuWJGnixIlasmSJnnzySdXU1Oihhx7S8ePHNWjQIK1du1Zt2rQJXdcAgBbPZYwxtps4n9frlcfj0VCNVoQr0nY7aKHC23uCqnt8x2bHNUPanAlqW1eDT76g6n73bV/HNduGpTiuqf/2mOMaNH9nTZ0KtUpVVVUXva5v/S44AMC1iQACAFhBAAEArCCAAABWEEAAACsIIACAFQQQAMAKAggAYAUBBACwggACAFhBAAEArCCAAABWEEAAACscfx0D0BKUPdorqLohbTaGuBO7VtXEB1X3ed+oIKp4sjWc4QwIAGAFAQQAsIIAAgBYQQABAKwggAAAVhBAAAArCCAAgBUEEADACgIIAGAFAQQAsIIAAgBYQQABAKzgYaRAK3ZX28NB1c2d9YDjmrqYoDblWGLxWcc1bT7+ogk6wZXiDAgAYAUBBACwggACAFhBAAEArCCAAABWEEAAACsIIACAFQQQAMAKAggAYAUBBACwggACAFhBAAEArOBhpGiV0jacDKpuxyTnNf3cQW3qqogJC665HY+/EeJOQmfOkR87rtnxMb9rN0f8rQAArCCAAABWEEAAACsIIACAFQQQAMAKAggAYAUBBACwggACAFhBAAEArCCAAABWEEAAACsIIACAFTyMFK2S67OdQdW9NOp+xzW1STGOa6of9zqu+eyW5Y5rWqNnE7Y7rrnzwUeD2pbn3a1B1eHycAYEALCCAAIAWOE4gDZt2qRRo0YpNTVVLpdLH330UcDySZMmyeVyBUwjRowIVb8AgFbCcQDV1NSob9++WrBgQaNjRowYocOHD/un995774qaBAC0Po5vQsjJyVFOTs5Fx7jdbiUnJwfdFACg9WuSa0CFhYVKTExUz549NW3aNB07dqzRsbW1tfJ6vQETAKD1C3kAjRgxQu+88442bNigf/7nf1ZRUZFycnJUX1/f4Pj8/Hx5PB7/lJaWFuqWAADNUMg/BzR+/Hj/n2+++Wb16dNHGRkZKiws1LBhwy4YP3v2bM2aNcv/2uv1EkIAcA1o8tuwu3fvrvj4eO3bt6/B5W63W3FxcQETAKD1a/IAOnjwoI4dO6aUlJSm3hQAoAVx/BbciRMnAs5mysrKtHPnTnXs2FEdO3bUiy++qHHjxik5OVmlpaV68skndf311ys7OzukjQMAWjbHAbR9+3bdcccd/tffX7+ZOHGiFi5cqF27dunf//3fdfz4caWmpmr48OH67W9/K7fbHbquAQAtnuMAGjp0qIwxjS7/9NNPr6ghwKb6r0sc10R87Xw7HQpcjmtGRf3Ucc3+P/VwXCNJn2QudFzTOaJtUNtyKtIV7rjmdMfgrjZ4gqrC5eJZcAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALAi5F/JDeAyXOSJ8o2W1NY6run6y68c10jSnW/NdFzz3/csCmpbuHZxBgQAsIIAAgBYQQABAKwggAAAVhBAAAArCCAAgBUEEADACgIIAGAFAQQAsIIAAgBYQQABAKwggAAAVvAwUqAVc0VGBVfXtj7EnYTOrjPOe0vcXtMEneBKcQYEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFbwMFKgFSt545ag6v572MLQNhJCMx6f7rgm+vNtTdAJrhRnQAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQ8jRasUFhsbXF17T4g7adiRu9Ic19w1/TPHNf+RuMBxzTlX53fTD04kOq6J2/KN45qzjitwNXAGBACwggACAFjhKIDy8/N12223KTY2VomJiRozZoxKSkoCxpw+fVq5ubnq1KmTYmJiNG7cOFVWVoa0aQBAy+cogIqKipSbm6utW7dq3bp1qqur0/Dhw1VTU+MfM3PmTH388cdasWKFioqKdOjQIY0dOzbkjQMAWjZHNyGsXbs24PWSJUuUmJioHTt2aMiQIaqqqtK//du/admyZbrzzjslSYsXL9ZNN92krVu36p/+6Z9C1zkAoEW7omtAVVVVkqSOHTtKknbs2KG6ujplZWX5x9x4443q0qWLtmzZ0uA6amtr5fV6AyYAQOsXdAD5fD7NmDFDAwcOVO/evSVJFRUVioqKUvv27QPGJiUlqaKiosH15Ofny+Px+Ke0NOe3pwIAWp6gAyg3N1e7d+/W8uXLr6iB2bNnq6qqyj+Vl5df0foAAC1DUB9EzcvL0+rVq7Vp0yZ17tzZPz85OVlnzpzR8ePHA86CKisrlZyc3OC63G633G53MG0AAFowR2dAxhjl5eVp5cqV2rhxo9LT0wOW9+vXT5GRkdqwYYN/XklJiQ4cOKABAwaEpmMAQKvg6AwoNzdXy5Yt06pVqxQbG+u/ruPxeNS2bVt5PB796le/0qxZs9SxY0fFxcVp+vTpGjBgAHfAAQACOAqghQsXSpKGDh0aMH/x4sWaNGmSJOn3v/+9wsLCNG7cONXW1io7O1tvvvlmSJoFALQeLmOMsd3E+bxerzwej4ZqtCJckbbbuSaE9b0pqLo9uTGOa5LT/q/jmiMlCY5rJt9Z6LhGkp7q9HVQdQhOn88nOa7p8ouvQt8IQuqsqVOhVqmqqkpxcXGNjuNZcAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALAiqG9ERfPl6tfLcU3b3x8Jalv/nfFuUHWO9bk6m2nuak2d45pIV3hQ26qsr3VcM+dQjuOazq8F1x9aB86AAABWEEAAACsIIACAFQQQAMAKAggAYAUBBACwggACAFhBAAEArCCAAABWEEAAACsIIACAFQQQAMAKHkbaytR0jXFc8373fw1ya1FB1jU9n3xB1c08NNhxzW8S1zuuyf4813FNbGG045rqbo5LJEnps7cEUVXtuCJMO4PYDloLzoAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoeRtrKRH+4zXHNLw4/HNS2jv64neMaXxDPL61z/nxVvf1//uC8SFLpbacd10z78VTHNek7dzmukTGOS+KdbwW4ajgDAgBYQQABAKwggAAAVhBAAAArCCAAgBUEEADACgIIAGAFAQQAsIIAAgBYQQABAKwggAAAVhBAAAArXMYE8YTDJuT1euXxeDRUoxXhirTdDgDAobOmToVapaqqKsXFxTU6jjMgAIAVBBAAwApHAZSfn6/bbrtNsbGxSkxM1JgxY1RSUhIwZujQoXK5XAHTww8H930zAIDWy1EAFRUVKTc3V1u3btW6detUV1en4cOHq6amJmDclClTdPjwYf80b968kDYNAGj5HH0j6tq1awNeL1myRImJidqxY4eGDBninx8dHa3k5OTQdAgAaJWu6BpQVVWVJKljx44B85cuXar4+Hj17t1bs2fP1smTJxtdR21trbxeb8AEAGj9HJ0Bnc/n82nGjBkaOHCgevfu7Z9///33q2vXrkpNTdWuXbv01FNPqaSkRB9++GGD68nPz9eLL74YbBsAgBYq6M8BTZs2TZ988ok2b96szp07Nzpu48aNGjZsmPbt26eMjIwLltfW1qq2ttb/2uv1Ki0tjc8BAUALdbmfAwrqDCgvL0+rV6/Wpk2bLho+kpSZmSlJjQaQ2+2W2+0Opg0AQAvmKICMMZo+fbpWrlypwsJCpaenX7Jm586dkqSUlJSgGgQAtE6OAig3N1fLli3TqlWrFBsbq4qKCkmSx+NR27ZtVVpaqmXLlmnkyJHq1KmTdu3apZkzZ2rIkCHq06dPk/wAAICWydE1IJfL1eD8xYsXa9KkSSovL9eDDz6o3bt3q6amRmlpabr33nv17LPPXvR9wPPxLDgAaNma5BrQpbIqLS1NRUVFTlYJALhG8Sw4AIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAVEbYb+CFjjCTprOokY7kZAIBjZ1Un6X/+P29Mswug6upqSdJmrbHcCQDgSlRXV8vj8TS63GUuFVFXmc/n06FDhxQbGyuXyxWwzOv1Ki0tTeXl5YqLi7PUoX3sh3PYD+ewH85hP5zTHPaDMUbV1dVKTU1VWFjjV3qa3RlQWFiYOnfufNExcXFx1/QB9j32wznsh3PYD+ewH86xvR8udubzPW5CAABYQQABAKxoUQHkdrs1Z84cud1u261YxX44h/1wDvvhHPbDOS1pPzS7mxAAANeGFnUGBABoPQggAIAVBBAAwAoCCABgBQEEALCixQTQggUL1K1bN7Vp00aZmZn64osvbLd01b3wwgtyuVwB04033mi7rSa3adMmjRo1SqmpqXK5XProo48Clhtj9PzzzyslJUVt27ZVVlaW9u7da6fZJnSp/TBp0qQLjo8RI0bYabaJ5Ofn67bbblNsbKwSExM1ZswYlZSUBIw5ffq0cnNz1alTJ8XExGjcuHGqrKy01HHTuJz9MHTo0AuOh4cffthSxw1rEQH0/vvva9asWZozZ46+/PJL9e3bV9nZ2Tpy5Ijt1q66Xr166fDhw/5p8+bNtltqcjU1Nerbt68WLFjQ4PJ58+bp9ddf16JFi7Rt2za1a9dO2dnZOn369FXutGldaj9I0ogRIwKOj/fee+8qdtj0ioqKlJubq61bt2rdunWqq6vT8OHDVVNT4x8zc+ZMffzxx1qxYoWKiop06NAhjR071mLXoXc5+0GSpkyZEnA8zJs3z1LHjTAtQP/+/U1ubq7/dX19vUlNTTX5+fkWu7r65syZY/r27Wu7DaskmZUrV/pf+3w+k5ycbF5++WX/vOPHjxu3223ee+89Cx1eHT/cD8YYM3HiRDN69Ggr/dhy5MgRI8kUFRUZY8793UdGRpoVK1b4x/zjH/8wksyWLVtstdnkfrgfjDHm9ttvN4899pi9pi5Dsz8DOnPmjHbs2KGsrCz/vLCwMGVlZWnLli0WO7Nj7969Sk1NVffu3fXAAw/owIEDtluyqqysTBUVFQHHh8fjUWZm5jV5fBQWFioxMVE9e/bUtGnTdOzYMdstNamqqipJUseOHSVJO3bsUF1dXcDxcOONN6pLly6t+nj44X743tKlSxUfH6/evXtr9uzZOnnypI32GtXsnob9Q99++63q6+uVlJQUMD8pKUl79uyx1JUdmZmZWrJkiXr27KnDhw/rxRdf1ODBg7V7927Fxsbabs+KiooKSWrw+Ph+2bVixIgRGjt2rNLT01VaWqpnnnlGOTk52rJli8LDw223F3I+n08zZszQwIED1bt3b0nnjoeoqCi1b98+YGxrPh4a2g+SdP/996tr165KTU3Vrl279NRTT6mkpEQffvihxW4DNfsAwv/Iycnx/7lPnz7KzMxU165d9cEHH+hXv/qVxc7QHIwfP97/55tvvll9+vRRRkaGCgsLNWzYMIudNY3c3Fzt3r37mrgOejGN7YeHHnrI/+ebb75ZKSkpGjZsmEpLS5WRkXG122xQs38LLj4+XuHh4RfcxVJZWank5GRLXTUP7du3V48ePbRv3z7brVjz/THA8XGh7t27Kz4+vlUeH3l5eVq9erUKCgoCvj8sOTlZZ86c0fHjxwPGt9bjobH90JDMzExJalbHQ7MPoKioKPXr108bNmzwz/P5fNqwYYMGDBhgsTP7Tpw4odLSUqWkpNhuxZr09HQlJycHHB9er1fbtm275o+PgwcP6tixY63q+DDGKC8vTytXrtTGjRuVnp4esLxfv36KjIwMOB5KSkp04MCBVnU8XGo/NGTnzp2S1LyOB9t3QVyO5cuXG7fbbZYsWWL+/ve/m4ceesi0b9/eVFRU2G7tqnr88cdNYWGhKSsrM5999pnJysoy8fHx5siRI7Zba1LV1dWmuLjYFBcXG0nm1VdfNcXFxeabb74xxhjzu9/9zrRv396sWrXK7Nq1y4wePdqkp6ebU6dOWe48tC62H6qrq80TTzxhtmzZYsrKysz69evNT37yE3PDDTeY06dP2249ZKZNm2Y8Ho8pLCw0hw8f9k8nT570j3n44YdNly5dzMaNG8327dvNgAEDzIABAyx2HXqX2g/79u0zL730ktm+fbspKyszq1atMt27dzdDhgyx3HmgFhFAxhjzxhtvmC5dupioqCjTv39/s3XrVtstXXX33XefSUlJMVFRUea6664z9913n9m3b5/ttppcQUGBkXTBNHHiRGPMuVuxn3vuOZOUlGTcbrcZNmyYKSkpsdt0E7jYfjh58qQZPny4SUhIMJGRkaZr165mypQpre6XtIZ+fklm8eLF/jGnTp0yjzzyiOnQoYOJjo429957rzl8+LC9ppvApfbDgQMHzJAhQ0zHjh2N2+02119/vfnNb35jqqqq7Db+A3wfEADAimZ/DQgA0DoRQAAAKwggAIAVBBAAwAoCCABgBQEEALCCAAIAWEEAAQCsIIAAAFYQQAAAKwggAIAV/w/hgVLrpVGHsAAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGzCAYAAABpdMNsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAiTklEQVR4nO3dfXBV9b3v8c/O0+YpCYQ8S8CAAhYET1FyuCCipAlBHVF6KmrvBY4FpQHFHGsPTgVRZtJDTzmoTcE59xTaUxAP0wK3lKKAJBQKdEAYBqu5kMYCAwnImAQChIf9u39w2cdNArg2O3zz8H7NrJnstX7ftb5ZLPiw9lp7bZ9zzgkAgFssyroBAED7RAABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEe3H777Zo0aVLwdWlpqXw+n0pLSyO2DZ/Pp9dffz1i6wNaKgIIrcbSpUvl8/mCU4cOHdS3b19Nnz5d1dXV1u15sm7dulYTMl/d51dP3/rWt6zbQysWY90A4NUbb7yh7OxsnTt3Tlu3btWiRYu0bt067d+/X506dbqlvYwcOVJnz55VXFycp7p169appKSkyRA6e/asYmJazl/N//zP/2w0b9euXXrrrbeUl5dn0BHaipZzlANfU0FBge69915J0ve+9z11795dCxYs0Jo1a/TUU081WVNfX6/OnTtHvJeoqCh16NAhouuM9Ppu1ne/+91G86689Xit/Q18HbwFh1bvoYcekiRVVlZKkiZNmqQuXbqooqJCY8eOVXx8vJ555hlJUiAQ0MKFCzVgwAB16NBBaWlpeu655/Tll1+GrNM5p3nz5qlHjx7q1KmTHnzwQX3yySeNtn2ta0A7d+7U2LFj1a1bN3Xu3FmDBg3SW2+9FeyvpKREUujbW1c0dQ1oz549KigoUEJCgrp06aLRo0drx44dIWOuvEW5bds2FRUVKSUlRZ07d9bjjz+uEydOhIytra3VZ599ptra2q+zi0M0NDToN7/5jR544AH16NHDcz1wBWdAaPUqKiokSd27dw/Ou3jxovLz8zVixAj967/+a/Ctueeee05Lly7V5MmT9cILL6iyslI/+9nPtGfPHm3btk2xsbGSpNmzZ2vevHkaO3asxo4dq48//lh5eXk6f/78DfvZsGGDHnnkEWVkZOjFF19Uenq6Pv30U61du1YvvviinnvuOR09elQbNmxo8u2tq33yySe6//77lZCQoFdeeUWxsbF69913NWrUKJWVlSknJydk/IwZM9StWzfNmTNHn3/+uRYuXKjp06fr/fffD45ZtWqVJk+erCVLloTcVPF1rFu3TjU1NcFQB8LmgFZiyZIlTpLbuHGjO3HihDt8+LBbsWKF6969u+vYsaM7cuSIc865iRMnOknun//5n0Pq//jHPzpJbtmyZSHz169fHzL/+PHjLi4uzj388MMuEAgEx7366qtOkps4cWJw3ubNm50kt3nzZueccxcvXnTZ2dmuV69e7ssvvwzZzlfXVVhY6K7110+SmzNnTvD1uHHjXFxcnKuoqAjOO3r0qIuPj3cjR45stH9yc3NDtvXSSy+56OhoV1NT02jskiVLmuzhesaPH+/8fn+j3w/wirfg0Ork5uYqJSVFWVlZmjBhgrp06aJVq1bptttuCxk3bdq0kNcrV65UYmKivvWtb+mLL74ITkOGDFGXLl20efNmSdLGjRt1/vx5zZgxI+StsZkzZ96wtz179qiyslIzZ85U165dQ5Z9dV1f16VLl/Thhx9q3Lhx6t27d3B+RkaGnn76aW3dulV1dXUhNVOnTg3Z1v33369Lly7pb3/7W3DepEmT5JzzfPZTV1en3//+9xo7dmyj3w/wirfg0OqUlJSob9++iomJUVpamvr166eoqND/S8XExDS6PnHgwAHV1tYqNTW1yfUeP35ckoL/UN95550hy1NSUtStW7fr9nbl7cCBAwd+/V/oOk6cOKEzZ86oX79+jZbdddddCgQCOnz4sAYMGBCc37Nnz5BxV3q++jpXOH7zm9/o3LlzvP2GiCCA0OoMHTo0eBfctfj9/kahFAgElJqaqmXLljVZk5KSErEeLUVHRzc53zl30+tetmyZEhMT9cgjj9z0ugACCO1Gnz59tHHjRg0fPlwdO3a85rhevXpJunzG9NW3vU6cOHHDs4g+ffpIkvbv36/c3Nxrjvu6b8elpKSoU6dOKi8vb7Tss88+U1RUlLKysr7Wum7WsWPHtHnzZk2aNEl+v/+WbBNtG9eA0G585zvf0aVLl/Tmm282Wnbx4kXV1NRIunyNKTY2Vu+8807IWcPChQtvuI1vfvObys7O1sKFC4Pru+Kr67rymaSrx1wtOjpaeXl5WrNmjT7//PPg/Orqai1fvlwjRoxQQkLCDfu6Wji3Ya9YsUKBQIC33xAxnAGh3XjggQf03HPPqbi4WHv37lVeXp5iY2N14MABrVy5Um+99Za+/e1vKyUlRS+//LKKi4v1yCOPaOzYsdqzZ4/+8Ic/KDk5+brbiIqK0qJFi/Too4/qnnvu0eTJk5WRkaHPPvtMn3zyiT744ANJ0pAhQyRJL7zwgvLz8xUdHa0JEyY0uc558+Zpw4YNGjFihL7//e8rJiZG7777rhoaGjR//vyw9kU4t2EvW7ZMmZmZGjVqVFjbBK5GAKFdWbx4sYYMGaJ3331Xr776qmJiYnT77bfru9/9roYPHx4cN2/ePHXo0EGLFy/W5s2blZOTow8//FAPP/zwDbeRn5+vzZs3a+7cufrpT3+qQCCgPn36aMqUKcExTzzxhGbMmKEVK1bo17/+tZxz1wygAQMG6I9//KNmzZql4uJiBQIB5eTk6Ne//nWjzwA1l/Lycu3evVtFRUWNrq0B4fK5SFyZBADAI/4rAwAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMtLjPAQUCAR09elTx8fFhPT0YAGDLOadTp04pMzPzup8ba3EBdPTo0Vv2bCsAQPM5fPjwdb81t8UFUHx8vCRphMYqRrHG3QAAvLqoC9qqdcF/z6+l2QKopKREP/nJT1RVVaXBgwfrnXfe0dChQ29Yd+VttxjFKsZHAAFAq/P/n69zo8sozXITwvvvv6+ioiLNmTNHH3/8sQYPHqz8/PzgF34BANAsAbRgwQJNmTJFkydP1je+8Q0tXrxYnTp10i9+8Yvm2BwAoBWKeACdP39eu3fvDvkyrqioKOXm5mr79u2Nxjc0NKiuri5kAgC0fREPoC+++EKXLl1SWlpayPy0tDRVVVU1Gl9cXKzExMTgxB1wANA+mH8QddasWaqtrQ1Ohw8ftm4JAHALRPwuuOTkZEVHR6u6ujpkfnV1tdLT0xuN9/v9fL88ALRDET8DiouL05AhQ7Rp06bgvEAgoE2bNmnYsGGR3hwAoJVqls8BFRUVaeLEibr33ns1dOhQLVy4UPX19Zo8eXJzbA4A0Ao1SwA9+eSTOnHihGbPnq2qqirdc889Wr9+faMbEwAA7ZfPOeesm/iquro6JSYmapQe40kIANAKXXQXVKo1qq2tVUJCwjXHmd8FBwBonwggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGAi4gH0+uuvy+fzhUz9+/eP9GYAAK1cTHOsdMCAAdq4ceN/bySmWTYDAGjFmiUZYmJilJ6e3hyrBgC0Ec1yDejAgQPKzMxU79699cwzz+jQoUPXHNvQ0KC6urqQCQDQ9kU8gHJycrR06VKtX79eixYtUmVlpe6//36dOnWqyfHFxcVKTEwMTllZWZFuCQDQAvmcc645N1BTU6NevXppwYIFevbZZxstb2hoUENDQ/B1XV2dsrKyNEqPKcYX25ytAQCawUV3QaVao9raWiUkJFxzXLPfHdC1a1f17dtXBw8ebHK53++X3+9v7jYAAC1Ms38O6PTp06qoqFBGRkZzbwoA0IpEPIBefvlllZWV6fPPP9ef/vQnPf7444qOjtZTTz0V6U0BAFqxiL8Fd+TIET311FM6efKkUlJSNGLECO3YsUMpKSmR3hQAoBWLeACtWLEi0qtEOxc9oJ/nmpqB3cLa1qkJ3j8G8D9uq/Rcs+1Ib881w3v81XPN1lV/57lGknq+tddzTeDMmbC2hfaLZ8EBAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAw0exfSAd8VfQd2Z5rpq7+veeahzvVeq6RpCj5PNcEFMaXCt+21XtNGKKmbwurrl9SoeeaPj/YHta20H5xBgQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMMHTsHFLueovPNcU/eEZzzUPj/+55xpJ+jJw1nPNfRtf8FwTdyTOc83+f/yZ55pw/fzx/+255q238z3XXDx8xHMN2g7OgAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJjgYaS4pQKnTnmu6f/mXz3X3HPb//JcI0kd1yd4run779s918Rk9/Jco3/0XhKu1OjTnmtcpw7N0AnaMs6AAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmOBhpGjxLp044bmmx3jvNbdSQ6/unmui5GuGTq6xLZ+7ZdtC+8UZEADABAEEADDhOYC2bNmiRx99VJmZmfL5fFq9enXIcuecZs+erYyMDHXs2FG5ubk6cOBApPoFALQRngOovr5egwcPVklJSZPL58+fr7fffluLFy/Wzp071blzZ+Xn5+vcuXM33SwAoO3wfBNCQUGBCgoKmlzmnNPChQv1ox/9SI899pgk6Ve/+pXS0tK0evVqTZgw4ea6BQC0GRG9BlRZWamqqirl5uYG5yUmJionJ0fbtzf9tcUNDQ2qq6sLmQAAbV9EA6iqqkqSlJaWFjI/LS0tuOxqxcXFSkxMDE5ZWVmRbAkA0EKZ3wU3a9Ys1dbWBqfDhw9btwQAuAUiGkDp6emSpOrq6pD51dXVwWVX8/v9SkhICJkAAG1fRAMoOztb6enp2rRpU3BeXV2ddu7cqWHDhkVyUwCAVs7zXXCnT5/WwYMHg68rKyu1d+9eJSUlqWfPnpo5c6bmzZunO++8U9nZ2XrttdeUmZmpcePGRbJvAEAr5zmAdu3apQcffDD4uqioSJI0ceJELV26VK+88orq6+s1depU1dTUaMSIEVq/fr06dOgQua4BAK2e5wAaNWqUnLv2gwp9Pp/eeOMNvfHGGzfVGNCWHc71e64JyPsDQsN9gGlS1EXPNYEu3n8ntG/md8EBANonAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJz0/DBnDzfH1PW7dwXfOPP3jjQVdxuz9phk7QlnEGBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQPIwVu0ul/yPFc839yFoSxpQ5h1ITngz/c67nmdm1vhk7QlnEGBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQPIwVu0tHcgOeaPjEdm6GTyMncdtG6BbQDnAEBAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwwcNIga+I7p7kueahwZ96rgnIea4JR9/fPx9e3YcfR7gToDHOgAAAJgggAIAJzwG0ZcsWPfroo8rMzJTP59Pq1atDlk+aNEk+ny9kGjNmTKT6BQC0EZ4DqL6+XoMHD1ZJSck1x4wZM0bHjh0LTu+9995NNQkAaHs834RQUFCggoKC647x+/1KT08PuykAQNvXLNeASktLlZqaqn79+mnatGk6efLkNcc2NDSorq4uZAIAtH0RD6AxY8boV7/6lTZt2qR/+Zd/UVlZmQoKCnTp0qUmxxcXFysxMTE4ZWVlRbolAEALFPHPAU2YMCH48913361BgwapT58+Ki0t1ejRoxuNnzVrloqKioKv6+rqCCEAaAea/Tbs3r17Kzk5WQcPHmxyud/vV0JCQsgEAGj7mj2Ajhw5opMnTyojI6O5NwUAaEU8vwV3+vTpkLOZyspK7d27V0lJSUpKStLcuXM1fvx4paenq6KiQq+88oruuOMO5efnR7RxAEDr5jmAdu3apQcffDD4+sr1m4kTJ2rRokXat2+ffvnLX6qmpkaZmZnKy8vTm2++Kb/fH7muAQCtnucAGjVqlJy79oMUP/jgg5tqCLBUOaO/55o1We80QyeR8Y3Zh8Kquxho+q5VIJJ4FhwAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwETEv5IbaM1Gjt1j3cI13VX6Pc81fapa7u8DcAYEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABA8jBb7i57dtC6PK57ni/14457mm32tfeq656LkCuHU4AwIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCh5GiTTr9DzlhVn7suSIg57nmO3u+57km869/8VwDtGScAQEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADDBw0jR4kV3TfRc8z/nrm2GTiIn/adx1i0A5jgDAgCYIIAAACY8BVBxcbHuu+8+xcfHKzU1VePGjVN5eXnImHPnzqmwsFDdu3dXly5dNH78eFVXV0e0aQBA6+cpgMrKylRYWKgdO3Zow4YNunDhgvLy8lRfXx8c89JLL+l3v/udVq5cqbKyMh09elRPPPFExBsHALRunm5CWL9+fcjrpUuXKjU1Vbt379bIkSNVW1ur//iP/9Dy5cv10EMPSZKWLFmiu+66Szt27NDf//3fR65zAECrdlPXgGprayVJSUlJkqTdu3frwoULys3NDY7p37+/evbsqe3btze5joaGBtXV1YVMAIC2L+wACgQCmjlzpoYPH66BAwdKkqqqqhQXF6euXbuGjE1LS1NVVVWT6ykuLlZiYmJwysrKCrclAEArEnYAFRYWav/+/VqxYsVNNTBr1izV1tYGp8OHD9/U+gAArUNYH0SdPn261q5dqy1btqhHjx7B+enp6Tp//rxqampCzoKqq6uVnp7e5Lr8fr/8fn84bQAAWjFPZ0DOOU2fPl2rVq3SRx99pOzs7JDlQ4YMUWxsrDZt2hScV15erkOHDmnYsGGR6RgA0CZ4OgMqLCzU8uXLtWbNGsXHxwev6yQmJqpjx45KTEzUs88+q6KiIiUlJSkhIUEzZszQsGHDuAMOABDCUwAtWrRIkjRq1KiQ+UuWLNGkSZMkSf/2b/+mqKgojR8/Xg0NDcrPz9fPf/7ziDQLAGg7PAWQc+6GYzp06KCSkhKVlJSE3RTwVb5uXT3XPJt4KNythVkHwCueBQcAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMBHWN6ICLV1UmE+1jvaF8X8yFwhrW0B7xxkQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEzyMFC1e5TO3ea4JyIW3sTAeLJr36TjPNbE7/+K5JszfCGixOAMCAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABggoeRosVL3n/Rc83imt5hbevb8Z94rhmZctBzzZ8uxHmuAdoazoAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCY4GGkaPE6rv6z55r1+/4urG0t+EG+55r4g97/GmXoT55rgLaGMyAAgAkCCABgwlMAFRcX67777lN8fLxSU1M1btw4lZeXh4wZNWqUfD5fyPT8889HtGkAQOvnKYDKyspUWFioHTt2aMOGDbpw4YLy8vJUX18fMm7KlCk6duxYcJo/f35EmwYAtH6erp6uX78+5PXSpUuVmpqq3bt3a+TIkcH5nTp1Unp6emQ6BAC0STd1Dai2tlaSlJSUFDJ/2bJlSk5O1sCBAzVr1iydOXPmmutoaGhQXV1dyAQAaPvCvg07EAho5syZGj58uAYOHBic//TTT6tXr17KzMzUvn379MMf/lDl5eX67W9/2+R6iouLNXfu3HDbAAC0UmEHUGFhofbv36+tW7eGzJ86dWrw57vvvlsZGRkaPXq0Kioq1KdPn0brmTVrloqKioKv6+rqlJWVFW5bAIBWIqwAmj59utauXastW7aoR48e1x2bk5MjSTp48GCTAeT3++X3+8NpAwDQinkKIOecZsyYoVWrVqm0tFTZ2dk3rNm7d68kKSMjI6wGAQBtk6cAKiws1PLly7VmzRrFx8erqqpKkpSYmKiOHTuqoqJCy5cv19ixY9W9e3ft27dPL730kkaOHKlBgwY1yy8AAGidPAXQokWLJF3+sOlXLVmyRJMmTVJcXJw2btyohQsXqr6+XllZWRo/frx+9KMfRaxhAEDb4PktuOvJyspSWVnZTTUEAGgfeBo22qSLf/08rLq+08KrA+AdDyMFAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgIsa6gas55yRJF3VBcsbNAAA8u6gLkv773/NraXEBdOrUKUnSVq0z7gQAcDNOnTqlxMTEay73uRtF1C0WCAR09OhRxcfHy+fzhSyrq6tTVlaWDh8+rISEBKMO7bEfLmM/XMZ+uIz9cFlL2A/OOZ06dUqZmZmKirr2lZ4WdwYUFRWlHj16XHdMQkJCuz7ArmA/XMZ+uIz9cBn74TLr/XC9M58ruAkBAGCCAAIAmGhVAeT3+zVnzhz5/X7rVkyxHy5jP1zGfriM/XBZa9oPLe4mBABA+9CqzoAAAG0HAQQAMEEAAQBMEEAAABMEEADARKsJoJKSEt1+++3q0KGDcnJy9Oc//9m6pVvu9ddfl8/nC5n69+9v3Vaz27Jlix599FFlZmbK5/Np9erVIcudc5o9e7YyMjLUsWNH5ebm6sCBAzbNNqMb7YdJkyY1Oj7GjBlj02wzKS4u1n333af4+HilpqZq3LhxKi8vDxlz7tw5FRYWqnv37urSpYvGjx+v6upqo46bx9fZD6NGjWp0PDz//PNGHTetVQTQ+++/r6KiIs2ZM0cff/yxBg8erPz8fB0/fty6tVtuwIABOnbsWHDaunWrdUvNrr6+XoMHD1ZJSUmTy+fPn6+3335bixcv1s6dO9W5c2fl5+fr3Llzt7jT5nWj/SBJY8aMCTk+3nvvvVvYYfMrKytTYWGhduzYoQ0bNujChQvKy8tTfX19cMxLL72k3/3ud1q5cqXKysp09OhRPfHEE4ZdR97X2Q+SNGXKlJDjYf78+UYdX4NrBYYOHeoKCwuDry9duuQyMzNdcXGxYVe33pw5c9zgwYOt2zAlya1atSr4OhAIuPT0dPeTn/wkOK+mpsb5/X733nvvGXR4a1y9H5xzbuLEie6xxx4z6cfK8ePHnSRXVlbmnLv8Zx8bG+tWrlwZHPPpp586SW779u1WbTa7q/eDc8498MAD7sUXX7Rr6mto8WdA58+f1+7du5WbmxucFxUVpdzcXG3fvt2wMxsHDhxQZmamevfurWeeeUaHDh2ybslUZWWlqqqqQo6PxMRE5eTktMvjo7S0VKmpqerXr5+mTZumkydPWrfUrGprayVJSUlJkqTdu3frwoULIcdD//791bNnzzZ9PFy9H65YtmyZkpOTNXDgQM2aNUtnzpyxaO+aWtzTsK/2xRdf6NKlS0pLSwuZn5aWps8++8yoKxs5OTlaunSp+vXrp2PHjmnu3Lm6//77tX//fsXHx1u3Z6KqqkqSmjw+rixrL8aMGaMnnnhC2dnZqqio0KuvvqqCggJt375d0dHR1u1FXCAQ0MyZMzV8+HANHDhQ0uXjIS4uTl27dg0Z25aPh6b2gyQ9/fTT6tWrlzIzM7Vv3z798Ic/VHl5uX77298adhuqxQcQ/ltBQUHw50GDBiknJ0e9evXSf/3Xf+nZZ5817AwtwYQJE4I/33333Ro0aJD69Omj0tJSjR492rCz5lFYWKj9+/e3i+ug13Ot/TB16tTgz3fffbcyMjI0evRoVVRUqE+fPre6zSa1+LfgkpOTFR0d3egulurqaqWnpxt11TJ07dpVffv21cGDB61bMXPlGOD4aKx3795KTk5uk8fH9OnTtXbtWm3evDnk+8PS09N1/vx51dTUhIxvq8fDtfZDU3JyciSpRR0PLT6A4uLiNGTIEG3atCk4LxAIaNOmTRo2bJhhZ/ZOnz6tiooKZWRkWLdiJjs7W+np6SHHR11dnXbu3Nnuj48jR47o5MmTber4cM5p+vTpWrVqlT766CNlZ2eHLB8yZIhiY2NDjofy8nIdOnSoTR0PN9oPTdm7d68ktazjwfouiK9jxYoVzu/3u6VLl7q//OUvburUqa5r166uqqrKurVb6p/+6Z9caWmpq6ysdNu2bXO5ubkuOTnZHT9+3Lq1ZnXq1Cm3Z88et2fPHifJLViwwO3Zs8f97W9/c8459+Mf/9h17drVrVmzxu3bt8899thjLjs72509e9a488i63n44deqUe/nll9327dtdZWWl27hxo/vmN7/p7rzzTnfu3Dnr1iNm2rRpLjEx0ZWWlrpjx44FpzNnzgTHPP/8865nz57uo48+crt27XLDhg1zw4YNM+w68m60Hw4ePOjeeOMNt2vXLldZWenWrFnjevfu7UaOHGnceahWEUDOOffOO++4nj17uri4ODd06FC3Y8cO65ZuuSeffNJlZGS4uLg4d9ttt7knn3zSHTx40LqtZrd582YnqdE0ceJE59zlW7Ffe+01l5aW5vx+vxs9erQrLy+3bboZXG8/nDlzxuXl5bmUlBQXGxvrevXq5aZMmdLm/pPW1O8vyS1ZsiQ45uzZs+773/++69atm+vUqZN7/PHH3bFjx+yabgY32g+HDh1yI0eOdElJSc7v97s77rjD/eAHP3C1tbW2jV+F7wMCAJho8deAAABtEwEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBM/D/AaY3Zb7z6aAAAAABJRU5ErkJggg==", "text/plain": [ "
" ] @@ -1938,254 +1959,364 @@ }, { "cell_type": "markdown", - "id": "5961593d-182e-4620-9a5e-f98ba3d2534d", + "id": "d3dc87a7", "metadata": {}, "source": [ - "### Using Triton Inference Server\n", + "## Using Triton Inference Server\n", + "In this section, we demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. \n", + "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server. \n", + "\n", + "The process looks like this:\n", + "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n", + "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n", + "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n", + "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n", "\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment." + "\"drawing\"" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "cfc841c3", + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial" + ] + }, + { + "cell_type": "markdown", + "id": "d1e63867", + "metadata": {}, + "source": [ + "Import the utility functions from pytriton_utils.py:" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "d7af3599", + "metadata": {}, + "outputs": [], + "source": [ + "sc.addPyFile(\"pytriton_utils.py\")\n", + "\n", + "from pytriton_utils import (\n", + " use_stage_level_scheduling,\n", + " find_ports,\n", + " start_triton,\n", + " stop_triton\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "32cbe1cb", + "metadata": {}, + "source": [ + "Define the Triton Server function:" ] }, { "cell_type": "code", "execution_count": 52, - "id": "a64d19b1-ba4a-4dc7-b3a9-368dc47d0fd8", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "id": "c3539d1b", + "metadata": {}, "outputs": [], "source": [ - "import os\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import col, struct\n", - "from pyspark.sql.types import ArrayType, FloatType" + "def triton_server(ports, model_path):\n", + " import time\n", + " import signal\n", + " import numpy as np\n", + " from pytriton.decorators import batch\n", + " from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n", + " from pytriton.triton import Triton, TritonConfig\n", + " from pyspark import TaskContext\n", + " import tensorflow as tf\n", + " from tensorflow import keras\n", + "\n", + " print(f\"SERVER: Initializing model on worker {TaskContext.get().partitionId()}.\")\n", + "\n", + " # Enable GPU memory growth\n", + " gpus = tf.config.experimental.list_physical_devices('GPU')\n", + " if gpus:\n", + " try:\n", + " for gpu in gpus:\n", + " tf.config.experimental.set_memory_growth(gpu, True)\n", + " except RuntimeError as e:\n", + " print(e)\n", + "\n", + " model = keras.models.load_model(model_path)\n", + "\n", + " @batch\n", + " def _infer_fn(**inputs):\n", + " images = np.squeeze(inputs[\"images\"])\n", + " print(f\"SERVER: Received batch of size {len(images)}.\")\n", + " return {\n", + " \"labels\": model.predict(images)\n", + " }\n", + "\n", + " workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n", + " triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n", + " with Triton(config=triton_conf, workspace=workspace_path) as triton:\n", + " triton.bind(\n", + " model_name=\"ImageClassifier\",\n", + " infer_func=_infer_fn,\n", + " inputs=[\n", + " Tensor(name=\"images\", dtype=np.float64, shape=(-1,)),\n", + " ],\n", + " outputs=[\n", + " Tensor(name=\"labels\", dtype=np.float32, shape=(-1,)),\n", + " ],\n", + " config=ModelConfig(\n", + " max_batch_size=128,\n", + " batcher=DynamicBatcher(max_queue_delay_microseconds=5000), # 5ms\n", + " ),\n", + " strict=True,\n", + " )\n", + "\n", + " def _stop_triton(signum, frame):\n", + " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", + " triton.stop()\n", + "\n", + " signal.signal(signal.SIGTERM, _stop_triton)\n", + "\n", + " print(\"SERVER: Serving inference\")\n", + " triton.serve()" + ] + }, + { + "cell_type": "markdown", + "id": "ce4c7701", + "metadata": {}, + "source": [ + "#### Start Triton servers" + ] + }, + { + "cell_type": "markdown", + "id": "629541a2", + "metadata": {}, + "source": [ + "**Specify the number of nodes in the cluster.** \n", + "Following the README, the example standalone cluster uses 1 node. The example Databricks/Dataproc cluster scripts use 4 nodes by default. " ] }, { "cell_type": "code", "execution_count": 53, - "id": "8fa92fe4-2e04-4d82-a357-bfdfca38bd8c", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "id": "36374a82", + "metadata": {}, "outputs": [], "source": [ - "%%bash\n", - "# copy model to expected layout for Triton\n", - "rm -rf models\n", - "mkdir -p models/mnist_model/1\n", - "cp -r mnist_model models/mnist_model/1/model.savedmodel\n", - "\n", - "# add config.pbtxt\n", - "cp models_config/mnist_model/config.pbtxt models/mnist_model/config.pbtxt" + "# Change based on cluster setup\n", + "num_nodes = 1 if on_standalone else 4" ] }, { "cell_type": "markdown", - "id": "f1673e0e-5c75-44e1-88c6-5f5cf1275e4b", + "id": "73d1e5cb", "metadata": {}, "source": [ - "#### Start Triton Server on each executor" + "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " ] }, { "cell_type": "code", "execution_count": 54, - "id": "0f7ecb25-be16-40c4-bdbb-441e2f537000", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "id": "4deae3b1", + "metadata": {}, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - " \r" + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" ] - }, - { - "data": { - "text/plain": [ - "[True]" - ] - }, - "execution_count": 54, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ - "num_executors = 1\n", - "triton_models_dir = \"{}/models\".format(os.getcwd())\n", - "nodeRDD = sc.parallelize(list(range(num_executors)), num_executors)\n", - "\n", - "def start_triton(it):\n", - " import docker\n", - " import time\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " if containers:\n", - " \n", - " print(\">>>> containers: {}\".format([c.short_id for c in containers]))\n", - " else:\n", - " container=client.containers.run(\n", - " \"nvcr.io/nvidia/tritonserver:24.08-py3\", \"tritonserver --model-repository=/models\",\n", - " detach=True,\n", - " device_requests=[docker.types.DeviceRequest(device_ids=[\"0\"], capabilities=[['gpu']])],\n", - " name=\"spark-triton\",\n", - " network_mode=\"host\",\n", - " remove=True,\n", - " shm_size=\"64M\",\n", - " volumes={triton_models_dir: {\"bind\": \"/models\", \"mode\": \"ro\"}}\n", - " )\n", - " print(\">>>> starting triton: {}\".format(container.short_id))\n", - "\n", - " # wait for triton to be running\n", - " time.sleep(15)\n", - " client = grpcclient.InferenceServerClient(\"localhost:8001\")\n", - " ready = False\n", - " while not ready:\n", - " try:\n", - " ready = client.is_server_ready()\n", - " except Exception as e:\n", - " time.sleep(5)\n", - " \n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(start_triton).collect()" + "sc = spark.sparkContext\n", + "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" + ] + }, + { + "cell_type": "markdown", + "id": "7fc49b0b", + "metadata": {}, + "source": [ + "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." ] }, { "cell_type": "code", "execution_count": 55, - "id": "43b93753-1d52-4060-9986-f24c30a67528", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "id": "3ffd2734", + "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "StructType([StructField('data', ArrayType(DoubleType(), True), True)])" - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "Using ports [7000, 7001, 7002]\n" + ] } ], "source": [ - "df = spark.read.parquet(\"mnist_1\")\n", - "df.schema" + "model_name = \"ImageClassifier\"\n", + "ports = find_ports()\n", + "assert len(ports) == 3\n", + "print(f\"Using ports {ports}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "id": "b6913b2c", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 16:> (0 + 1) / 1]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Triton Server PIDs:\n", + " {\n", + " \"cb4ae00-lcedt\": 3034797\n", + "}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", + " ports=ports,\n", + " model_name=model_name,\n", + " model_path=model_path)).collectAsMap()\n", + "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" ] }, { "cell_type": "markdown", - "id": "036680eb-babd-4b07-8b2c-ce6e724f4e85", + "id": "77847814", "metadata": {}, "source": [ - "#### Run inference" + "#### Define client function" ] }, { "cell_type": "code", - "execution_count": 56, - "id": "3af08bd0-3838-4769-a8de-2643db4101c6", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 57, + "id": "53f58831", + "metadata": {}, "outputs": [], "source": [ - "def triton_fn(triton_uri, model_name):\n", - " import numpy as np\n", - " import tritonclient.grpc as grpcclient\n", - "\n", - " np_types = {\n", - " \"BOOL\": np.dtype(np.bool_),\n", - " \"INT8\": np.dtype(np.int8),\n", - " \"INT16\": np.dtype(np.int16),\n", - " \"INT32\": np.dtype(np.int32),\n", - " \"INT64\": np.dtype(np.int64),\n", - " \"FP16\": np.dtype(np.float16),\n", - " \"FP32\": np.dtype(np.float32),\n", - " \"FP64\": np.dtype(np.float64),\n", - " \"FP64\": np.dtype(np.double),\n", - " \"BYTES\": np.dtype(object)\n", - " }\n", - "\n", - " client = grpcclient.InferenceServerClient(triton_uri)\n", - " model_meta = client.get_model_metadata(model_name)\n", - "\n", - " def predict(inputs):\n", - " if isinstance(inputs, np.ndarray):\n", - " # single ndarray input\n", - " request = [grpcclient.InferInput(model_meta.inputs[0].name, inputs.shape, model_meta.inputs[0].datatype)]\n", - " request[0].set_data_from_numpy(inputs.astype(np_types[model_meta.inputs[0].datatype]))\n", - " else:\n", - " # dict of multiple ndarray inputs\n", - " request = [grpcclient.InferInput(i.name, inputs[i.name].shape, i.datatype) for i in model_meta.inputs]\n", - " for i in request:\n", - " i.set_data_from_numpy(inputs[i.name()].astype(np_types[i.datatype()]))\n", - "\n", - " response = client.infer(model_name, inputs=request)\n", + "url = f\"http://localhost:{ports[0]}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "id": "92ba2e26", + "metadata": {}, + "outputs": [], + "source": [ + "def triton_fn(url, model_name):\n", + " from pytriton.client import ModelClient\n", "\n", - " if len(model_meta.outputs) > 1:\n", - " # return dictionary of numpy arrays\n", - " return {o.name: response.as_numpy(o.name) for o in model_meta.outputs}\n", - " else:\n", - " # return single numpy array\n", - " return response.as_numpy(model_meta.outputs[0].name)\n", + " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", "\n", - " return predict" + " def infer_batch(inputs):\n", + " with ModelClient(url, model_name, inference_timeout_s=240) as client:\n", + " result_data = client.infer_batch(inputs)\n", + " return result_data[\"labels\"]\n", + " \n", + " return infer_batch" + ] + }, + { + "cell_type": "markdown", + "id": "3842c263", + "metadata": {}, + "source": [ + "#### Run inference" ] }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 59, + "id": "43b93753-1d52-4060-9986-f24c30a67528", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "StructType([StructField('data', ArrayType(DoubleType(), True), True)])" + ] + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = spark.read.parquet(data_path_1)\n", + "df.schema" + ] + }, + { + "cell_type": "code", + "execution_count": 60, "id": "6658d2a1-ef7b-4ca1-9fb6-f2ac9050f3e5", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ - "from functools import partial\n", - "\n", - "predict = predict_batch_udf(partial(triton_fn, \"localhost:8001\", \"mnist_model\"),\n", - " return_type=ArrayType(FloatType()),\n", + "predict = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", " input_tensor_shapes=[[784]],\n", - " batch_size=8192)" + " return_type=ArrayType(FloatType()),\n", + " batch_size=128)" ] }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 61, "id": "8397aa14-82fd-4351-a477-dc8e8b321fa2", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 18:> (0 + 8) / 8]\r" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 20.1 ms, sys: 3.41 ms, total: 23.5 ms\n", - "Wall time: 625 ms\n" + "CPU times: user 21.3 ms, sys: 5.16 ms, total: 26.5 ms\n", + "Wall time: 1.7 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" ] } ], @@ -2196,20 +2327,16 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 62, "id": "82698bd9-377a-4415-8971-835487f876cc", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 30.3 ms, sys: 8.81 ms, total: 39.2 ms\n", - "Wall time: 154 ms\n" + "CPU times: user 17.4 ms, sys: 7.83 ms, total: 25.3 ms\n", + "Wall time: 445 ms\n" ] } ], @@ -2220,20 +2347,30 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 63, "id": "419ad7bd-fa28-49d3-b98d-db9fba5aeaef", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 20:==============> (2 + 6) / 8]\r" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.67 ms, sys: 4.2 ms, total: 6.87 ms\n", - "Wall time: 131 ms\n" + "CPU times: user 8.15 ms, sys: 2.67 ms, total: 10.8 ms\n", + "Wall time: 892 ms\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" ] }, { @@ -2265,52 +2402,52 @@ " \n", " 0\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-5.7614846, -3.52228, -1.1202906, 13.053683, ...\n", + " [-2.9825501, -2.4956138, 1.2573452, 12.891572,...\n", " \n", " \n", " 1\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-3.1390061, -8.71185, 0.82955813, -4.034869, ...\n", + " [-0.9547625, -5.622221, 2.2415693, -2.0017662,...\n", " \n", " \n", " 2\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-3.046528, 0.3521706, 0.6788677, 0.72303534, ...\n", + " [-0.9178927, 0.4341122, 1.9357599, 1.7975069, ...\n", " \n", " \n", " 3\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-2.401024, -7.6780066, 11.145876, 1.2857256, ...\n", + " [0.2115398, -5.582211, 11.52417, 2.8064039, -3...\n", " \n", " \n", " 4\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-5.0012593, 3.806796, -0.8154834, -0.9550028,...\n", + " [-3.395469, 3.7420897, -0.25319868, 0.16790543...\n", " \n", " \n", " 5\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-5.0425925, -3.4815094, 1.641246, 3.608149, -...\n", + " [-3.1469467, -2.4765825, 3.0782876, 4.3063807,...\n", " \n", " \n", " 6\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-4.288771, 5.0072904, 0.27649477, -0.797148, ...\n", + " [-2.9109812, 4.020501, 1.359716, -0.003118489,...\n", " \n", " \n", " 7\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-2.2032878, -1.6879876, -5.874276, -0.5945335...\n", + " [-0.3475021, -0.7891858, -4.275926, 0.51047605...\n", " \n", " \n", " 8\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [1.1337761, -3.1751056, -2.5246286, -5.028277,...\n", + " [2.0500505, -2.2071157, -0.07787818, -3.207583...\n", " \n", " \n", " 9\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-0.92484117, -2.4703276, -5.023897, 1.46669, ...\n", + " [-0.0029557291, -1.7499021, -2.1225464, 2.1881...\n", " \n", " \n", "\n", @@ -2330,19 +2467,19 @@ "9 [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ... \n", "\n", " preds \n", - "0 [-5.7614846, -3.52228, -1.1202906, 13.053683, ... \n", - "1 [-3.1390061, -8.71185, 0.82955813, -4.034869, ... \n", - "2 [-3.046528, 0.3521706, 0.6788677, 0.72303534, ... \n", - "3 [-2.401024, -7.6780066, 11.145876, 1.2857256, ... \n", - "4 [-5.0012593, 3.806796, -0.8154834, -0.9550028,... \n", - "5 [-5.0425925, -3.4815094, 1.641246, 3.608149, -... \n", - "6 [-4.288771, 5.0072904, 0.27649477, -0.797148, ... \n", - "7 [-2.2032878, -1.6879876, -5.874276, -0.5945335... \n", - "8 [1.1337761, -3.1751056, -2.5246286, -5.028277,... \n", - "9 [-0.92484117, -2.4703276, -5.023897, 1.46669, ... " + "0 [-2.9825501, -2.4956138, 1.2573452, 12.891572,... \n", + "1 [-0.9547625, -5.622221, 2.2415693, -2.0017662,... \n", + "2 [-0.9178927, 0.4341122, 1.9357599, 1.7975069, ... \n", + "3 [0.2115398, -5.582211, 11.52417, 2.8064039, -3... \n", + "4 [-3.395469, 3.7420897, -0.25319868, 0.16790543... \n", + "5 [-3.1469467, -2.4765825, 3.0782876, 4.3063807,... \n", + "6 [-2.9109812, 4.020501, 1.359716, -0.003118489,... \n", + "7 [-0.3475021, -0.7891858, -4.275926, 0.51047605... \n", + "8 [2.0500505, -2.2071157, -0.07787818, -3.207583... \n", + "9 [-0.0029557291, -1.7499021, -2.1225464, 2.1881... " ] }, - "execution_count": 60, + "execution_count": 63, "metadata": {}, "output_type": "execute_result" } @@ -2355,13 +2492,9 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 64, "id": "79d90a26", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -2370,13 +2503,9 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 65, "id": "4ca495f5", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ "sample = preds.iloc[0]\n", @@ -2388,13 +2517,9 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 66, "id": "a5d10903", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "data": { @@ -2426,14 +2551,17 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 67, "id": "9c9fd967-5cd9-4265-add9-db5c1ccf9893", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + }, { "name": "stderr", "output_type": "stream", @@ -2447,36 +2575,26 @@ "[True]" ] }, - "execution_count": 64, + "execution_count": 67, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "def stop_triton(it):\n", - " import docker\n", - " import time\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " print(\">>>> stopping containers: {}\".format([c.short_id for c in containers]))\n", - " if containers:\n", - " container=containers[0]\n", - " container.stop(timeout=120)\n", - "\n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(stop_triton).collect()" + "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", + "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" ] }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 68, "id": "f612dc0b-538f-4ecf-81f7-ef6b58c493ab", "metadata": {}, "outputs": [], "source": [ - "spark.stop()" + "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n", + " spark.stop()" ] }, { diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras-metadata_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras-metadata_tf.ipynb deleted file mode 100644 index 007f6d8a..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras-metadata_tf.ipynb +++ /dev/null @@ -1,1259 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "8e6810cc-5982-4293-bfbd-c91ef0aca204", - "metadata": {}, - "source": [ - "# Distributed model inference using TensorFlow Keras\n", - "From: https://docs.databricks.com/_static/notebooks/deep-learning/keras-metadata.html" - ] - }, - { - "cell_type": "markdown", - "id": "858e3a8d", - "metadata": {}, - "source": [ - "### Using TensorFlow\n", - "Note that cuFFT/cuDNN/cuBLAS registration errors are expected with `tf=2.17.0` and will not affect behavior, as noted in [this issue.](https://github.com/tensorflow/tensorflow/issues/62075) \n", - "This notebook does not demonstrate inference with TensorRT, as [TF-TRT](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/index.html#tensorrt-10) does not yet support `tf=2.17.0`. See the `pytorch` notebooks for TensorRT demos." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "cf329ac8-0763-44bc-b0f6-b634b7dc480e", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-10-03 17:41:30.112764: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2024-10-03 17:41:30.119504: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2024-10-03 17:41:30.126948: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2024-10-03 17:41:30.129111: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2024-10-03 17:41:30.134946: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", - "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2024-10-03 17:41:30.497048: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" - ] - } - ], - "source": [ - "import os\n", - "import shutil\n", - "import subprocess\n", - "import time\n", - "import pandas as pd\n", - "from PIL import Image\n", - "import numpy as np\n", - "import uuid\n", - " \n", - "import tensorflow as tf\n", - "from tensorflow.keras.applications.resnet50 import ResNet50\n", - " \n", - "from pyspark.sql.functions import col, pandas_udf, PandasUDFType\n", - "from pyspark.sql import SparkSession\n", - "from pyspark import SparkConf" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "44d72768", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "conda_env = os.environ.get(\"CONDA_PREFIX\")\n", - "\n", - "conf = SparkConf()\n", - "if 'spark' not in globals():\n", - " # If Spark is not already started with Jupyter, attach to Spark Standalone\n", - " import socket\n", - " hostname = socket.gethostname()\n", - " conf.setMaster(f\"spark://{hostname}:7077\") # assuming Master is on default port 7077\n", - "conf.set(\"spark.task.maxFailures\", \"1\")\n", - "conf.set(\"spark.driver.memory\", \"8g\")\n", - "conf.set(\"spark.executor.memory\", \"8g\")\n", - "conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled\", \"false\")\n", - "conf.set(\"spark.sql.pyspark.jvmStacktrace.enabled\", \"true\")\n", - "conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", - "conf.set(\"spark.python.worker.reuse\", \"true\")\n", - "# Create Spark Session\n", - "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", - "sc = spark.sparkContext" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "833e36bc", - "metadata": {}, - "outputs": [], - "source": [ - "# Enable GPU memory growth\n", - "gpus = tf.config.experimental.list_physical_devices('GPU')\n", - "if gpus:\n", - " try:\n", - " for gpu in gpus:\n", - " tf.config.experimental.set_memory_growth(gpu, True)\n", - " except RuntimeError as e:\n", - " print(e)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "950b0470-a21e-4778-a80e-b8f6ef792dff", - "metadata": {}, - "outputs": [], - "source": [ - "file_name = \"image_data.parquet\"\n", - "output_file_path = \"predictions\"" - ] - }, - { - "cell_type": "markdown", - "id": "968d08a7-66b9-444f-b362-d8df692aef1c", - "metadata": {}, - "source": [ - "### Prepare trained model and data for inference" - ] - }, - { - "cell_type": "markdown", - "id": "da083168-137f-492c-8769-d8f1e2111756", - "metadata": {}, - "source": [ - "Load the ResNet-50 Model and broadcast the weights." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "2ddc715a-cdbc-4c49-93e9-58c9d88511da", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-10-03 17:41:32.482802: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 45311 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet50_weights_tf_dim_ordering_tf_kernels.h5\n", - "\u001b[1m102967424/102967424\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 0us/step\n" - ] - } - ], - "source": [ - "model = ResNet50()\n", - "bc_model_weights = sc.broadcast(model.get_weights())" - ] - }, - { - "cell_type": "markdown", - "id": "77dddfa3-e8df-4e8e-8251-64457f1ebf80", - "metadata": {}, - "source": [ - "Load the data and save the datasets to one Parquet file." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "c0738bec-97d4-4946-8c49-5e6d07ff1afc", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloading data from https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz\n", - "\u001b[1m228813984/228813984\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 0us/step\n" - ] - } - ], - "source": [ - "import pathlib\n", - "dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz\"\n", - "data_dir = tf.keras.utils.get_file(origin=dataset_url,\n", - " fname='flower_photos',\n", - " untar=True)\n", - "data_dir = pathlib.Path(data_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "014644f4-2a45-4474-8afb-0daf90043253", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3670\n" - ] - } - ], - "source": [ - "image_count = len(list(data_dir.glob('*/*.jpg')))\n", - "print(image_count)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "d54f470a-d308-4426-8ed0-33f95155bb4f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "2048" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import os\n", - "files = [os.path.join(dp, f) for dp, dn, filenames in os.walk(data_dir) for f in filenames if os.path.splitext(f)[1] == '.jpg']\n", - "files = files[:2048]\n", - "len(files)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "fd883dc0-4846-4411-a4d6-4f5f252ac707", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/home/rishic/.keras/datasets/flower_photos\n" - ] - } - ], - "source": [ - "print(data_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "64f94ee0-f1ea-47f6-a77e-be8da5d1b87a", - "metadata": {}, - "outputs": [], - "source": [ - "image_data = []\n", - "for file in files:\n", - " img = Image.open(file)\n", - " img = img.resize([224, 224])\n", - " data = np.asarray(img, dtype=\"float32\").reshape([224*224*3])\n", - "\n", - " image_data.append({\"data\": data})\n", - "\n", - "pandas_df = pd.DataFrame(image_data, columns=['data'])\n", - "pandas_df.to_parquet(file_name)\n", - "# os.makedirs(dbfs_file_path)\n", - "# shutil.copyfile(file_name, dbfs_file_path+file_name)" - ] - }, - { - "cell_type": "markdown", - "id": "f2414b0f-58f2-4e4a-9d09-8ea95b38d413", - "metadata": {}, - "source": [ - "### Save Model\n" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "670328e3-7274-4d78-b315-487750166a3f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "subprocess.call(\"rm -rf resnet50_model\".split())\n", - "model.export(\"resnet50_model\", verbose=0)" - ] - }, - { - "cell_type": "markdown", - "id": "b827ad56-1af0-41b7-be68-94bd203a2a70", - "metadata": {}, - "source": [ - "### Load the data into Spark DataFrames" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "8ddc22d0-b88a-4906-bd47-bf247e34feeb", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2048\n" - ] - } - ], - "source": [ - "from pyspark.sql.types import *\n", - "df = spark.read.parquet(file_name)\n", - "print(df.count())" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "c7adf1d9-1fa7-4456-ae32-cf7d1d43bfd3", - "metadata": {}, - "outputs": [], - "source": [ - "# spark.conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"1024\")\n", - "spark.conf.set(\"spark.sql.parquet.columnarReaderBatchSize\", \"1024\")" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "97173c07-a96e-4262-b60f-82865b997e99", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - } - ], - "source": [ - "assert len(df.head()) > 0, \"`df` should not be empty\" # This line will fail if the vectorized reader runs out of memory" - ] - }, - { - "cell_type": "markdown", - "id": "865929b0-b016-4de4-996d-7f16176cf49c", - "metadata": { - "tags": [] - }, - "source": [ - "### Model inference via pandas UDF" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "a67b3128-13c1-44f1-a0c0-7cf7a836fee3", - "metadata": {}, - "outputs": [], - "source": [ - "def parse_image(image_data):\n", - " image = tf.image.convert_image_dtype(\n", - " image_data, dtype=tf.float32) * (2. / 255) - 1\n", - " image = tf.reshape(image, [224, 224, 3])\n", - " return image" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "7b33185f-6d1e-4ca9-9757-fdc3d736496b", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/rishic/anaconda3/envs/spark-dl-tf/lib/python3.11/site-packages/pyspark/sql/pandas/functions.py:407: UserWarning: In Python 3.6+ and Spark 3.0+, it is preferred to specify type hints for pandas UDF instead of specifying pandas UDF type which will be deprecated in the future releases. See SPARK-28264 for more details.\n", - " warnings.warn(\n" - ] - } - ], - "source": [ - "@pandas_udf(ArrayType(FloatType()), PandasUDFType.SCALAR_ITER)\n", - "def predict_batch_udf(image_batch_iter):\n", - "\n", - " # Enable GPU memory growth to avoid CUDA OOM\n", - " gpus = tf.config.experimental.list_physical_devices('GPU')\n", - " if gpus:\n", - " try:\n", - " for gpu in gpus:\n", - " tf.config.experimental.set_memory_growth(gpu, True)\n", - " except RuntimeError as e:\n", - " print(e)\n", - "\n", - " batch_size = 64\n", - " model = ResNet50(weights=None)\n", - " model.set_weights(bc_model_weights.value)\n", - " for image_batch in image_batch_iter:\n", - " images = np.vstack(image_batch)\n", - " dataset = tf.data.Dataset.from_tensor_slices(images)\n", - " dataset = dataset.map(parse_image, num_parallel_calls=8).prefetch(\n", - " 5000).batch(batch_size)\n", - " preds = model.predict(dataset)\n", - " yield pd.Series(list(preds))" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "ad8c05da-db38-45ef-81d0-1f862f575ced", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| prediction|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "|[1.2938889E-4, 2.4666305E-4, 6.765791E-5, 1.2263245E-4, 5.7486624E-5, 3.9616702E-4, 7.0566134E-6, 4.1722178E-5, 1.225...|\n", - "|[4.4501914E-5, 3.5403698E-4, 4.6702033E-5, 8.102543E-5, 3.1704556E-5, 1.9194305E-4, 7.905952E-6, 1.3744082E-4, 1.9563...|\n", - "|[1.05672516E-4, 2.2686279E-4, 3.0055395E-5, 6.523785E-5, 2.352077E-5, 3.7122983E-4, 3.3315896E-6, 2.2584054E-5, 9.775...|\n", - "|[2.0331638E-5, 2.2746396E-4, 7.828012E-5, 6.986782E-5, 4.705316E-5, 9.80732E-5, 5.561918E-6, 2.3519044E-4, 1.3803913E...|\n", - "|[1.130241E-4, 2.3187004E-4, 5.296914E-5, 1.0871329E-4, 4.027478E-5, 3.7183522E-4, 5.5931855E-6, 3.4792112E-5, 1.14155...|\n", - "|[9.094467E-5, 2.06384E-4, 4.514821E-5, 7.665891E-5, 3.2262324E-5, 3.3875552E-4, 3.831814E-6, 4.1848412E-5, 9.94389E-6...|\n", - "|[1.07847634E-4, 3.7848807E-4, 7.660533E-5, 1.2446754E-4, 4.7595917E-5, 3.333814E-4, 1.0669675E-5, 9.133265E-5, 1.8015...|\n", - "|[2.2261223E-5, 2.734666E-4, 3.8122747E-5, 6.2266954E-5, 1.7935155E-5, 1.7268128E-4, 6.034271E-6, 1.06450585E-4, 1.789...|\n", - "|[1.1065645E-4, 2.900581E-4, 4.2585547E-5, 1.074203E-4, 3.052314E-5, 4.794604E-4, 6.4872897E-6, 3.646897E-5, 1.3717402...|\n", - "|[9.673917E-5, 2.058331E-4, 7.4652424E-5, 1.1323769E-4, 4.6106186E-5, 2.8604185E-4, 5.62365E-6, 5.471466E-5, 9.664386E...|\n", - "|[7.411196E-5, 3.291524E-4, 1.3454164E-4, 1.7738447E-4, 8.467504E-5, 2.2466244E-4, 1.3621126E-5, 1.1778668E-4, 1.83372...|\n", - "|[8.721524E-5, 2.7338538E-4, 3.5964815E-5, 7.792533E-5, 2.3559302E-5, 3.6789547E-4, 3.5665628E-6, 3.648153E-5, 1.07589...|\n", - "|[9.723709E-5, 2.7619812E-4, 5.7464153E-5, 1.10104906E-4, 3.8317143E-5, 3.490506E-4, 6.1553183E-6, 4.413095E-5, 1.1236...|\n", - "|[6.940235E-5, 2.5377885E-4, 5.057188E-5, 1.1485363E-4, 3.0059196E-5, 2.7862669E-4, 5.024019E-6, 5.1511077E-5, 1.16149...|\n", - "|[4.2095784E-5, 2.4891715E-4, 1.236292E-4, 1.4306813E-4, 7.3354306E-5, 1.6047148E-4, 7.958807E-6, 1.3556339E-4, 1.4698...|\n", - "|[2.7327887E-5, 3.8553146E-4, 1.2939748E-4, 1.5762268E-4, 7.307493E-5, 8.5530424E-5, 1.2648808E-5, 1.9154618E-4, 2.307...|\n", - "|[3.036101E-5, 3.5572305E-4, 1.600718E-4, 2.1437313E-4, 8.063033E-5, 1.02061334E-4, 1.3876456E-5, 1.561292E-4, 1.63637...|\n", - "|[3.3109587E-5, 2.8182982E-4, 1.7998899E-4, 2.0246049E-4, 1.3720036E-4, 1.01000114E-4, 3.427488E-5, 3.887249E-4, 3.189...|\n", - "|[4.549448E-5, 2.8782588E-4, 2.3703449E-4, 2.448979E-4, 1.20997625E-4, 1.3744453E-4, 1.62803E-5, 2.2094708E-4, 1.56962...|\n", - "|[1.2242574E-4, 2.8095162E-4, 6.332559E-5, 1.0209269E-4, 4.335324E-5, 3.906304E-4, 8.205706E-6, 6.202823E-5, 1.5312888...|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "only showing top 20 rows\n", - "\n", - "CPU times: user 10.8 ms, sys: 4.93 ms, total: 15.7 ms\n", - "Wall time: 9.25 s\n" - ] - } - ], - "source": [ - "%%time\n", - "predictions_df = df.select(predict_batch_udf(col(\"data\")).alias(\"prediction\"))\n", - "predictions_df.show(truncate=120)" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "40799f8e-443e-40ca-919b-391f901cb3f4", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 8:===================================================> (7 + 1) / 8]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 9.96 ms, sys: 3.32 ms, total: 13.3 ms\n", - "Wall time: 14 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - } - ], - "source": [ - "%%time\n", - "predictions_df.write.mode(\"overwrite\").parquet(output_file_path)" - ] - }, - { - "cell_type": "markdown", - "id": "16726357-65d8-4d3d-aea1-6800101741cc", - "metadata": { - "tags": [] - }, - "source": [ - "### Model inference using Spark DL API" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "e6af27b2-ddc0-42ee-94cc-9ba5ffee6868", - "metadata": {}, - "outputs": [], - "source": [ - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import struct, col\n", - "from pyspark.sql.types import ArrayType, FloatType" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "dda88b46-6300-4bf7-bc10-7403f4fbbf92", - "metadata": {}, - "outputs": [], - "source": [ - "def predict_batch_fn():\n", - " import tensorflow as tf\n", - " from tensorflow.keras.applications.resnet50 import ResNet50\n", - "\n", - " # Enable GPU memory growth\n", - " gpus = tf.config.experimental.list_physical_devices('GPU')\n", - " if gpus:\n", - " try:\n", - " for gpu in gpus:\n", - " tf.config.experimental.set_memory_growth(gpu, True)\n", - " except RuntimeError as e:\n", - " print(e)\n", - "\n", - " model = ResNet50()\n", - " def predict(inputs):\n", - " inputs = inputs * (2. / 255) - 1\n", - " return model.predict(inputs)\n", - " return predict" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "cff0e851-563d-40b6-9d05-509c22b3b7f9", - "metadata": {}, - "outputs": [], - "source": [ - "classify = predict_batch_udf(predict_batch_fn,\n", - " input_tensor_shapes=[[224, 224, 3]],\n", - " return_type=ArrayType(FloatType()),\n", - " batch_size=50)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "f733c38b-867d-48c1-b9a6-74a931561896", - "metadata": {}, - "outputs": [], - "source": [ - "# spark.conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"1024\")\n", - "spark.conf.set(\"spark.sql.parquet.columnarReaderBatchSize\", \"1024\")" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "aa7c156f-e2b3-4837-9427-ccf3a5720412", - "metadata": {}, - "outputs": [], - "source": [ - "df = spark.read.parquet(\"image_data.parquet\")" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "80bc50ad-eaf5-4fce-a354-5e17d65e2da5", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 11:===========================================> (3 + 1) / 4]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| prediction|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "|[1.296447E-4, 2.465122E-4, 6.7463385E-5, 1.2231144E-4, 5.731739E-5, 3.9644213E-4, 7.0297688E-6, 4.1668914E-5, 1.22212...|\n", - "|[4.4481887E-5, 3.526653E-4, 4.683818E-5, 8.1168495E-5, 3.178377E-5, 1.9188467E-4, 7.885617E-6, 1.3758946E-4, 1.956621...|\n", - "|[1.05946536E-4, 2.2744355E-4, 3.0219735E-5, 6.548672E-5, 2.3649674E-5, 3.7177472E-4, 3.353236E-6, 2.271976E-5, 9.8115...|\n", - "|[2.0392703E-5, 2.2817637E-4, 7.840744E-5, 6.9875685E-5, 4.702542E-5, 9.8244425E-5, 5.5829764E-6, 2.3530141E-4, 1.3836...|\n", - "|[1.1312391E-4, 2.31244E-4, 5.279228E-5, 1.0859927E-4, 4.0202678E-5, 3.721753E-4, 5.563934E-6, 3.4674114E-5, 1.1389492...|\n", - "|[9.126345E-5, 2.0679034E-4, 4.5165678E-5, 7.679106E-5, 3.234611E-5, 3.3994843E-4, 3.84E-6, 4.1930372E-5, 9.949454E-6,...|\n", - "|[1.07930486E-4, 3.7741542E-4, 7.613175E-5, 1.2414041E-4, 4.7409427E-5, 3.332554E-4, 1.05853915E-5, 9.1026224E-5, 1.79...|\n", - "|[2.2216762E-5, 2.7354853E-4, 3.8192928E-5, 6.2340725E-5, 1.7952003E-5, 1.7253387E-4, 6.020507E-6, 1.0669143E-4, 1.786...|\n", - "|[1.10480236E-4, 2.89734E-4, 4.239379E-5, 1.0727814E-4, 3.047985E-5, 4.7992737E-4, 6.4530495E-6, 3.6428817E-5, 1.36967...|\n", - "|[9.6864875E-5, 2.0573521E-4, 7.4498465E-5, 1.1323085E-4, 4.6088306E-5, 2.8680824E-4, 5.604823E-6, 5.461046E-5, 9.6629...|\n", - "|[7.4198484E-5, 3.2886668E-4, 1.3441108E-4, 1.7755068E-4, 8.469927E-5, 2.2534095E-4, 1.3617541E-5, 1.1781904E-4, 1.833...|\n", - "|[8.7561886E-5, 2.7312653E-4, 3.5959012E-5, 7.7946424E-5, 2.3565723E-5, 3.6881721E-4, 3.5630535E-6, 3.642736E-5, 1.074...|\n", - "|[9.743975E-5, 2.7615853E-4, 5.74148E-5, 1.10329434E-4, 3.83045E-5, 3.500394E-4, 6.167429E-6, 4.4207005E-5, 1.1250093E...|\n", - "|[6.9320704E-5, 2.53287E-4, 5.0612853E-5, 1.14936556E-4, 3.0210098E-5, 2.7870742E-4, 5.031114E-6, 5.169024E-5, 1.16021...|\n", - "|[4.2203726E-5, 2.4911022E-4, 1.2378568E-4, 1.4274308E-4, 7.32259E-5, 1.6058519E-4, 7.9425035E-6, 1.3519496E-4, 1.4662...|\n", - "|[2.7190901E-5, 3.8381666E-4, 1.2918573E-4, 1.570463E-4, 7.310112E-5, 8.554618E-5, 1.2614603E-5, 1.9213595E-4, 2.30354...|\n", - "|[3.0573912E-5, 3.5561546E-4, 1.5945674E-4, 2.1361349E-4, 8.046549E-5, 1.0269262E-4, 1.3862439E-5, 1.5622783E-4, 1.638...|\n", - "|[3.3117096E-5, 2.8073433E-4, 1.7961214E-4, 2.020287E-4, 1.3662946E-4, 1.0117796E-4, 3.4090703E-5, 3.8897162E-4, 3.181...|\n", - "|[4.5728237E-5, 2.8880237E-4, 2.3783019E-4, 2.4589908E-4, 1.2160292E-4, 1.3812551E-4, 1.6343482E-5, 2.2073709E-4, 1.57...|\n", - "|[1.2280059E-4, 2.806991E-4, 6.3642765E-5, 1.02471764E-4, 4.351664E-5, 3.9150563E-4, 8.235125E-6, 6.211928E-5, 1.53269...|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "only showing top 20 rows\n", - "\n", - "CPU times: user 8.12 ms, sys: 3.38 ms, total: 11.5 ms\n", - "Wall time: 5.59 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - } - ], - "source": [ - "%%time\n", - "# first pass caches model/fn\n", - "predictions = df.select(classify(struct(\"data\")).alias(\"prediction\"))\n", - "predictions.show(truncate=120)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "41cace80-7a4b-4929-8e63-9c83f9745e02", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 13:===========================================> (3 + 1) / 4]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| prediction|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "|[1.293178E-4, 2.4644283E-4, 6.760039E-5, 1.2260793E-4, 5.7431564E-5, 3.9597694E-4, 7.0522524E-6, 4.1717416E-5, 1.2240...|\n", - "|[4.4487308E-5, 3.5378174E-4, 4.6667028E-5, 8.102564E-5, 3.168566E-5, 1.9189132E-4, 7.903805E-6, 1.3741471E-4, 1.95482...|\n", - "|[1.0566196E-4, 2.2684377E-4, 3.00564E-5, 6.5251304E-5, 2.3520754E-5, 3.7116173E-4, 3.331476E-6, 2.2584616E-5, 9.77515...|\n", - "|[2.0337258E-5, 2.2749524E-4, 7.8351426E-5, 6.991163E-5, 4.7081656E-5, 9.8092445E-5, 5.564894E-6, 2.3517481E-4, 1.3805...|\n", - "|[1.12979564E-4, 2.3172122E-4, 5.2946547E-5, 1.0876398E-4, 4.0259067E-5, 3.7143996E-4, 5.5940513E-6, 3.4814777E-5, 1.1...|\n", - "|[9.093228E-5, 2.0639994E-4, 4.5151268E-5, 7.666316E-5, 3.2264295E-5, 3.387436E-4, 3.832487E-6, 4.185193E-5, 9.944773E...|\n", - "|[1.0783461E-4, 3.7850672E-4, 7.660902E-5, 1.2446321E-4, 4.7591406E-5, 3.3328883E-4, 1.067249E-5, 9.131178E-5, 1.80121...|\n", - "|[2.2258617E-5, 2.7345872E-4, 3.814439E-5, 6.229726E-5, 1.79387E-5, 1.7259057E-4, 6.0371217E-6, 1.0649798E-4, 1.789726...|\n", - "|[1.1067773E-4, 2.8997674E-4, 4.2570035E-5, 1.0747747E-4, 3.0524247E-5, 4.7921995E-4, 6.489833E-6, 3.6502548E-5, 1.371...|\n", - "|[9.676251E-5, 2.0588847E-4, 7.467098E-5, 1.1326933E-4, 4.6123736E-5, 2.8609246E-4, 5.627118E-6, 5.4726373E-5, 9.66839...|\n", - "|[7.4104944E-5, 3.290917E-4, 1.3448784E-4, 1.7742367E-4, 8.463227E-5, 2.2462371E-4, 1.3614881E-5, 1.17794625E-4, 1.833...|\n", - "|[8.7211796E-5, 2.7337394E-4, 3.5953894E-5, 7.7924225E-5, 2.3554327E-5, 3.67775E-4, 3.5652213E-6, 3.647724E-5, 1.07577...|\n", - "|[9.7237185E-5, 2.762026E-4, 5.7450008E-5, 1.1019135E-4, 3.831896E-5, 3.4878452E-4, 6.1574788E-6, 4.415526E-5, 1.12374...|\n", - "|[6.938849E-5, 2.5376282E-4, 5.0565883E-5, 1.14880335E-4, 3.0061366E-5, 2.7866007E-4, 5.024482E-6, 5.152425E-5, 1.1617...|\n", - "|[4.2096388E-5, 2.4889092E-4, 1.2363133E-4, 1.4304162E-4, 7.337785E-5, 1.6042824E-4, 7.959722E-6, 1.3552785E-4, 1.4693...|\n", - "|[2.730248E-5, 3.851789E-4, 1.293143E-4, 1.5753493E-4, 7.302161E-5, 8.547956E-5, 1.26348905E-5, 1.9148648E-4, 2.304900...|\n", - "|[3.0354899E-5, 3.5562844E-4, 1.6008675E-4, 2.1440513E-4, 8.062159E-5, 1.02023136E-4, 1.3876455E-5, 1.5611007E-4, 1.63...|\n", - "|[3.3083066E-5, 2.8158593E-4, 1.7979987E-4, 2.0232225E-4, 1.3704685E-4, 1.0091762E-4, 3.4243407E-5, 3.8870922E-4, 3.18...|\n", - "|[4.5485373E-5, 2.878148E-4, 2.3707838E-4, 2.4493985E-4, 1.21028905E-4, 1.3738636E-4, 1.6280053E-5, 2.2104722E-4, 1.56...|\n", - "|[1.22468E-4, 2.809503E-4, 6.3342835E-5, 1.021957E-4, 4.3373006E-5, 3.905496E-4, 8.212427E-6, 6.2081075E-5, 1.5323925E...|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "only showing top 20 rows\n", - "\n", - "CPU times: user 2.75 ms, sys: 3.03 ms, total: 5.78 ms\n", - "Wall time: 4.79 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - } - ], - "source": [ - "%%time\n", - "predictions = df.select(classify(\"data\").alias(\"prediction\"))\n", - "predictions.show(truncate=120)" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "56a2ec8a-de09-4d7c-9666-1b3c76f10657", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 14:==================================================> (7 + 1) / 8]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 10.7 ms, sys: 4.25 ms, total: 14.9 ms\n", - "Wall time: 16.9 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - } - ], - "source": [ - "%%time\n", - "predictions = df.select(classify(col(\"data\")).alias(\"prediction\"))\n", - "predictions.write.mode(\"overwrite\").parquet(output_file_path + \"_1\")" - ] - }, - { - "cell_type": "markdown", - "id": "0efa48e9-2eda-4d57-8174-8850e5bca4af", - "metadata": {}, - "source": [ - "### Using Triton Inference Server\n", - "\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment." - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "2605d134-ef75-4d94-9b16-2c6d85f29bef", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "import os\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import col, struct\n", - "from pyspark.sql.types import ArrayType, FloatType" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "id": "4666e618-8038-4dc5-9be7-793aedbf4500", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "%%bash\n", - "# copy model to expected layout for Triton\n", - "rm -rf models\n", - "mkdir -p models/resnet50/1\n", - "cp -r resnet50_model models/resnet50/1/model.savedmodel\n", - "\n", - "# add config.pbtxt\n", - "cp models_config/resnet50/config.pbtxt models/resnet50/config.pbtxt" - ] - }, - { - "cell_type": "markdown", - "id": "e07f1d6d-334e-4f85-9472-171dda09bae4", - "metadata": {}, - "source": [ - "#### Start Triton Server on each executor" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "id": "8c8c0744-0558-4dac-bbfe-8bdde4b2af2d", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "data": { - "text/plain": [ - "[True]" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "num_executors = 1\n", - "triton_models_dir = \"{}/models\".format(os.getcwd())\n", - "nodeRDD = sc.parallelize(list(range(num_executors)), num_executors)\n", - "\n", - "def start_triton(it):\n", - " import docker\n", - " import time\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " if containers:\n", - " print(\">>>> containers: {}\".format([c.short_id for c in containers]))\n", - " else:\n", - " container=client.containers.run(\n", - " \"nvcr.io/nvidia/tritonserver:24.08-py3\", \"tritonserver --model-repository=/models\",\n", - " detach=True,\n", - " device_requests=[docker.types.DeviceRequest(device_ids=[\"0\"], capabilities=[['gpu']])],\n", - " name=\"spark-triton\",\n", - " network_mode=\"host\",\n", - " remove=True,\n", - " shm_size=\"512M\",\n", - " volumes={triton_models_dir: {\"bind\": \"/models\", \"mode\": \"ro\"}}\n", - " )\n", - " print(\">>>> starting triton: {}\".format(container.short_id))\n", - "\n", - " # wait for triton to be running\n", - " time.sleep(15)\n", - " client = grpcclient.InferenceServerClient(\"localhost:8001\")\n", - " ready = False\n", - " while not ready:\n", - " try:\n", - " ready = client.is_server_ready()\n", - " except Exception as e:\n", - " time.sleep(5)\n", - "\n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(start_triton).collect()" - ] - }, - { - "cell_type": "markdown", - "id": "8c07365c-0a14-49b3-9bd8-cfb35f48b089", - "metadata": {}, - "source": [ - "#### Run inference" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "id": "bcd46360-6851-4a9d-8590-c086e001242a", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "def triton_fn(triton_uri, model_name):\n", - " import numpy as np\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " np_types = {\n", - " \"BOOL\": np.dtype(np.bool_),\n", - " \"INT8\": np.dtype(np.int8),\n", - " \"INT16\": np.dtype(np.int16),\n", - " \"INT32\": np.dtype(np.int32),\n", - " \"INT64\": np.dtype(np.int64),\n", - " \"FP16\": np.dtype(np.float16),\n", - " \"FP32\": np.dtype(np.float32),\n", - " \"FP64\": np.dtype(np.float64),\n", - " \"FP64\": np.dtype(np.double),\n", - " \"BYTES\": np.dtype(object)\n", - " }\n", - "\n", - " client = grpcclient.InferenceServerClient(triton_uri)\n", - " model_meta = client.get_model_metadata(model_name)\n", - " \n", - " def predict(inputs):\n", - " if isinstance(inputs, np.ndarray):\n", - " # single ndarray input\n", - " inputs = inputs * (2. / 255) - 1 # add normalization\n", - " request = [grpcclient.InferInput(model_meta.inputs[0].name, inputs.shape, model_meta.inputs[0].datatype)]\n", - " request[0].set_data_from_numpy(inputs.astype(np_types[model_meta.inputs[0].datatype]))\n", - " else:\n", - " # dict of multiple ndarray inputs\n", - " request = [grpcclient.InferInput(i.name, inputs[i.name].shape, i.datatype) for i in model_meta.inputs]\n", - " for i in request:\n", - " i.set_data_from_numpy(inputs[i.name()].astype(np_types[i.datatype()]))\n", - " \n", - " response = client.infer(model_name, inputs=request)\n", - " \n", - " if len(model_meta.outputs) > 1:\n", - " # return dictionary of numpy arrays\n", - " return {o.name: response.as_numpy(o.name) for o in model_meta.outputs}\n", - " else:\n", - " # return single numpy array\n", - " return response.as_numpy(model_meta.outputs[0].name)\n", - " \n", - " return predict" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "id": "9fabcaeb-5a44-42bb-8097-5dbc2d0cee3e", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "from functools import partial\n", - "\n", - "classify = predict_batch_udf(partial(triton_fn, triton_uri=\"localhost:8001\", model_name=\"resnet50\"),\n", - " input_tensor_shapes=[[224, 224, 3]],\n", - " return_type=ArrayType(FloatType()),\n", - " batch_size=50)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "id": "b17f33c8-a0f0-4bce-91f8-5838ba9b12a7", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "# spark.conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"1024\")\n", - "spark.conf.set(\"spark.sql.parquet.columnarReaderBatchSize\", \"1024\")" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "id": "8e5b9e99-a1cf-43d3-a795-c7271a917057", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "df = spark.read.parquet(\"image_data.parquet\")" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "id": "e595473d-1a5d-46a6-a6ba-89d2ea903de9", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 18:===========================================> (3 + 1) / 4]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| prediction|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "|[1.2838157E-4, 2.442499E-4, 6.756602E-5, 1.223822E-4, 5.718728E-5, 3.9370774E-4, 6.9826538E-6, 4.180329E-5, 1.21474E-...|\n", - "|[4.3975022E-5, 3.5182733E-4, 4.6756446E-5, 8.051952E-5, 3.157192E-5, 1.8915786E-4, 7.8848925E-6, 1.3820908E-4, 1.9617...|\n", - "|[1.0483801E-4, 2.2482511E-4, 2.9800098E-5, 6.471683E-5, 2.3306355E-5, 3.6853546E-4, 3.2802545E-6, 2.2436941E-5, 9.655...|\n", - "|[2.0184121E-5, 2.2646098E-4, 7.754879E-5, 6.9126E-5, 4.6796213E-5, 9.757494E-5, 5.5280707E-6, 2.3486002E-4, 1.3758638...|\n", - "|[1.1207414E-4, 2.3036542E-4, 5.2748997E-5, 1.0843094E-4, 3.9970357E-5, 3.692824E-4, 5.5317682E-6, 3.467135E-5, 1.1321...|\n", - "|[9.028466E-5, 2.0533502E-4, 4.5085282E-5, 7.65107E-5, 3.217092E-5, 3.3741904E-4, 3.8024857E-6, 4.1927728E-5, 9.920564...|\n", - "|[1.0625615E-4, 3.759827E-4, 7.6174496E-5, 1.2342798E-4, 4.7335903E-5, 3.3091815E-4, 1.0598523E-5, 9.161089E-5, 1.7926...|\n", - "|[2.2157477E-5, 2.726377E-4, 3.831429E-5, 6.2276886E-5, 1.8050652E-5, 1.7177712E-4, 6.0331595E-6, 1.06755506E-4, 1.790...|\n", - "|[1.0993216E-4, 2.8824335E-4, 4.2543048E-5, 1.06903855E-4, 3.039875E-5, 4.7743318E-4, 6.441006E-6, 3.6423717E-5, 1.361...|\n", - "|[9.6276366E-5, 2.047977E-4, 7.4698546E-5, 1.128771E-4, 4.6044628E-5, 2.8445767E-4, 5.6014956E-6, 5.475251E-5, 9.63856...|\n", - "|[7.3160336E-5, 3.2700456E-4, 1.3447899E-4, 1.7689951E-4, 8.4440886E-5, 2.2350134E-4, 1.3515168E-5, 1.1746432E-4, 1.81...|\n", - "|[8.632592E-5, 2.7143923E-4, 3.583003E-5, 7.763873E-5, 2.3417528E-5, 3.6477615E-4, 3.527159E-6, 3.646688E-5, 1.0721673...|\n", - "|[9.640316E-5, 2.7391897E-4, 5.7131063E-5, 1.09568326E-4, 3.8045353E-5, 3.472495E-4, 6.057242E-6, 4.3799748E-5, 1.1118...|\n", - "|[6.912533E-5, 2.5222785E-4, 5.0288483E-5, 1.1415517E-4, 2.9881658E-5, 2.7816373E-4, 4.972507E-6, 5.121496E-5, 1.15293...|\n", - "|[4.189945E-5, 2.4779947E-4, 1.2303083E-4, 1.4200866E-4, 7.2787174E-5, 1.600041E-4, 7.901948E-6, 1.3503798E-4, 1.46427...|\n", - "|[2.7033573E-5, 3.8410365E-4, 1.2880778E-4, 1.5630701E-4, 7.2431474E-5, 8.455686E-5, 1.2551222E-5, 1.9146077E-4, 2.293...|\n", - "|[2.9902518E-5, 3.521676E-4, 1.6034822E-4, 2.1348803E-4, 8.053424E-5, 1.00774814E-4, 1.3777179E-5, 1.5595586E-4, 1.615...|\n", - "|[3.2834323E-5, 2.8044736E-4, 1.8003663E-4, 2.017913E-4, 1.3718085E-4, 1.0062256E-4, 3.4619785E-5, 3.8973117E-4, 3.187...|\n", - "|[4.4552748E-5, 2.8623734E-4, 2.3419394E-4, 2.4108509E-4, 1.1926766E-4, 1.3529808E-4, 1.6018543E-5, 2.210266E-4, 1.558...|\n", - "|[1.2160183E-4, 2.8021698E-4, 6.289166E-5, 1.0147789E-4, 4.3161614E-5, 3.8964444E-4, 8.174407E-6, 6.2043844E-5, 1.5228...|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "only showing top 20 rows\n", - "\n", - "CPU times: user 4.79 ms, sys: 1.93 ms, total: 6.72 ms\n", - "Wall time: 3.06 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - } - ], - "source": [ - "%%time\n", - "# first pass caches model/fn\n", - "predictions = df.select(classify(struct(\"data\")).alias(\"prediction\"))\n", - "predictions.show(truncate=120)" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "id": "5f66d468-e0b1-4589-8606-b3848063a823", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 20:===========================================> (3 + 1) / 4]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| prediction|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "|[1.2838157E-4, 2.442499E-4, 6.756602E-5, 1.223822E-4, 5.718728E-5, 3.9370774E-4, 6.9826538E-6, 4.180329E-5, 1.21474E-...|\n", - "|[4.3975022E-5, 3.5182733E-4, 4.6756446E-5, 8.051952E-5, 3.157192E-5, 1.8915786E-4, 7.8848925E-6, 1.3820908E-4, 1.9617...|\n", - "|[1.0483801E-4, 2.2482511E-4, 2.9800098E-5, 6.471683E-5, 2.3306355E-5, 3.6853546E-4, 3.2802545E-6, 2.2436941E-5, 9.655...|\n", - "|[2.0184121E-5, 2.2646098E-4, 7.754879E-5, 6.9126E-5, 4.6796213E-5, 9.757494E-5, 5.5280707E-6, 2.3486002E-4, 1.3758638...|\n", - "|[1.1207414E-4, 2.3036542E-4, 5.2748997E-5, 1.0843094E-4, 3.9970357E-5, 3.692824E-4, 5.5317682E-6, 3.467135E-5, 1.1321...|\n", - "|[9.028466E-5, 2.0533502E-4, 4.5085282E-5, 7.65107E-5, 3.217092E-5, 3.3741904E-4, 3.8024857E-6, 4.1927728E-5, 9.920564...|\n", - "|[1.0625615E-4, 3.759827E-4, 7.6174496E-5, 1.2342798E-4, 4.7335903E-5, 3.3091815E-4, 1.0598523E-5, 9.161089E-5, 1.7926...|\n", - "|[2.2157477E-5, 2.726377E-4, 3.831429E-5, 6.2276886E-5, 1.8050652E-5, 1.7177712E-4, 6.0331595E-6, 1.06755506E-4, 1.790...|\n", - "|[1.0993216E-4, 2.8824335E-4, 4.2543048E-5, 1.06903855E-4, 3.039875E-5, 4.7743318E-4, 6.441006E-6, 3.6423717E-5, 1.361...|\n", - "|[9.6276366E-5, 2.047977E-4, 7.4698546E-5, 1.128771E-4, 4.6044628E-5, 2.8445767E-4, 5.6014956E-6, 5.475251E-5, 9.63856...|\n", - "|[7.3160336E-5, 3.2700456E-4, 1.3447899E-4, 1.7689951E-4, 8.4440886E-5, 2.2350134E-4, 1.3515168E-5, 1.1746432E-4, 1.81...|\n", - "|[8.632592E-5, 2.7143923E-4, 3.583003E-5, 7.763873E-5, 2.3417528E-5, 3.6477615E-4, 3.527159E-6, 3.646688E-5, 1.0721673...|\n", - "|[9.640316E-5, 2.7391897E-4, 5.7131063E-5, 1.09568326E-4, 3.8045353E-5, 3.472495E-4, 6.057242E-6, 4.3799748E-5, 1.1118...|\n", - "|[6.912533E-5, 2.5222785E-4, 5.0288483E-5, 1.1415517E-4, 2.9881658E-5, 2.7816373E-4, 4.972507E-6, 5.121496E-5, 1.15293...|\n", - "|[4.189945E-5, 2.4779947E-4, 1.2303083E-4, 1.4200866E-4, 7.2787174E-5, 1.600041E-4, 7.901948E-6, 1.3503798E-4, 1.46427...|\n", - "|[2.7033573E-5, 3.8410365E-4, 1.2880778E-4, 1.5630701E-4, 7.2431474E-5, 8.455686E-5, 1.2551222E-5, 1.9146077E-4, 2.293...|\n", - "|[2.9902518E-5, 3.521676E-4, 1.6034822E-4, 2.1348803E-4, 8.053424E-5, 1.00774814E-4, 1.3777179E-5, 1.5595586E-4, 1.615...|\n", - "|[3.2834323E-5, 2.8044736E-4, 1.8003663E-4, 2.017913E-4, 1.3718085E-4, 1.0062256E-4, 3.4619785E-5, 3.8973117E-4, 3.187...|\n", - "|[4.4552748E-5, 2.8623734E-4, 2.3419394E-4, 2.4108509E-4, 1.1926766E-4, 1.3529808E-4, 1.6018543E-5, 2.210266E-4, 1.558...|\n", - "|[1.2160183E-4, 2.8021698E-4, 6.289166E-5, 1.0147789E-4, 4.3161614E-5, 3.8964444E-4, 8.174407E-6, 6.2043844E-5, 1.5228...|\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "only showing top 20 rows\n", - "\n", - "CPU times: user 3.16 ms, sys: 3.36 ms, total: 6.52 ms\n", - "Wall time: 2.24 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - } - ], - "source": [ - "%%time\n", - "predictions = df.select(classify(\"data\").alias(\"prediction\"))\n", - "predictions.show(truncate=120)" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "id": "632c4c3a-fa52-4c3d-b71e-7526286e353a", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 21:==================================================> (7 + 1) / 8]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 7.57 ms, sys: 5.2 ms, total: 12.8 ms\n", - "Wall time: 13.3 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - } - ], - "source": [ - "%%time\n", - "predictions = df.select(classify(col(\"data\")).alias(\"prediction\"))\n", - "predictions.write.mode(\"overwrite\").parquet(output_file_path + \"_2\")" - ] - }, - { - "cell_type": "markdown", - "id": "4dc06b7e-f750-40b5-9208-a035db11d937", - "metadata": { - "tags": [] - }, - "source": [ - "#### Stop Triton Server on each executor" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "id": "bbfcaa51-3b9f-43ff-a4a8-4b46766115b8", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "data": { - "text/plain": [ - "[True]" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def stop_triton(it):\n", - " import docker\n", - " import time\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " print(\">>>> stopping containers: {}\".format([c.short_id for c in containers]))\n", - " if containers:\n", - " container=containers[0]\n", - " container.stop(timeout=120)\n", - "\n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(stop_triton).collect()" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "id": "0d88639b-d934-4eb4-ae2f-cc13b9b10456", - "metadata": {}, - "outputs": [], - "source": [ - "spark.stop()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "df8cc28a-34d7-479c-be7e-9a380d39e25e", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "spark-dl-tf", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/feature_columns_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_preprocessing_tf.ipynb similarity index 56% rename from examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/feature_columns_tf.ipynb rename to examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_preprocessing_tf.ipynb index 2ff37b6c..a5edd2c6 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/feature_columns_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_preprocessing_tf.ipynb @@ -5,9 +5,13 @@ "id": "7fcc021a", "metadata": {}, "source": [ + "\n", + "\n", "# Pyspark TensorFlow Inference\n", "\n", - "## Feature Columns\n", + "### Classification using Keras Preprocessing Layers\n", + "\n", + "In this notebook, we demonstrate distributed inference using Keras preprocessing layers to classify structured data. \n", "From: https://www.tensorflow.org/tutorials/structured_data/preprocessing_layers" ] }, @@ -16,9 +20,7 @@ "id": "35203476", "metadata": {}, "source": [ - "### Using TensorFlow\n", - "Note that cuFFT/cuDNN/cuBLAS registration errors are expected with `tf=2.17.0` and will not affect behavior, as noted in [this issue.](https://github.com/tensorflow/tensorflow/issues/62075) \n", - "This notebook does not demonstrate inference with TensorRT, as [TF-TRT](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/index.html#tensorrt-10) does not yet support `tf=2.17.0`. See the `pytorch` notebooks for TensorRT demos." + "Note that cuFFT/cuDNN/cuBLAS registration errors are expected (as of `tf=2.17.0`) and will not affect behavior, as noted in [this issue.](https://github.com/tensorflow/tensorflow/issues/62075) " ] }, { @@ -31,17 +33,19 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-24 16:04:17.711230: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2024-10-24 16:04:17.719701: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2024-10-24 16:04:17.728758: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2024-10-24 16:04:17.731459: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2024-10-24 16:04:17.738797: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "2025-01-27 12:15:59.479063: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2025-01-27 12:15:59.486695: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2025-01-27 12:15:59.495438: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2025-01-27 12:15:59.498089: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2025-01-27 12:15:59.505021: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2024-10-24 16:04:18.115892: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" + "2025-01-27 12:15:59.913410: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" ] } ], "source": [ + "import os\n", + "import shutil\n", "import numpy as np\n", "import pandas as pd\n", "import tensorflow as tf\n", @@ -52,6 +56,16 @@ { "cell_type": "code", "execution_count": 2, + "id": "0d586fb8", + "metadata": {}, + "outputs": [], + "source": [ + "os.mkdir('models') if not os.path.exists('models') else None" + ] + }, + { + "cell_type": "code", + "execution_count": 3, "id": "9fa3e1b7-58cd-45f9-9fee-85f25a31c3c6", "metadata": {}, "outputs": [ @@ -61,6 +75,16 @@ "text": [ "2.17.0\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", + "I0000 00:00:1738008960.285379 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008960.309064 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008960.311808 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" + ] } ], "source": [ @@ -76,12 +100,31 @@ " print(e)" ] }, + { + "cell_type": "markdown", + "id": "b2402b9a", + "metadata": {}, + "source": [ + "#### Download dataset\n", + "\n", + "Download the PetFinder dataset from Kaggle, which where each row describes a pet and the goal is to predict adoption speed." + ] + }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 4, "id": "9326b072-a53c-40c4-a6cb-bd4d3d644d03", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading data from http://storage.googleapis.com/download.tensorflow.org/data/petfinder-mini.zip\n", + "\u001b[1m1668792/1668792\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 0us/step\n" + ] + } + ], "source": [ "import pathlib\n", "import os\n", @@ -100,7 +143,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 5, "id": "e98480ef-d13d-44c0-a227-e9a22f9bf2b0", "metadata": {}, "outputs": [ @@ -260,7 +303,7 @@ "4 This handsome yet cute boy is up for adoption.... 3 2 " ] }, - "execution_count": 11, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -269,6 +312,14 @@ "dataframe.head()" ] }, + { + "cell_type": "markdown", + "id": "27d844f1", + "metadata": {}, + "source": [ + "### Prepare dataset" + ] + }, { "cell_type": "code", "execution_count": 6, @@ -325,6 +376,14 @@ "print(len(test), 'test examples')" ] }, + { + "cell_type": "markdown", + "id": "a7fa64f8", + "metadata": {}, + "source": [ + "Create an input pipeline which converts each dataset into a tf.data.Dataset with shuffling and batching." + ] + }, { "cell_type": "code", "execution_count": 9, @@ -344,6 +403,14 @@ " return ds" ] }, + { + "cell_type": "markdown", + "id": "96065bed", + "metadata": {}, + "source": [ + "Check the format of the data returned by the pipeline:" + ] + }, { "cell_type": "code", "execution_count": 10, @@ -354,7 +421,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-03 17:38:53.526119: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 46022 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" + "I0000 00:00:1738008960.772797 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008960.775983 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008960.778707 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008960.893233 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008960.894355 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738008960.895274 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2025-01-27 12:16:00.896282: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 40284 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" ] } ], @@ -363,6 +436,14 @@ "train_ds = df_to_dataset(train, batch_size=batch_size)" ] }, + { + "cell_type": "markdown", + "id": "bdc8571c", + "metadata": {}, + "source": [ + "(Note that OUT_OF_RANGE errors are safe to ignore: https://github.com/tensorflow/tensorflow/issues/62963)." + ] + }, { "cell_type": "code", "execution_count": 11, @@ -375,11 +456,11 @@ "text": [ "Every feature: ['Type', 'Age', 'Breed1', 'Gender', 'Color1', 'Color2', 'MaturitySize', 'FurLength', 'Vaccinated', 'Sterilized', 'Health', 'Fee', 'PhotoAmt', 'target']\n", "A batch of ages: tf.Tensor(\n", - "[[18]\n", - " [ 5]\n", + "[[12]\n", " [ 2]\n", - " [ 5]\n", - " [ 1]], shape=(5, 1), dtype=int64)\n", + " [12]\n", + " [60]\n", + " [11]], shape=(5, 1), dtype=int64)\n", "A batch of targets: tf.Tensor([1 0 1 1 1], shape=(5,), dtype=int64)\n" ] }, @@ -387,7 +468,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-03 17:38:53.588272: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + "2025-01-27 12:16:00.969048: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" ] } ], @@ -398,6 +479,16 @@ "print('A batch of targets:', label_batch )" ] }, + { + "cell_type": "markdown", + "id": "d5a2d10c", + "metadata": {}, + "source": [ + "### Apply Keras preprocessing layers\n", + "\n", + "We'll define a normalization layer for numeric features, and a category encoding for categorical features." + ] + }, { "cell_type": "code", "execution_count": 12, @@ -428,18 +519,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-03 17:38:55.015073: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + "2025-01-27 12:16:02.497311: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" ] }, { "data": { "text/plain": [ "" + "array([[-0.5115027 ],\n", + " [ 0.7461632 ],\n", + " [-0.19708623],\n", + " [-0.82591915],\n", + " [ 1.0605797 ]], dtype=float32)>" ] }, "execution_count": 13, @@ -492,8 +583,8 @@ "data": { "text/plain": [ "" @@ -522,7 +613,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-03 17:38:56.454126: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + "2025-01-27 12:16:04.075100: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" ] }, { @@ -530,10 +621,10 @@ "text/plain": [ "" + " [1., 0., 0., 0., 0.],\n", + " [1., 0., 0., 0., 0.]], dtype=float32)>" ] }, "execution_count": 16, @@ -550,6 +641,16 @@ "test_age_layer(test_age_col)" ] }, + { + "cell_type": "markdown", + "id": "afefbcf2", + "metadata": {}, + "source": [ + "### Preprocess selected features\n", + "\n", + "Apply the preprocessing utility functions defined earlier. Add all the feature inputs to a list.\n" + ] + }, { "cell_type": "code", "execution_count": 17, @@ -610,8 +711,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-03 17:38:56.758056: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n", - "2024-10-03 17:38:57.171981: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + "2025-01-27 12:16:04.387422: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n", + "2025-01-27 12:16:04.854147: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" ] } ], @@ -630,6 +731,14 @@ " encoded_features.append(encoded_categorical_col)" ] }, + { + "cell_type": "markdown", + "id": "e0dfac0d", + "metadata": {}, + "source": [ + "### Create, compile, and train model" + ] + }, { "cell_type": "code", "execution_count": 21, @@ -668,38 +777,32 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 1/10\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.4109 - loss: 0.7333 - val_accuracy: 0.6898 - val_loss: 0.5666\n", + "Epoch 1/10\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.5245 - loss: 0.6416 - val_accuracy: 0.7314 - val_loss: 0.5624\n", "Epoch 2/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.6423 - loss: 0.5994 - val_accuracy: 0.7210 - val_loss: 0.5484\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.6565 - loss: 0.5904 - val_accuracy: 0.7392 - val_loss: 0.5425\n", "Epoch 3/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 17ms/step - accuracy: 0.6825 - loss: 0.5728 - val_accuracy: 0.7253 - val_loss: 0.5383\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.6856 - loss: 0.5653 - val_accuracy: 0.7400 - val_loss: 0.5317\n", "Epoch 4/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 17ms/step - accuracy: 0.6796 - loss: 0.5653 - val_accuracy: 0.7331 - val_loss: 0.5314\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.6971 - loss: 0.5572 - val_accuracy: 0.7496 - val_loss: 0.5231\n", "Epoch 5/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 16ms/step - accuracy: 0.6853 - loss: 0.5584 - val_accuracy: 0.7348 - val_loss: 0.5259\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.7148 - loss: 0.5377 - val_accuracy: 0.7400 - val_loss: 0.5203\n", "Epoch 6/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.7120 - loss: 0.5447 - val_accuracy: 0.7418 - val_loss: 0.5218\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.7088 - loss: 0.5373 - val_accuracy: 0.7392 - val_loss: 0.5177\n", "Epoch 7/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.7068 - loss: 0.5422 - val_accuracy: 0.7435 - val_loss: 0.5189\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.7082 - loss: 0.5378 - val_accuracy: 0.7426 - val_loss: 0.5157\n", "Epoch 8/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.7043 - loss: 0.5397 - val_accuracy: 0.7435 - val_loss: 0.5162\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.7213 - loss: 0.5289 - val_accuracy: 0.7426 - val_loss: 0.5146\n", "Epoch 9/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.7172 - loss: 0.5372 - val_accuracy: 0.7496 - val_loss: 0.5146\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.7393 - loss: 0.5138 - val_accuracy: 0.7366 - val_loss: 0.5147\n", "Epoch 10/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.7337 - loss: 0.5232 - val_accuracy: 0.7409 - val_loss: 0.5131\n" + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 16ms/step - accuracy: 0.7253 - loss: 0.5173 - val_accuracy: 0.7366 - val_loss: 0.5134\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 23, @@ -721,8 +824,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m5/5\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 7ms/step - accuracy: 0.7480 - loss: 0.5028 \n", - "Accuracy 0.753032922744751\n" + "\u001b[1m1/5\u001b[0m \u001b[32m━━━━\u001b[0m\u001b[37m━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - accuracy: 0.7539 - loss: 0.4781" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[1m5/5\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 7ms/step - accuracy: 0.7403 - loss: 0.5034 \n", + "Accuracy 0.737434983253479\n" ] } ], @@ -736,42 +846,34 @@ "id": "7534616c-8561-4869-b6e9-7254ebdb2c3f", "metadata": {}, "source": [ - "## Save and Reload Model" + "### Save and reload model\n", + "\n", + "Demonstrate saving the trained model and reloading it for inference." ] }, { "cell_type": "code", "execution_count": 25, - "id": "52425a31-7f21-415e-b166-7682c7eb282c", - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow as tf" - ] - }, - { - "cell_type": "code", - "execution_count": 26, "id": "6bf0d024", "metadata": {}, "outputs": [], "source": [ - "model.save('my_pet_classifier.keras')" + "model.save('models/my_pet_classifier.keras')" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 26, "id": "d1a7be62", "metadata": {}, "outputs": [], "source": [ - "reloaded_model = tf.keras.models.load_model('my_pet_classifier.keras')" + "reloaded_model = tf.keras.models.load_model('models/my_pet_classifier.keras')" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 27, "id": "f3d2a2d5-fd4d-4320-bacc-fd4571cec709", "metadata": {}, "outputs": [ @@ -779,8 +881,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 26ms/step\n", - "This particular pet had a 81.1 percent probability of getting adopted.\n" + "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 27ms/step\n", + "This particular pet had a 80.3 percent probability of getting adopted.\n" ] } ], @@ -821,223 +923,277 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 28, "id": "fc8a0536", "metadata": {}, "outputs": [], "source": [ + "from pyspark.sql.functions import col, struct, pandas_udf\n", + "from pyspark.ml.functions import predict_batch_udf\n", + "from pyspark.sql.types import *\n", + "from pyspark.sql import SparkSession\n", "from pyspark import SparkConf\n", - "from pyspark.sql import SparkSession" + "import json\n", + "import pandas as pd" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "60dff1da", + "cell_type": "markdown", + "id": "bb5aa875", "metadata": {}, - "outputs": [], "source": [ - "import os\n", - "conda_env = os.environ.get(\"CONDA_PREFIX\")\n", - "\n", - "conf = SparkConf()\n", - "if 'spark' not in globals():\n", - " # If Spark is not already started with Jupyter, attach to Spark Standalone\n", - " import socket\n", - " hostname = socket.gethostname()\n", - " conf.setMaster(f\"spark://{hostname}:7077\") # assuming Master is on default port 7077\n", - "conf.set(\"spark.task.maxFailures\", \"1\")\n", - "conf.set(\"spark.driver.memory\", \"8g\")\n", - "conf.set(\"spark.executor.memory\", \"8g\")\n", - "conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled\", \"false\")\n", - "conf.set(\"spark.sql.pyspark.jvmStacktrace.enabled\", \"true\")\n", - "conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", - "conf.set(\"spark.python.worker.reuse\", \"true\")\n", - "# Create Spark Session\n", - "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", - "sc = spark.sparkContext" + "Check the cluster environment to handle any platform-specific Spark configurations." ] }, { "cell_type": "code", - "execution_count": 31, - "id": "3c64fd7b-3d1e-40f8-ab64-b5c13f8bbe77", + "execution_count": 29, + "id": "7701420e", "metadata": {}, "outputs": [], "source": [ - "df = spark.createDataFrame(dataframe).repartition(8)" + "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n", + "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n", + "on_standalone = not (on_databricks or on_dataproc)" + ] + }, + { + "cell_type": "markdown", + "id": "5e231dbd", + "metadata": {}, + "source": [ + "#### Create Spark Session\n", + "\n", + "For local standalone clusters, we'll connect to the cluster and create the Spark Session. \n", + "For CSP environments, Spark will either be preconfigured (Databricks) or we'll need to create the Spark Session (Dataproc)." ] }, { "cell_type": "code", - "execution_count": 32, - "id": "1be8215b-5068-41b4-849c-1c3ea7bb108a", + "execution_count": 30, + "id": "60dff1da", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "25/01/27 20:16:12 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/01/27 20:16:12 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", + "25/01/27 20:16:12 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "25/01/27 20:16:13 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" ] } ], "source": [ - "df.write.mode(\"overwrite\").parquet(\"datasets/petfinder-mini\")" + "conf = SparkConf()\n", + "\n", + "if 'spark' not in globals():\n", + " if on_standalone:\n", + " import socket\n", + " \n", + " conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + " hostname = socket.gethostname()\n", + " conf.setMaster(f\"spark://{hostname}:7077\")\n", + " conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", + " conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_env}/lib:{conda_env}/lib/python3.11/site-packages/nvidia_pytriton.libs:$LD_LIBRARY_PATH\")\n", + " source = \"/usr/lib/x86_64-linux-gnu/libstdc++.so.6\"\n", + " target = f\"{conda_env}/lib/libstdc++.so.6\"\n", + " try:\n", + " if os.path.islink(target) or os.path.exists(target):\n", + " os.remove(target)\n", + " os.symlink(source, target)\n", + " except OSError as e:\n", + " print(f\"Error creating symlink: {e}\")\n", + " elif on_dataproc:\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conda_lib_path=\"/opt/conda/miniconda3/lib\"\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_lib_path}:$LD_LIBRARY_PATH\")\n", + " conf.set(\"spark.executorEnv.TF_GPU_ALLOCATOR\", \"cuda_malloc_async\")\n", + " conf.set(\"spark.executor.instances\", \"4\") # dataproc defaults to 2\n", + "\n", + " conf.set(\"spark.executor.cores\", \"8\")\n", + " conf.set(\"spark.task.resource.gpu.amount\", \"0.125\")\n", + " conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n", + " conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", + " conf.set(\"spark.python.worker.reuse\", \"true\")\n", + "\n", + "conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"1000\")\n", + "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", + "sc = spark.sparkContext" + ] + }, + { + "cell_type": "markdown", + "id": "fa2333d1", + "metadata": {}, + "source": [ + "#### Create PySpark DataFrame" ] }, { "cell_type": "code", - "execution_count": 33, - "id": "d4dbde99-cf65-4c15-a163-754a0201a48d", + "execution_count": 31, + "id": "3c64fd7b-3d1e-40f8-ab64-b5c13f8bbe77", + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.createDataFrame(dataframe).repartition(8)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "1be8215b-5068-41b4-849c-1c3ea7bb108a", "metadata": {}, "outputs": [ { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+-------+---+--------+------+\n", - "|Type|Age| Breed1|Gender|Color1| Color2|MaturitySize|FurLength|Vaccinated|Sterilized| Health|Fee|PhotoAmt|target|\n", - "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+-------+---+--------+------+\n", - "| Cat| 3| Tabby| Male| Black| White| Small| Short| No| No|Healthy|100| 1| 1|\n", - "| Cat| 1|Domestic Medium Hair| Male| Black| Brown| Medium| Medium| Not Sure| Not Sure|Healthy| 0| 2| 1|\n", - "| Dog| 1| Mixed Breed| Male| Brown| White| Medium| Medium| Yes| No|Healthy| 0| 7| 1|\n", - "| Dog| 4| Mixed Breed|Female| Black| Brown| Medium| Short| Yes| No|Healthy|150| 8| 1|\n", - "| Dog| 1| Mixed Breed| Male| Black|No Color| Medium| Short| No| No|Healthy| 0| 3| 1|\n", - "| Cat| 3| Domestic Short Hair|Female| Cream| Gray| Medium| Short| No| No|Healthy| 0| 2| 1|\n", - "| Cat| 12| Domestic Long Hair| Male| Black|No Color| Medium| Long| No| Not Sure|Healthy|300| 3| 1|\n", - "| Cat| 2|Domestic Medium Hair|Female| Gray|No Color| Medium| Medium| No| No|Healthy| 0| 6| 1|\n", - "| Cat| 12|Domestic Medium Hair|Female| Black| White| Medium| Medium| Not Sure| Not Sure|Healthy| 0| 2| 0|\n", - "| Dog| 2| Mixed Breed| Male| Black| Brown| Medium| Short| No| No|Healthy| 0| 7| 1|\n", - "| Cat| 3| Domestic Long Hair|Female| Black| Brown| Large| Long| Yes| No|Healthy| 50| 2| 1|\n", - "| Dog| 2| Mixed Breed| Male| Brown| Cream| Medium| Long| Yes| No|Healthy| 0| 1| 1|\n", - "| Dog| 3| Mixed Breed|Female| Brown| Cream| Medium| Medium| Not Sure| Not Sure|Healthy| 0| 2| 1|\n", - "| Dog| 78| Terrier| Male| Black| White| Medium| Medium| Not Sure| Not Sure|Healthy| 0| 2| 0|\n", - "| Cat| 6| Domestic Short Hair|Female| Brown|No Color| Small| Short| Yes| Yes|Healthy| 0| 1| 1|\n", - "| Dog| 8| Mixed Breed|Female| Brown|No Color| Medium| Short| No| Yes|Healthy| 10| 2| 0|\n", - "| Dog| 2| Mixed Breed|Female| Black|No Color| Medium| Short| No| No|Healthy| 0| 8| 1|\n", - "| Dog| 12| Mixed Breed|Female| Brown| White| Medium| Medium| No| Yes|Healthy| 0| 7| 1|\n", - "| Dog| 10| Mixed Breed|Female| Black| Brown| Medium| Medium| Yes| Yes|Healthy| 0| 0| 0|\n", - "| Cat| 3| Domestic Short Hair| Male| Brown| White| Small| Short| No| No|Healthy| 0| 19| 1|\n", - "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+-------+---+--------+------+\n", - "only showing top 20 rows\n", - "\n" + " \r" ] } ], "source": [ - "df.show()" + "data_path = \"spark-dl-datasets/petfinder-mini\"\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " data_path = \"dbfs:/FileStore/\" + data_path\n", + "\n", + "df.write.mode(\"overwrite\").parquet(data_path)" ] }, { "cell_type": "markdown", - "id": "efa3e424-2920-44eb-afa0-885e40b620ed", + "id": "7cec4e0e", "metadata": {}, "source": [ - "## Inference using Spark DL API" + "#### Load and preprocess DataFrame" ] }, { "cell_type": "code", - "execution_count": 34, - "id": "4c21296c-20ed-43f8-921a-c85a820d1819", + "execution_count": 33, + "id": "0892f845", "metadata": {}, "outputs": [], "source": [ - "import numpy as np\n", - "import os\n", - "\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import struct, col\n", - "from pyspark.sql.types import ArrayType, FloatType" + "df = spark.read.parquet(data_path).cache()" ] }, { "cell_type": "code", - "execution_count": 35, - "id": "04b38f3a-70ea-4746-9f52-c50087401508", + "execution_count": 34, + "id": "952645dd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+-------+---+--------+------+\n", - "|Type|Age| Breed1|Gender|Color1| Color2|MaturitySize|FurLength|Vaccinated|Sterilized| Health|Fee|PhotoAmt|target|\n", - "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+-------+---+--------+------+\n", - "| Cat| 1|Domestic Medium Hair|Female| White|No Color| Small| Medium| No| No|Healthy| 0| 2| 1|\n", - "| Dog| 2| Mixed Breed|Female| Black| Brown| Medium| Medium| No| No|Healthy| 0| 3| 1|\n", - "| Dog| 18| Dalmatian|Female| Black| White| Medium| Medium| Yes| No|Healthy|350| 5| 1|\n", - "| Dog| 3| Mixed Breed|Female| Black|No Color| Medium| Short| No| No|Healthy| 0| 1| 0|\n", - "| Dog| 2| Mixed Breed| Male| Black| Brown| Medium| Short| No| No|Healthy| 0| 1| 1|\n", - "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+-------+---+--------+------+\n", - "only showing top 5 rows\n", - "\n" + "['Type', 'Age', 'Breed1', 'Gender', 'Color1', 'Color2', 'MaturitySize', 'FurLength', 'Vaccinated', 'Sterilized', 'Health', 'Fee', 'PhotoAmt', 'target']\n" ] } ], "source": [ - "df = spark.read.parquet(\"datasets/petfinder-mini\").cache()\n", - "df.show(5)" + "columns = df.columns\n", + "print(columns)" ] }, { "cell_type": "code", - "execution_count": 36, - "id": "29c27243-7c74-4045-aaf1-f75a322c0530", + "execution_count": 35, + "id": "b9c24c0d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "['Type', 'Age', 'Breed1', 'Gender', 'Color1', 'Color2', 'MaturitySize', 'FurLength', 'Vaccinated', 'Sterilized', 'Health', 'Fee', 'PhotoAmt', 'target']\n" + "['Type', 'Age', 'Breed1', 'Gender', 'Color1', 'Color2', 'MaturitySize', 'FurLength', 'Vaccinated', 'Sterilized', 'Health', 'Fee', 'PhotoAmt']\n" ] } ], "source": [ - "columns = df.columns\n", + "# remove label column\n", + "columns.remove(\"target\")\n", "print(columns)" ] }, { "cell_type": "code", - "execution_count": 37, - "id": "47508b14-97fa-42ee-a7d0-6175e6408283", - "metadata": { - "tags": [] - }, + "execution_count": 36, + "id": "d4dbde99-cf65-4c15-a163-754a0201a48d", + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "['Type', 'Age', 'Breed1', 'Gender', 'Color1', 'Color2', 'MaturitySize', 'FurLength', 'Vaccinated', 'Sterilized', 'Health', 'Fee', 'PhotoAmt']\n" + "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+-------+---+--------+------+\n", + "|Type|Age| Breed1|Gender|Color1| Color2|MaturitySize|FurLength|Vaccinated|Sterilized| Health|Fee|PhotoAmt|target|\n", + "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+-------+---+--------+------+\n", + "| Dog| 3| Mixed Breed| Male| Black|No Color| Small| Medium| Not Sure| Not Sure|Healthy| 0| 2| 0|\n", + "| Dog| 9| Mixed Breed| Male| Gray|No Color| Medium| Short| Not Sure| No|Healthy| 0| 4| 1|\n", + "| Cat| 4| Domestic Short Hair| Male| Black| Gray| Medium| Short| Not Sure| Not Sure|Healthy| 0| 4| 1|\n", + "| Cat| 6| Domestic Short Hair| Male|Yellow| White| Medium| Short| No| No|Healthy| 0| 3| 1|\n", + "| Cat| 6|Domestic Medium Hair| Male| Gray|No Color| Small| Medium| No| No|Healthy| 0| 4| 1|\n", + "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+-------+---+--------+------+\n", + "only showing top 5 rows\n", + "\n" ] } ], "source": [ - "# remove label column\n", - "columns.remove(\"target\")\n", - "print(columns)" + "df.show(5)" + ] + }, + { + "cell_type": "markdown", + "id": "824d7f97", + "metadata": {}, + "source": [ + "## Inference using Spark DL API\n", + "\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "\n", + "- predict_batch_fn uses Tensorflow APIs to load the model and return a predict function which operates on numpy arrays \n", + "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" ] }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 37, "id": "d62eb95a-54c6-44d2-9279-38fb65e0e160", "metadata": {}, "outputs": [], "source": [ "# get absolute path to model\n", - "model_dir = \"{}/my_pet_classifier.keras\".format(os.getcwd())" + "model_path = \"{}/models/my_pet_classifier.keras\".format(os.getcwd())\n", + "\n", + "# For cloud environments, copy the model to the distributed file system.\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-models\")\n", + " dbfs_model_path = \"/dbfs/FileStore/spark-dl-models/my_pet_classifier.keras\"\n", + " shutil.copy(model_path, dbfs_model_path)\n", + " model_path = dbfs_model_path\n", + "elif on_dataproc:\n", + " # GCS is mounted at /mnt/gcs by the init script\n", + " models_dir = \"/mnt/gcs/spark-dl/models\"\n", + " os.mkdir(models_dir) if not os.path.exists(models_dir) else None\n", + " gcs_model_path = models_dir + \"/my_pet_classifier.keras\"\n", + " shutil.copy(model_path, gcs_model_path)\n", + " model_path = gcs_model_path" ] }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 38, "id": "45665acf-50c8-445b-a985-b3dabd734709", "metadata": {}, "outputs": [], @@ -1055,7 +1211,7 @@ " except RuntimeError as e:\n", " print(e)\n", "\n", - " model = tf.keras.models.load_model(model_dir)\n", + " model = tf.keras.models.load_model(model_path)\n", "\n", " def predict(t, a, b, g, c1, c2, m, f, v, s, h, fee, p):\n", " inputs = {\n", @@ -1081,7 +1237,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 39, "id": "815e3b5f-7914-4235-85fa-50153dcd3d30", "metadata": {}, "outputs": [], @@ -1094,7 +1250,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 40, "id": "da03a0c6-2d39-425e-a9fa-57c139cca1ed", "metadata": {}, "outputs": [ @@ -1102,16 +1258,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "24/10/03 17:39:09 WARN SparkStringUtils: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n", - "[Stage 4:> (0 + 8) / 8]\r" + "25/01/27 20:16:16 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n", + "[Stage 5:=============================> (4 + 4) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 24.1 ms, sys: 6.37 ms, total: 30.4 ms\n", - "Wall time: 4.58 s\n" + "CPU times: user 25.8 ms, sys: 5.79 ms, total: 31.6 ms\n", + "Wall time: 5.04 s\n" ] }, { @@ -1130,7 +1286,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 41, "id": "03990c76-7198-49a7-bb5d-6870be915fb3", "metadata": {}, "outputs": [ @@ -1138,22 +1294,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 5:> (0 + 8) / 8]\r" + " \r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 98.2 ms, sys: 8.34 ms, total: 107 ms\n", - "Wall time: 1.57 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" + "CPU times: user 87.7 ms, sys: 11.4 ms, total: 99.1 ms\n", + "Wall time: 1.59 s\n" ] } ], @@ -1165,7 +1314,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 42, "id": "edb93cf3-c248-40c9-b8dc-acc8f51786a9", "metadata": {}, "outputs": [ @@ -1173,15 +1322,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 6:> (0 + 8) / 8]\r" + "[Stage 7:> (0 + 8) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 17.6 ms, sys: 2.19 ms, total: 19.8 ms\n", - "Wall time: 1.51 s\n" + "CPU times: user 14.9 ms, sys: 9.97 ms, total: 24.9 ms\n", + "Wall time: 1.53 s\n" ] }, { @@ -1200,7 +1349,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 43, "id": "a91f19cb-f7f1-4669-aff1-be594bea5378", "metadata": { "scrolled": true @@ -1213,26 +1362,26 @@ "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+------------+---+--------+------+-----------+\n", "|Type|Age| Breed1|Gender|Color1| Color2|MaturitySize|FurLength|Vaccinated|Sterilized| Health|Fee|PhotoAmt|target| preds|\n", "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+------------+---+--------+------+-----------+\n", - "| Cat| 1|Domestic Medium Hair|Female| White|No Color| Small| Medium| No| No| Healthy| 0| 2| 1| 1.9543833|\n", - "| Dog| 2| Mixed Breed|Female| Black| Brown| Medium| Medium| No| No| Healthy| 0| 3| 1| 1.7454995|\n", - "| Dog| 18| Dalmatian|Female| Black| White| Medium| Medium| Yes| No| Healthy|350| 5| 1| 1.5183508|\n", - "| Dog| 3| Mixed Breed|Female| Black|No Color| Medium| Short| No| No| Healthy| 0| 1| 0| 0.67013955|\n", - "| Dog| 2| Mixed Breed| Male| Black| Brown| Medium| Short| No| No| Healthy| 0| 1| 1| 1.5492269|\n", - "| Dog| 36| Mixed Breed| Male| Brown|No Color| Medium| Medium| Not Sure| Yes|Minor Injury| 0| 1| 0|-0.27595556|\n", - "| Cat| 6| Domestic Short Hair|Female| Black| White| Medium| Short| Yes| No| Healthy| 0| 5| 0| 0.7229555|\n", - "| Dog| 72| Golden Retriever|Female| Cream|No Color| Large| Medium| Yes| Not Sure| Healthy| 0| 1| 1| 0.7397226|\n", - "| Cat| 2| Domestic Short Hair| Male| Black| White| Small| Short| No| Yes| Healthy| 0| 4| 1| 1.4016482|\n", - "| Dog| 3| Irish Terrier| Male| Brown| Cream| Medium| Medium| Yes| No|Minor Injury|200| 3| 0| 1.3436754|\n", - "| Dog| 2| Mixed Breed|Female| White|No Color| Medium| Medium| Yes| No| Healthy| 0| 1| 1| 1.156439|\n", - "| Dog| 2| Mixed Breed| Male| Brown|No Color| Medium| Medium| Yes| No| Healthy| 0| 4| 1| 1.7760799|\n", - "| Cat| 2| Domestic Short Hair| Male| Black| Gray| Medium| Short| No| No| Healthy| 0| 2| 1| 1.8319463|\n", - "| Dog| 1| German Shepherd Dog| Male| Gray|No Color| Medium| Medium| No| No| Healthy| 0| 6| 1| 2.5471144|\n", - "| Dog| 24| Golden Retriever| Male|Yellow| Cream| Medium| Long| Yes| Yes| Healthy| 0| 7| 1| 1.4675076|\n", - "| Dog| 1| Mixed Breed|Female| Black| Brown| Small| Medium| No| Yes| Healthy| 0| 1| 0| 0.8451028|\n", - "| Cat| 12| Tuxedo| Male| Black|No Color| Small| Medium| Yes| Yes| Healthy| 50| 1| 1| 0.6487097|\n", - "| Cat| 3| Domestic Short Hair|Female| Black|No Color| Small| Short| No| No| Healthy| 0| 1| 1| 1.0688435|\n", - "| Dog| 2| Mixed Breed|Female| Brown| White| Medium| Short| No| No| Healthy| 0| 1| 1| 1.4086031|\n", - "| Dog| 11| Mixed Breed|Female|Golden|No Color| Medium| Short| Yes| Yes| Healthy| 0| 9| 1| 0.28429908|\n", + "| Dog| 3| Mixed Breed| Male| Black|No Color| Small| Medium| Not Sure| Not Sure| Healthy| 0| 2| 0| 0.1276686|\n", + "| Dog| 9| Mixed Breed| Male| Gray|No Color| Medium| Short| Not Sure| No| Healthy| 0| 4| 1| 0.48716992|\n", + "| Cat| 4| Domestic Short Hair| Male| Black| Gray| Medium| Short| Not Sure| Not Sure| Healthy| 0| 4| 1| 0.80358183|\n", + "| Cat| 6| Domestic Short Hair| Male|Yellow| White| Medium| Short| No| No| Healthy| 0| 3| 1| 0.8270739|\n", + "| Cat| 6|Domestic Medium Hair| Male| Gray|No Color| Small| Medium| No| No| Healthy| 0| 4| 1| 0.89303124|\n", + "| Cat| 5|Domestic Medium Hair|Female| Gray|No Color| Medium| Medium| Yes| Not Sure| Healthy| 0| 1| 0| 0.21722752|\n", + "| Dog| 24| Beagle|Female| Black| Golden| Medium| Short| Not Sure| Not Sure|Minor Injury| 0| 1| 1| 0.09258729|\n", + "| Cat| 29| Tabby| Male| Brown| Golden| Medium| Short| No| No| Healthy| 0| 1| 0| 0.74957174|\n", + "| Dog| 9| Mixed Breed|Female| Black| Brown| Medium| Short| Yes| Yes| Healthy| 0| 2| 0|-0.28708756|\n", + "| Dog| 2| Mixed Breed|Female| Cream| White| Medium| Short| No| No| Healthy| 0| 1| 0| 1.632988|\n", + "| Dog| 2| Mixed Breed| Male| Brown| White| Medium| Short| Yes| No| Healthy| 0| 1| 1| 1.512331|\n", + "| Dog| 60| Golden Retriever| Male| Brown| Yellow| Medium| Medium| Yes| Yes| Healthy| 0| 5| 1| 1.2067251|\n", + "| Cat| 9| Siamese| Male| White|No Color| Medium| Short| Yes| No| Healthy| 0| 2| 1| 1.07974|\n", + "| Dog| 19| Doberman Pinscher|Female| Black| Brown| Large| Short| Yes| Yes| Healthy|500| 2| 1| 0.48468828|\n", + "| Cat| 11| Domestic Short Hair| Male| Cream|No Color| Medium| Short| Yes| Yes| Healthy|100| 6| 0| 0.4290621|\n", + "| Dog| 18| Mixed Breed|Female| Brown| White| Small| Short| Yes| No| Healthy| 0| 5| 0| 0.26826438|\n", + "| Dog| 4| Mixed Breed|Female| Brown| White| Medium| Medium| Not Sure| Not Sure| Healthy| 0| 3| 0| 0.2859868|\n", + "| Dog| 96| Golden Retriever| Male|Golden|No Color| Large| Long| Yes| Yes| Healthy| 0| 2| 1| 1.5675049|\n", + "| Dog| 54| Golden Retriever| Male|Golden|No Color| Large| Medium| Yes| No| Healthy|350| 20| 1| 2.913511|\n", + "| Cat| 5|Domestic Medium Hair|Female| Brown| White| Medium| Medium| No| No| Healthy| 0| 5| 1| 1.0410445|\n", "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+------------+---+--------+------+-----------+\n", "only showing top 20 rows\n", "\n" @@ -1245,415 +1394,404 @@ }, { "cell_type": "markdown", - "id": "467b02a1-9f08-4fe8-a99c-581b7a01b8f6", - "metadata": {}, - "source": [ - "### Using Triton Inference Server\n", - "\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment." - ] - }, - { - "cell_type": "markdown", - "id": "22d1805b-7cac-4b27-9359-7a25b4ef3f71", + "id": "0c3e0390", "metadata": {}, "source": [ - "This notebook uses the [Python backend with a custom execution environment](https://github.com/triton-inference-server/python_backend#creating-custom-execution-environments) for Triton 24.08, using a conda-pack environment created as follows:\n", - "```\n", - "conda create -n tf-gpu -c conda-forge python=3.10.0\n", - "conda activate tf-gpu\n", + "## Using Triton Inference Server\n", + "In this section, we demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. \n", + "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server. \n", "\n", - "export PYTHONNOUSERSITE=True\n", - "pip install numpy==1.26.4 tensorflow[and-cuda] conda-pack\n", + "The process looks like this:\n", + "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n", + "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n", + "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n", + "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n", "\n", - "conda-pack # tf-gpu.tar.gz\n", - "```" + "\"drawing\"" ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 44, "id": "2605d134-ef75-4d94-9b16-2c6d85f29bef", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ - "import numpy as np\n", - "import os\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import col, struct\n", - "from pyspark.sql.types import ArrayType, FloatType" + "from functools import partial" + ] + }, + { + "cell_type": "markdown", + "id": "ea407357", + "metadata": {}, + "source": [ + "Import the utility functions from pytriton_utils.py:" ] }, { "cell_type": "code", - "execution_count": 46, - "id": "4666e618-8038-4dc5-9be7-793aedbf4500", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "sudo: a terminal is required to read the password; either use the -S option to read from standard input or configure an askpass helper\n", - "sudo: a password is required\n" - ] - } - ], + "execution_count": 45, + "id": "7e1e716f", + "metadata": {}, + "outputs": [], "source": [ - "%%bash\n", - "# copy custom model to expected layout for Triton\n", - "sudo rm -rf models\n", - "mkdir -p models\n", - "cp -r models_config/feature_columns models\n", + "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "# add custom execution environment\n", - "cp tf-gpu.tar.gz models" + "from pytriton_utils import (\n", + " use_stage_level_scheduling,\n", + " find_ports,\n", + " start_triton,\n", + " stop_triton\n", + ")" ] }, { "cell_type": "markdown", - "id": "91bd1003-46c7-42d1-ab4d-869e52d62146", + "id": "fcd28e7d", "metadata": {}, "source": [ - "#### Start Triton Server on each executor" + "Define the Triton Server function:" ] }, { "cell_type": "code", - "execution_count": 47, - "id": "a7fb146c-5319-4831-85f7-f2f3c084b042", - "metadata": { - "scrolled": true, - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "data": { - "text/plain": [ - "[True]" - ] - }, - "execution_count": 47, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": 46, + "id": "4666e618-8038-4dc5-9be7-793aedbf4500", + "metadata": {}, + "outputs": [], "source": [ - "num_executors = 1\n", - "triton_models_dir = \"{}/models\".format(os.getcwd())\n", - "my_pet_classifier_dir = \"{}/my_pet_classifier.keras\".format(os.getcwd())\n", - "nodeRDD = sc.parallelize(list(range(num_executors)), num_executors)\n", - "\n", - "def start_triton(it):\n", - " import docker\n", + "def triton_server(ports, model_path):\n", " import time\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " if containers:\n", - " print(\">>>> containers: {}\".format([c.short_id for c in containers]))\n", - " else:\n", - " container=client.containers.run(\n", - " \"nvcr.io/nvidia/tritonserver:24.08-py3\", \"tritonserver --model-repository=/models\",\n", - " detach=True,\n", - " device_requests=[docker.types.DeviceRequest(device_ids=[\"0\"], capabilities=[['gpu']])],\n", - " name=\"spark-triton\",\n", - " network_mode=\"host\",\n", - " remove=True,\n", - " shm_size=\"128M\",\n", - " volumes={\n", - " triton_models_dir: {\"bind\": \"/models\", \"mode\": \"ro\"},\n", - " my_pet_classifier_dir: {\"bind\": \"/my_pet_classifier.keras\", \"mode\": \"ro\"}\n", - " }\n", + " import signal\n", + " import numpy as np\n", + " import tensorflow as tf\n", + " from pytriton.decorators import batch\n", + " from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n", + " from pytriton.triton import Triton, TritonConfig\n", + " from pyspark import TaskContext\n", + "\n", + " print(f\"SERVER: Initializing model on worker {TaskContext.get().partitionId()}.\")\n", + " # Enable GPU memory growth\n", + " gpus = tf.config.experimental.list_physical_devices('GPU')\n", + " if gpus:\n", + " try:\n", + " for gpu in gpus:\n", + " tf.config.experimental.set_memory_growth(gpu, True)\n", + " except RuntimeError as e:\n", + " print(e)\n", + "\n", + " model = tf.keras.models.load_model(model_path)\n", + "\n", + " def decode(input_tensor):\n", + " return tf.convert_to_tensor(np.vectorize(lambda x: x.decode('utf-8'))(input_tensor))\n", + "\n", + " def identity(input_tensor):\n", + " return tf.convert_to_tensor(input_tensor)\n", + "\n", + " input_transforms = {\n", + " \"Type\": decode,\n", + " \"Age\": identity,\n", + " \"Breed1\": decode,\n", + " \"Gender\": decode,\n", + " \"Color1\": decode,\n", + " \"Color2\": decode,\n", + " \"MaturitySize\": decode,\n", + " \"FurLength\": decode,\n", + " \"Vaccinated\": decode,\n", + " \"Sterilized\": decode,\n", + " \"Health\": decode,\n", + " \"Fee\": identity,\n", + " \"PhotoAmt\": identity\n", + " }\n", + "\n", + " @batch\n", + " def _infer_fn(**inputs):\n", + " decoded_inputs = {k: input_transforms[k](v) for k, v in inputs.items()}\n", + " print(f\"SERVER: Received batch of size {len(decoded_inputs['Type'])}.\")\n", + " return {\n", + " \"preds\": model.predict(decoded_inputs)\n", + " }\n", + "\n", + " workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n", + " triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n", + " with Triton(config=triton_conf, workspace=workspace_path) as triton:\n", + " triton.bind(\n", + " model_name=\"PetClassifier\",\n", + " infer_func=_infer_fn,\n", + " inputs=[\n", + " Tensor(name=\"Type\", dtype=np.bytes_, shape=(-1,)),\n", + " Tensor(name=\"Age\", dtype=np.int64, shape=(-1,)),\n", + " Tensor(name=\"Breed1\", dtype=np.bytes_, shape=(-1,)),\n", + " Tensor(name=\"Gender\", dtype=np.bytes_, shape=(-1,)),\n", + " Tensor(name=\"Color1\", dtype=np.bytes_, shape=(-1,)),\n", + " Tensor(name=\"Color2\", dtype=np.bytes_, shape=(-1,)),\n", + " Tensor(name=\"MaturitySize\", dtype=np.bytes_, shape=(-1,)),\n", + " Tensor(name=\"FurLength\", dtype=np.bytes_, shape=(-1,)),\n", + " Tensor(name=\"Vaccinated\", dtype=np.bytes_, shape=(-1,)),\n", + " Tensor(name=\"Sterilized\", dtype=np.bytes_, shape=(-1,)),\n", + " Tensor(name=\"Health\", dtype=np.bytes_, shape=(-1,)),\n", + " Tensor(name=\"Fee\", dtype=np.int64, shape=(-1,)),\n", + " Tensor(name=\"PhotoAmt\", dtype=np.int64, shape=(-1,)),\n", + " ],\n", + " outputs=[\n", + " Tensor(name=\"preds\", dtype=np.float32, shape=(-1,)),\n", + " ],\n", + " config=ModelConfig(\n", + " max_batch_size=128,\n", + " batcher=DynamicBatcher(max_queue_delay_microseconds=5000), # 5ms\n", + " ),\n", + " strict=True,\n", " )\n", - " print(\">>>> starting triton: {}\".format(container.short_id))\n", "\n", - " # wait for triton to be running\n", - " time.sleep(15)\n", - " client = grpcclient.InferenceServerClient(\"localhost:8001\")\n", - " ready = False\n", - " while not ready:\n", - " try:\n", - " ready = client.is_server_ready()\n", - " except Exception as e:\n", - " time.sleep(5)\n", - " \n", - " return [True]\n", + " def _stop_triton(signum, frame):\n", + " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", + " triton.stop()\n", "\n", - "nodeRDD.barrier().mapPartitions(start_triton).collect()" + " signal.signal(signal.SIGTERM, _stop_triton)\n", + "\n", + " print(\"SERVER: Serving inference\")\n", + " triton.serve()" ] }, { "cell_type": "markdown", - "id": "b75e6f20-f06c-4f4c-ada1-c562e078ed4b", + "id": "617525a5", "metadata": {}, "source": [ - "#### Run inference" + "#### Start Triton servers" + ] + }, + { + "cell_type": "markdown", + "id": "08095b39", + "metadata": {}, + "source": [ + "**Specify the number of nodes in the cluster.** \n", + "Following the README, the example standalone cluster uses 1 node. The example Databricks/Dataproc cluster scripts use 4 nodes by default. " ] }, { "cell_type": "code", - "execution_count": 48, - "id": "fe8dc3e6-f1b1-4a24-85f4-0a5ecabef4c5", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 47, + "id": "1d8c358a", + "metadata": {}, "outputs": [], "source": [ - "df = spark.read.parquet(\"datasets/petfinder-mini\")" + "# Change based on cluster setup\n", + "num_nodes = 1 if on_standalone else 4" + ] + }, + { + "cell_type": "markdown", + "id": "1d96f480", + "metadata": {}, + "source": [ + "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " ] }, { "cell_type": "code", - "execution_count": 49, - "id": "ce92f041-930f-48ed-9a03-19f6c249ca27", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 48, + "id": "c9b98208", + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+-------+---+--------+------+\n", - "|Type|Age| Breed1|Gender|Color1| Color2|MaturitySize|FurLength|Vaccinated|Sterilized| Health|Fee|PhotoAmt|target|\n", - "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+-------+---+--------+------+\n", - "| Cat| 1|Domestic Medium Hair|Female| White|No Color| Small| Medium| No| No|Healthy| 0| 2| 1|\n", - "| Dog| 2| Mixed Breed|Female| Black| Brown| Medium| Medium| No| No|Healthy| 0| 3| 1|\n", - "| Dog| 18| Dalmatian|Female| Black| White| Medium| Medium| Yes| No|Healthy|350| 5| 1|\n", - "| Dog| 3| Mixed Breed|Female| Black|No Color| Medium| Short| No| No|Healthy| 0| 1| 0|\n", - "| Dog| 2| Mixed Breed| Male| Black| Brown| Medium| Short| No| No|Healthy| 0| 1| 1|\n", - "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+-------+---+--------+------+\n", - "only showing top 5 rows\n", - "\n" + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" ] } ], "source": [ - "df.show(5)" + "sc = spark.sparkContext\n", + "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" + ] + }, + { + "cell_type": "markdown", + "id": "ee5a2d8b", + "metadata": {}, + "source": [ + "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." ] }, { "cell_type": "code", - "execution_count": 50, - "id": "4cfb3f34-a215-4781-91bf-2bec85e15633", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 49, + "id": "918f14b8", + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "['Type', 'Age', 'Breed1', 'Gender', 'Color1', 'Color2', 'MaturitySize', 'FurLength', 'Vaccinated', 'Sterilized', 'Health', 'Fee', 'PhotoAmt', 'target']\n" + "Using ports [7000, 7001, 7002]\n" ] } ], "source": [ - "columns = df.columns\n", - "print(columns)" + "model_name = \"PetClassifier\"\n", + "ports = find_ports()\n", + "assert len(ports) == 3\n", + "print(f\"Using ports {ports}\")" ] }, { "cell_type": "code", - "execution_count": 51, - "id": "b315ee72-62af-476b-a994-0dba72d5f96e", - "metadata": { - "scrolled": true, - "tags": [ - "TRITON" - ] - }, + "execution_count": 50, + "id": "dc4ff00f", + "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 9:> (0 + 1) / 1]\r" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "['Type', 'Age', 'Breed1', 'Gender', 'Color1', 'Color2', 'MaturitySize', 'FurLength', 'Vaccinated', 'Sterilized', 'Health', 'Fee', 'PhotoAmt']\n" + "Triton Server PIDs:\n", + " {\n", + " \"cb4ae00-lcedt\": 3056153\n", + "}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" ] } ], "source": [ - "# remove label column\n", - "columns.remove(\"target\")\n", - "print(columns)" + "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", + " ports=ports,\n", + " model_name=model_name,\n", + " model_path=model_path)).collectAsMap()\n", + "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + ] + }, + { + "cell_type": "markdown", + "id": "cb560288", + "metadata": {}, + "source": [ + "#### Define client function" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "3eec95bb", + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{ports[0]}\"" ] }, { "cell_type": "code", "execution_count": 52, - "id": "da004eca-f7ad-4ee3-aa88-a6a20c1b72e5", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "id": "e50b5fc8", + "metadata": {}, "outputs": [], "source": [ - "def triton_fn(triton_uri, model_name):\n", + "def triton_fn(url, model_name):\n", " import numpy as np\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " np_types = {\n", - " \"BOOL\": np.dtype(np.bool_),\n", - " \"INT8\": np.dtype(np.int8),\n", - " \"INT16\": np.dtype(np.int16),\n", - " \"INT32\": np.dtype(np.int32),\n", - " \"INT64\": np.dtype(np.int64),\n", - " \"FP16\": np.dtype(np.float16),\n", - " \"FP32\": np.dtype(np.float32),\n", - " \"FP64\": np.dtype(np.float64),\n", - " \"FP64\": np.dtype(np.double),\n", - " \"BYTES\": np.dtype(object)\n", - " }\n", + " from pytriton.client import ModelClient\n", "\n", - " client = grpcclient.InferenceServerClient(triton_uri)\n", - " model_meta = client.get_model_metadata(model_name)\n", - " \n", - " def predict(t, a, b, g, c1, c2, m, f, v, s, h, fee, p):\n", - " # convert input ndarrays into a dictionary of ndarrays\n", - " inputs = {\n", - " \"Type\": t, \n", - " \"Age\": a, \n", - " \"Breed1\": b, \n", - " \"Gender\": g,\n", - " \"Color1\": c1,\n", - " \"Color2\": c2,\n", - " \"MaturitySize\": m,\n", - " \"FurLength\": f,\n", - " \"Vaccinated\": v, \n", - " \"Sterilized\": s,\n", - " \"Health\": h,\n", - " \"Fee\": fee,\n", - " \"PhotoAmt\": p\n", - " }\n", - " return _predict(inputs)\n", - " \n", - " def _predict(inputs):\n", - " if isinstance(inputs, np.ndarray):\n", - " # single ndarray input\n", - " request = [grpcclient.InferInput(model_meta.inputs[0].name, inputs.shape, model_meta.inputs[0].datatype)]\n", - " request[0].set_data_from_numpy(inputs.astype(np_types[model_meta.inputs[0].datatype]))\n", - " else:\n", - " # dict of multiple ndarray inputs\n", - " request = [grpcclient.InferInput(i.name, inputs[i.name].shape, i.datatype) for i in model_meta.inputs]\n", - " for i in request:\n", - " i.set_data_from_numpy(inputs[i.name()].astype(np_types[i.datatype()]))\n", - " \n", - " response = client.infer(model_name, inputs=request)\n", - " \n", - " if len(model_meta.outputs) > 1:\n", - " # return dictionary of numpy arrays\n", - " return {o.name: response.as_numpy(o.name) for o in model_meta.outputs}\n", - " else:\n", - " # return single numpy array\n", - " return response.as_numpy(model_meta.outputs[0].name)\n", - " \n", + " print(f\"CLIENT: Connecting to {model_name} at {url}\")\n", + "\n", + " def infer_batch(t, a, b, g, c1, c2, m, f, v, s, h, fee, p):\n", " \n", - " return predict" + " def encode(value):\n", + " return np.vectorize(lambda x: x.encode(\"utf-8\"))(value).astype(np.bytes_)\n", + "\n", + " with ModelClient(url, model_name, inference_timeout_s=240) as client:\n", + " encoded_inputs = {\n", + " \"Type\": encode(t), \n", + " \"Age\": a, \n", + " \"Breed1\": encode(b), \n", + " \"Gender\": encode(g),\n", + " \"Color1\": encode(c1),\n", + " \"Color2\": encode(c2),\n", + " \"MaturitySize\": encode(m),\n", + " \"FurLength\": encode(f),\n", + " \"Vaccinated\": encode(v),\n", + " \"Sterilized\": encode(s),\n", + " \"Health\": encode(h),\n", + " \"Fee\": fee,\n", + " \"PhotoAmt\": p\n", + " }\n", + " result_data = client.infer_batch(**encoded_inputs)\n", + " return result_data[\"preds\"]\n", + " \n", + " return infer_batch" + ] + }, + { + "cell_type": "markdown", + "id": "2edd887f", + "metadata": {}, + "source": [ + "#### Load and preprocess DataFrame" ] }, { "cell_type": "code", "execution_count": 53, - "id": "2ffb020e-dc93-456b-bee6-405611eee1e1", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "id": "fe8dc3e6-f1b1-4a24-85f4-0a5ecabef4c5", + "metadata": {}, "outputs": [], "source": [ - "from functools import partial\n", - "\n", - "# need to pass the list of columns into the model_udf\n", - "classify = predict_batch_udf(partial(triton_fn, triton_uri=\"localhost:8001\", model_name=\"feature_columns\"),\n", - " input_tensor_shapes=[[1]] * len(columns),\n", - " return_type=FloatType(),\n", - " batch_size=1024)" + "df = spark.read.parquet(data_path)" ] }, { "cell_type": "code", "execution_count": 54, - "id": "7657f820-5ec2-4ac8-a107-4b58773d204a", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----+---+----------+------+------+--------+------------+---------+----------+----------+----------+---+--------+------+----------+\n", - "|Type|Age| Breed1|Gender|Color1| Color2|MaturitySize|FurLength|Vaccinated|Sterilized| Health|Fee|PhotoAmt|target| preds|\n", - "+----+---+----------+------+------+--------+------------+---------+----------+----------+----------+---+--------+------+----------+\n", - "| Cat| 1|Domesti...|Female| White|No Color| Small| Medium| No| No| Healthy| 0| 2| 1| 1.9543833|\n", - "| Dog| 2|Mixed B...|Female| Black| Brown| Medium| Medium| No| No| Healthy| 0| 3| 1| 1.7454995|\n", - "| Dog| 18| Dalmatian|Female| Black| White| Medium| Medium| Yes| No| Healthy|350| 5| 1| 1.5183508|\n", - "| Dog| 3|Mixed B...|Female| Black|No Color| Medium| Short| No| No| Healthy| 0| 1| 0|0.67013955|\n", - "| Dog| 2|Mixed B...| Male| Black| Brown| Medium| Short| No| No| Healthy| 0| 1| 1| 1.5492269|\n", - "| Dog| 36|Mixed B...| Male| Brown|No Color| Medium| Medium| Not Sure| Yes|Minor I...| 0| 1| 0|-0.2759...|\n", - "| Cat| 6|Domesti...|Female| Black| White| Medium| Short| Yes| No| Healthy| 0| 5| 0| 0.7229555|\n", - "| Dog| 72|Golden ...|Female| Cream|No Color| Large| Medium| Yes| Not Sure| Healthy| 0| 1| 1| 0.7397226|\n", - "| Cat| 2|Domesti...| Male| Black| White| Small| Short| No| Yes| Healthy| 0| 4| 1| 1.4016482|\n", - "| Dog| 3|Irish T...| Male| Brown| Cream| Medium| Medium| Yes| No|Minor I...|200| 3| 0| 1.3436754|\n", - "| Dog| 2|Mixed B...|Female| White|No Color| Medium| Medium| Yes| No| Healthy| 0| 1| 1| 1.156439|\n", - "| Dog| 2|Mixed B...| Male| Brown|No Color| Medium| Medium| Yes| No| Healthy| 0| 4| 1| 1.7760799|\n", - "| Cat| 2|Domesti...| Male| Black| Gray| Medium| Short| No| No| Healthy| 0| 2| 1| 1.8319463|\n", - "| Dog| 1|German ...| Male| Gray|No Color| Medium| Medium| No| No| Healthy| 0| 6| 1| 2.5471144|\n", - "| Dog| 24|Golden ...| Male|Yellow| Cream| Medium| Long| Yes| Yes| Healthy| 0| 7| 1| 1.4675076|\n", - "| Dog| 1|Mixed B...|Female| Black| Brown| Small| Medium| No| Yes| Healthy| 0| 1| 0| 0.8451028|\n", - "| Cat| 12| Tuxedo| Male| Black|No Color| Small| Medium| Yes| Yes| Healthy| 50| 1| 1| 0.6487097|\n", - "| Cat| 3|Domesti...|Female| Black|No Color| Small| Short| No| No| Healthy| 0| 1| 1| 1.0688435|\n", - "| Dog| 2|Mixed B...|Female| Brown| White| Medium| Short| No| No| Healthy| 0| 1| 1| 1.4086031|\n", - "| Dog| 11|Mixed B...|Female|Golden|No Color| Medium| Short| Yes| Yes| Healthy| 0| 9| 1|0.28429908|\n", - "+----+---+----------+------+------+--------+------------+---------+----------+----------+----------+---+--------+------+----------+\n", - "only showing top 20 rows\n", - "\n" - ] - } - ], + "id": "4cfb3f34-a215-4781-91bf-2bec85e15633", + "metadata": {}, + "outputs": [], + "source": [ + "columns = df.columns\n", + "# remove label column\n", + "columns.remove(\"target\")" + ] + }, + { + "cell_type": "markdown", + "id": "b75e6f20-f06c-4f4c-ada1-c562e078ed4b", + "metadata": {}, "source": [ - "# WITHOUT custom python backend, FAILS with: Op type not registered 'DenseBincount' \n", - "df.withColumn(\"preds\", classify(struct(*columns))).show(truncate=10)" + "#### Run inference" ] }, { "cell_type": "code", "execution_count": 55, + "id": "2ffb020e-dc93-456b-bee6-405611eee1e1", + "metadata": {}, + "outputs": [], + "source": [ + "# need to pass the list of columns into the model_udf\n", + "classify = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", + " input_tensor_shapes=[[1]] * len(columns),\n", + " return_type=FloatType(),\n", + " batch_size=64)" + ] + }, + { + "cell_type": "code", + "execution_count": 56, "id": "e6ff0356-becd-421f-aebb-272497d5ad6a", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stderr", @@ -1666,8 +1804,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 17.2 ms, sys: 2.85 ms, total: 20 ms\n", - "Wall time: 2.5 s\n" + "CPU times: user 23 ms, sys: 8.87 ms, total: 31.8 ms\n", + "Wall time: 6.34 s\n" ] } ], @@ -1679,27 +1817,23 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 57, "id": "ce18ee7c-5958-4986-b200-6d986fcc6243", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 13:==================================================> (7 + 1) / 8]\r" + "[Stage 12:==================================================> (7 + 1) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 23.4 ms, sys: 984 μs, total: 24.4 ms\n", - "Wall time: 2.5 s\n" + "CPU times: user 16.6 ms, sys: 3.1 ms, total: 19.7 ms\n", + "Wall time: 5.83 s\n" ] }, { @@ -1718,13 +1852,9 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 58, "id": "0888ce40-b2c4-4aed-8ccb-6a8bcd00abc8", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stderr", @@ -1737,8 +1867,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 14.7 ms, sys: 4.61 ms, total: 19.3 ms\n", - "Wall time: 2.47 s\n" + "CPU times: user 85.3 ms, sys: 928 μs, total: 86.2 ms\n", + "Wall time: 5.8 s\n" ] } ], @@ -1750,13 +1880,9 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 59, "id": "d45812b5-f584-41a4-a821-2b59e065671c", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -1765,26 +1891,26 @@ "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+------------+---+--------+------+-----------+\n", "|Type|Age| Breed1|Gender|Color1| Color2|MaturitySize|FurLength|Vaccinated|Sterilized| Health|Fee|PhotoAmt|target| preds|\n", "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+------------+---+--------+------+-----------+\n", - "| Cat| 1|Domestic Medium Hair|Female| White|No Color| Small| Medium| No| No| Healthy| 0| 2| 1| 1.9543833|\n", - "| Dog| 2| Mixed Breed|Female| Black| Brown| Medium| Medium| No| No| Healthy| 0| 3| 1| 1.7454995|\n", - "| Dog| 18| Dalmatian|Female| Black| White| Medium| Medium| Yes| No| Healthy|350| 5| 1| 1.5183508|\n", - "| Dog| 3| Mixed Breed|Female| Black|No Color| Medium| Short| No| No| Healthy| 0| 1| 0| 0.67013955|\n", - "| Dog| 2| Mixed Breed| Male| Black| Brown| Medium| Short| No| No| Healthy| 0| 1| 1| 1.5492269|\n", - "| Dog| 36| Mixed Breed| Male| Brown|No Color| Medium| Medium| Not Sure| Yes|Minor Injury| 0| 1| 0|-0.27595556|\n", - "| Cat| 6| Domestic Short Hair|Female| Black| White| Medium| Short| Yes| No| Healthy| 0| 5| 0| 0.7229555|\n", - "| Dog| 72| Golden Retriever|Female| Cream|No Color| Large| Medium| Yes| Not Sure| Healthy| 0| 1| 1| 0.7397226|\n", - "| Cat| 2| Domestic Short Hair| Male| Black| White| Small| Short| No| Yes| Healthy| 0| 4| 1| 1.4016482|\n", - "| Dog| 3| Irish Terrier| Male| Brown| Cream| Medium| Medium| Yes| No|Minor Injury|200| 3| 0| 1.3436754|\n", - "| Dog| 2| Mixed Breed|Female| White|No Color| Medium| Medium| Yes| No| Healthy| 0| 1| 1| 1.156439|\n", - "| Dog| 2| Mixed Breed| Male| Brown|No Color| Medium| Medium| Yes| No| Healthy| 0| 4| 1| 1.7760799|\n", - "| Cat| 2| Domestic Short Hair| Male| Black| Gray| Medium| Short| No| No| Healthy| 0| 2| 1| 1.8319463|\n", - "| Dog| 1| German Shepherd Dog| Male| Gray|No Color| Medium| Medium| No| No| Healthy| 0| 6| 1| 2.5471144|\n", - "| Dog| 24| Golden Retriever| Male|Yellow| Cream| Medium| Long| Yes| Yes| Healthy| 0| 7| 1| 1.4675076|\n", - "| Dog| 1| Mixed Breed|Female| Black| Brown| Small| Medium| No| Yes| Healthy| 0| 1| 0| 0.8451028|\n", - "| Cat| 12| Tuxedo| Male| Black|No Color| Small| Medium| Yes| Yes| Healthy| 50| 1| 1| 0.6487097|\n", - "| Cat| 3| Domestic Short Hair|Female| Black|No Color| Small| Short| No| No| Healthy| 0| 1| 1| 1.0688435|\n", - "| Dog| 2| Mixed Breed|Female| Brown| White| Medium| Short| No| No| Healthy| 0| 1| 1| 1.4086031|\n", - "| Dog| 11| Mixed Breed|Female|Golden|No Color| Medium| Short| Yes| Yes| Healthy| 0| 9| 1| 0.28429908|\n", + "| Dog| 3| Mixed Breed| Male| Black|No Color| Small| Medium| Not Sure| Not Sure| Healthy| 0| 2| 0| 0.1276686|\n", + "| Dog| 9| Mixed Breed| Male| Gray|No Color| Medium| Short| Not Sure| No| Healthy| 0| 4| 1| 0.48716992|\n", + "| Cat| 4| Domestic Short Hair| Male| Black| Gray| Medium| Short| Not Sure| Not Sure| Healthy| 0| 4| 1| 0.80358183|\n", + "| Cat| 6| Domestic Short Hair| Male|Yellow| White| Medium| Short| No| No| Healthy| 0| 3| 1| 0.8270739|\n", + "| Cat| 6|Domestic Medium Hair| Male| Gray|No Color| Small| Medium| No| No| Healthy| 0| 4| 1| 0.89303124|\n", + "| Cat| 5|Domestic Medium Hair|Female| Gray|No Color| Medium| Medium| Yes| Not Sure| Healthy| 0| 1| 0| 0.21722752|\n", + "| Dog| 24| Beagle|Female| Black| Golden| Medium| Short| Not Sure| Not Sure|Minor Injury| 0| 1| 1| 0.09258729|\n", + "| Cat| 29| Tabby| Male| Brown| Golden| Medium| Short| No| No| Healthy| 0| 1| 0| 0.74957174|\n", + "| Dog| 9| Mixed Breed|Female| Black| Brown| Medium| Short| Yes| Yes| Healthy| 0| 2| 0|-0.28708756|\n", + "| Dog| 2| Mixed Breed|Female| Cream| White| Medium| Short| No| No| Healthy| 0| 1| 0| 1.632988|\n", + "| Dog| 2| Mixed Breed| Male| Brown| White| Medium| Short| Yes| No| Healthy| 0| 1| 1| 1.512331|\n", + "| Dog| 60| Golden Retriever| Male| Brown| Yellow| Medium| Medium| Yes| Yes| Healthy| 0| 5| 1| 1.2067251|\n", + "| Cat| 9| Siamese| Male| White|No Color| Medium| Short| Yes| No| Healthy| 0| 2| 1| 1.07974|\n", + "| Dog| 19| Doberman Pinscher|Female| Black| Brown| Large| Short| Yes| Yes| Healthy|500| 2| 1| 0.48468828|\n", + "| Cat| 11| Domestic Short Hair| Male| Cream|No Color| Medium| Short| Yes| Yes| Healthy|100| 6| 0| 0.4290621|\n", + "| Dog| 18| Mixed Breed|Female| Brown| White| Small| Short| Yes| No| Healthy| 0| 5| 0| 0.26826438|\n", + "| Dog| 4| Mixed Breed|Female| Brown| White| Medium| Medium| Not Sure| Not Sure| Healthy| 0| 3| 0| 0.2859868|\n", + "| Dog| 96| Golden Retriever| Male|Golden|No Color| Large| Long| Yes| Yes| Healthy| 0| 2| 1| 1.5675049|\n", + "| Dog| 54| Golden Retriever| Male|Golden|No Color| Large| Medium| Yes| No| Healthy|350| 20| 1| 2.913511|\n", + "| Cat| 5|Domestic Medium Hair|Female| Brown| White| Medium| Medium| No| No| Healthy| 0| 5| 1| 1.0410445|\n", "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+------------+---+--------+------+-----------+\n", "only showing top 20 rows\n", "\n" @@ -1807,14 +1933,17 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 60, "id": "6914f44f-677f-4db3-be09-783df8d11b8a", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + }, { "name": "stderr", "output_type": "stream", @@ -1828,36 +1957,26 @@ "[True]" ] }, - "execution_count": 59, + "execution_count": 60, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "def stop_triton(it):\n", - " import docker\n", - " import time\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " print(\">>>> stopping containers: {}\".format([c.short_id for c in containers]))\n", - " if containers:\n", - " container=containers[0]\n", - " container.stop(timeout=120)\n", - "\n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(stop_triton).collect()" + "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", + "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" ] }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 61, "id": "f8c6ee43-8891-4446-986e-1447c5d48bac", "metadata": {}, "outputs": [], "source": [ - "spark.stop()" + "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n", + " spark.stop()" ] }, { diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_resnet50_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_resnet50_tf.ipynb new file mode 100644 index 00000000..1a9b82e8 --- /dev/null +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_resnet50_tf.ipynb @@ -0,0 +1,1415 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8e6810cc-5982-4293-bfbd-c91ef0aca204", + "metadata": {}, + "source": [ + "\n", + "\n", + "# PySpark Tensorflow Inference\n", + "\n", + "### Flower Recognition with Keras Resnet50\n", + "\n", + "In this notebook, we demonstrate distribute inference with Resnet50 on the Databricks flower photos dataset. \n", + "From: https://docs.databricks.com/_static/notebooks/deep-learning/keras-metadata.html" + ] + }, + { + "cell_type": "markdown", + "id": "858e3a8d", + "metadata": {}, + "source": [ + "Note that cuFFT/cuDNN/cuBLAS registration errors are expected (as of `tf=2.17.0`) and will not affect behavior, as noted in [this issue.](https://github.com/tensorflow/tensorflow/issues/62075) " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "cf329ac8-0763-44bc-b0f6-b634b7dc480e", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-01-27 12:17:01.457688: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2025-01-27 12:17:01.464818: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2025-01-27 12:17:01.472654: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2025-01-27 12:17:01.474956: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2025-01-27 12:17:01.481116: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "2025-01-27 12:17:01.844814: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" + ] + } + ], + "source": [ + "import os\n", + "import shutil\n", + "import subprocess\n", + "import time\n", + "import json\n", + "import pandas as pd\n", + "from PIL import Image\n", + "import numpy as np\n", + "import uuid\n", + " \n", + "import tensorflow as tf\n", + "from tensorflow.keras.applications.resnet50 import ResNet50" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "532d562d", + "metadata": {}, + "outputs": [], + "source": [ + "os.mkdir('models') if not os.path.exists('models') else None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "75175140", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2.17.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", + "I0000 00:00:1738009022.171730 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738009022.193480 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738009022.196408 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" + ] + } + ], + "source": [ + "print(tf.__version__)\n", + "\n", + "# Enable GPU memory growth\n", + "gpus = tf.config.experimental.list_physical_devices('GPU')\n", + "if gpus:\n", + " try:\n", + " for gpu in gpus:\n", + " tf.config.experimental.set_memory_growth(gpu, True)\n", + " except RuntimeError as e:\n", + " print(e)" + ] + }, + { + "cell_type": "markdown", + "id": "02fe61b8", + "metadata": {}, + "source": [ + "## PySpark" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b474339c", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql.functions import col, struct, pandas_udf, PandasUDFType\n", + "from pyspark.ml.functions import predict_batch_udf\n", + "from pyspark.sql.types import *\n", + "from pyspark.sql import SparkSession\n", + "from pyspark import SparkConf\n", + "from typing import Iterator, Tuple" + ] + }, + { + "cell_type": "markdown", + "id": "e182cacb", + "metadata": {}, + "source": [ + "Check the cluster environment to handle any platform-specific Spark configurations." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "564b1d33", + "metadata": {}, + "outputs": [], + "source": [ + "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n", + "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n", + "on_standalone = not (on_databricks or on_dataproc)" + ] + }, + { + "cell_type": "markdown", + "id": "016cdd0b", + "metadata": {}, + "source": [ + "#### Create Spark Session\n", + "\n", + "For local standalone clusters, we'll connect to the cluster and create the Spark Session. \n", + "For CSP environments, Spark will either be preconfigured (Databricks) or we'll need to create the Spark Session (Dataproc)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "44d72768", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/27 20:17:02 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/01/27 20:17:02 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", + "25/01/27 20:17:03 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "25/01/27 20:17:03 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + ] + } + ], + "source": [ + "conf = SparkConf()\n", + "\n", + "if 'spark' not in globals():\n", + " if on_standalone:\n", + " import socket\n", + " \n", + " conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + " hostname = socket.gethostname()\n", + " conf.setMaster(f\"spark://{hostname}:7077\")\n", + " conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", + " conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_env}/lib:{conda_env}/lib/python3.11/site-packages/nvidia_pytriton.libs:$LD_LIBRARY_PATH\")\n", + " source = \"/usr/lib/x86_64-linux-gnu/libstdc++.so.6\"\n", + " target = f\"{conda_env}/lib/libstdc++.so.6\"\n", + " try:\n", + " if os.path.islink(target) or os.path.exists(target):\n", + " os.remove(target)\n", + " os.symlink(source, target)\n", + " except OSError as e:\n", + " print(f\"Error creating symlink: {e}\")\n", + " elif on_dataproc:\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conda_lib_path=\"/opt/conda/miniconda3/lib\"\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_lib_path}:$LD_LIBRARY_PATH\") \n", + " conf.set(\"spark.executorEnv.TF_GPU_ALLOCATOR\", \"cuda_malloc_async\")\n", + " conf.set(\"spark.executor.instances\", \"4\") # dataproc defaults to 2\n", + "\n", + " conf.set(\"spark.executor.cores\", \"8\")\n", + " conf.set(\"spark.task.resource.gpu.amount\", \"0.125\")\n", + " conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n", + " conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", + " conf.set(\"spark.python.worker.reuse\", \"true\")\n", + " conf.set(\"spark.driver.memory\", \"8g\")\n", + " conf.set(\"spark.executor.memory\", \"8g\")\n", + "\n", + "conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"512\")\n", + "conf.set(\"spark.sql.parquet.columnarReaderBatchSize\", \"1024\")\n", + "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", + "sc = spark.sparkContext" + ] + }, + { + "cell_type": "markdown", + "id": "61c406fa", + "metadata": {}, + "source": [ + "Define the input and output directories." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "c566dc17", + "metadata": {}, + "outputs": [], + "source": [ + "os.mkdirs(\"spark-dl-datasets\") if not os.path.exists(\"spark-dl-datasets\") else None\n", + "data_path = \"spark-dl-datasets/flowers_{uuid}.parquet\".format(uuid=str(uuid.uuid1()))\n", + "local_file_path = f\"{os.getcwd()}/{data_path}\"\n", + "output_file_path = \"predictions/predictions\"" + ] + }, + { + "cell_type": "markdown", + "id": "968d08a7-66b9-444f-b362-d8df692aef1c", + "metadata": {}, + "source": [ + "### Prepare trained model and data for inference" + ] + }, + { + "cell_type": "markdown", + "id": "da083168-137f-492c-8769-d8f1e2111756", + "metadata": {}, + "source": [ + "Load the ResNet-50 Model and broadcast the weights." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "2ddc715a-cdbc-4c49-93e9-58c9d88511da", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "I0000 00:00:1738009023.868557 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738009023.871444 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738009023.874007 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738009023.990970 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738009023.992104 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738009023.993027 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2025-01-27 12:17:03.993936: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 40005 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" + ] + } + ], + "source": [ + "model = ResNet50()\n", + "bc_model_weights = sc.broadcast(model.get_weights())" + ] + }, + { + "cell_type": "markdown", + "id": "77dddfa3-e8df-4e8e-8251-64457f1ebf80", + "metadata": {}, + "source": [ + "Load the data and save the datasets to one Parquet file." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "c0738bec-97d4-4946-8c49-5e6d07ff1afc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Image count: 3670\n" + ] + } + ], + "source": [ + "import pathlib\n", + "dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz\"\n", + "data_dir = tf.keras.utils.get_file(origin=dataset_url,\n", + " fname='flower_photos',\n", + " untar=True)\n", + "data_dir = pathlib.Path(data_dir)\n", + "image_count = len(list(data_dir.glob('*/*.jpg')))\n", + "print(f\"Image count: {image_count}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "d54f470a-d308-4426-8ed0-33f95155bb4f", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "files = [os.path.join(dp, f) for dp, dn, filenames in os.walk(data_dir) for f in filenames if os.path.splitext(f)[1] == '.jpg']\n", + "files = files[:2048]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "64f94ee0-f1ea-47f6-a77e-be8da5d1b87a", + "metadata": {}, + "outputs": [], + "source": [ + "image_data = []\n", + "for file in files:\n", + " img = Image.open(file)\n", + " img = img.resize([224, 224])\n", + " data = np.asarray(img, dtype=\"float32\").reshape([224*224*3])\n", + "\n", + " image_data.append({\"data\": data})" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "b4ae1a98", + "metadata": {}, + "outputs": [], + "source": [ + "pd.DataFrame(image_data, columns=['data']).to_parquet(data_path)\n", + "\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " shutil.copy(local_file_path, \"/dbfs/FileStore/{}\".format(data_path))\n", + " data_path = \"/dbfs/FileStore/{}\".format(data_path)\n", + "elif on_dataproc:\n", + " data_dir = \"/mnt/gcs/spark-dl/spark-dl-datasets\"\n", + " os.mkdir(data_dir) if not os.path.exists(data_dir) else None\n", + " shutil.copy(local_file_path, \"/mnt/gcs/spark-dl/\" + data_path)\n", + " data_path = \"file:///mnt/gcs/spark-dl/\" + data_path" + ] + }, + { + "cell_type": "markdown", + "id": "f2414b0f-58f2-4e4a-9d09-8ea95b38d413", + "metadata": {}, + "source": [ + "### Save Model\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "670328e3-7274-4d78-b315-487750166a3f", + "metadata": {}, + "outputs": [], + "source": [ + "model_path = 'models/resnet50_model.keras'\n", + "model.save(model_path)\n", + "\n", + "# For cloud environments, copy the model to the distributed file system.\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-models\")\n", + " dbfs_model_path = \"/dbfs/FileStore/spark-dl-models/resnet50_model.keras\"\n", + " shutil.copy(model_path, dbfs_model_path)\n", + " model_path = dbfs_model_path\n", + "elif on_dataproc:\n", + " # GCS is mounted at /mnt/gcs by the init script\n", + " models_dir = \"/mnt/gcs/spark-dl/models\"\n", + " os.mkdir(models_dir) if not os.path.exists(models_dir) else None\n", + " gcs_model_path = models_dir + \"/resnet50_model.keras\"\n", + " shutil.copy(model_path, gcs_model_path)\n", + " model_path = gcs_model_path" + ] + }, + { + "cell_type": "markdown", + "id": "b827ad56-1af0-41b7-be68-94bd203a2a70", + "metadata": {}, + "source": [ + "### Load the data into Spark DataFrames" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "8ddc22d0-b88a-4906-bd47-bf247e34feeb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2048\n" + ] + } + ], + "source": [ + "df = spark.read.parquet(data_path)\n", + "print(df.count())" + ] + }, + { + "cell_type": "markdown", + "id": "865929b0-b016-4de4-996d-7f16176cf49c", + "metadata": { + "tags": [] + }, + "source": [ + "### Model inference via Pandas UDF" + ] + }, + { + "cell_type": "markdown", + "id": "b1f5a747", + "metadata": {}, + "source": [ + "Define the function to parse the input data." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "a67b3128-13c1-44f1-a0c0-7cf7a836fee3", + "metadata": {}, + "outputs": [], + "source": [ + "def parse_image(image_data):\n", + " image = tf.image.convert_image_dtype(\n", + " image_data, dtype=tf.float32) * (2. / 255) - 1\n", + " image = tf.reshape(image, [224, 224, 3])\n", + " return image" + ] + }, + { + "cell_type": "markdown", + "id": "024e4ba2", + "metadata": {}, + "source": [ + "Define the function for model inference." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "7b33185f-6d1e-4ca9-9757-fdc3d736496b", + "metadata": {}, + "outputs": [], + "source": [ + "@pandas_udf(ArrayType(FloatType()))\n", + "def pandas_predict_udf(iter: Iterator[Tuple[pd.Series]]) -> Iterator[pd.Series]:\n", + "\n", + " # Enable GPU memory growth to avoid CUDA OOM\n", + " gpus = tf.config.experimental.list_physical_devices('GPU')\n", + " if gpus:\n", + " try:\n", + " for gpu in gpus:\n", + " tf.config.experimental.set_memory_growth(gpu, True)\n", + " except RuntimeError as e:\n", + " print(e)\n", + "\n", + " batch_size = 64\n", + " model = ResNet50(weights=None)\n", + " model.set_weights(bc_model_weights.value)\n", + " for image_batch in iter:\n", + " images = np.vstack(image_batch)\n", + " dataset = tf.data.Dataset.from_tensor_slices(images)\n", + " dataset = dataset.map(parse_image, num_parallel_calls=8).prefetch(\n", + " 5000).batch(batch_size)\n", + " preds = model.predict(dataset)\n", + " yield pd.Series(list(preds))" + ] + }, + { + "cell_type": "markdown", + "id": "08190547", + "metadata": {}, + "source": [ + "Run model inference and save the results to Parquet." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ad8c05da-db38-45ef-81d0-1f862f575ced", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 48.6 ms, sys: 19.9 ms, total: 68.5 ms\n", + "Wall time: 15 s\n" + ] + } + ], + "source": [ + "%%time\n", + "predictions_1 = df.select(pandas_predict_udf(col(\"data\")).alias(\"prediction\"))\n", + "results = predictions_1.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "08cb2a10", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 6:============================================> (3 + 1) / 4]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------+\n", + "| prediction|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "|[1.296447E-4, 2.465122E-4, 6.7463385E-5, 1.2231144E-4, 5.731739E-5, 3.9644213E-4, 7.0297688E-6, 4...|\n", + "|[4.4481887E-5, 3.526653E-4, 4.683818E-5, 8.1168495E-5, 3.178377E-5, 1.9188467E-4, 7.885617E-6, 1....|\n", + "|[1.05946536E-4, 2.2744355E-4, 3.0219735E-5, 6.548672E-5, 2.3649674E-5, 3.7177472E-4, 3.353236E-6,...|\n", + "|[2.0392703E-5, 2.2817637E-4, 7.840744E-5, 6.9875685E-5, 4.702542E-5, 9.8244425E-5, 5.5829764E-6, ...|\n", + "|[1.1312391E-4, 2.31244E-4, 5.279228E-5, 1.0859927E-4, 4.0202678E-5, 3.721753E-4, 5.563934E-6, 3.4...|\n", + "|[9.126345E-5, 2.0679034E-4, 4.5165678E-5, 7.679106E-5, 3.234611E-5, 3.3994843E-4, 3.84E-6, 4.1930...|\n", + "|[1.07930486E-4, 3.7741542E-4, 7.613175E-5, 1.2414041E-4, 4.7409427E-5, 3.332554E-4, 1.05853915E-5...|\n", + "|[2.2216762E-5, 2.7354853E-4, 3.8192928E-5, 6.2340725E-5, 1.7952003E-5, 1.7253387E-4, 6.020507E-6,...|\n", + "|[1.10480236E-4, 2.89734E-4, 4.239379E-5, 1.0727814E-4, 3.047985E-5, 4.7992737E-4, 6.4530495E-6, 3...|\n", + "|[9.6864875E-5, 2.0573521E-4, 7.4498465E-5, 1.1323085E-4, 4.6088306E-5, 2.8680824E-4, 5.604823E-6,...|\n", + "|[7.4198484E-5, 3.2886668E-4, 1.3441108E-4, 1.7755068E-4, 8.469927E-5, 2.2534095E-4, 1.3617541E-5,...|\n", + "|[8.7561886E-5, 2.7312653E-4, 3.5959012E-5, 7.7946424E-5, 2.3565723E-5, 3.6881721E-4, 3.5630535E-6...|\n", + "|[9.743975E-5, 2.7615853E-4, 5.74148E-5, 1.10329434E-4, 3.83045E-5, 3.500394E-4, 6.167429E-6, 4.42...|\n", + "|[6.9320704E-5, 2.53287E-4, 5.0612853E-5, 1.14936556E-4, 3.0210098E-5, 2.7870742E-4, 5.031114E-6, ...|\n", + "|[4.2203726E-5, 2.4911022E-4, 1.2378568E-4, 1.4274308E-4, 7.32259E-5, 1.6058519E-4, 7.9425035E-6, ...|\n", + "|[2.7190901E-5, 3.8381666E-4, 1.2918573E-4, 1.570463E-4, 7.310112E-5, 8.554618E-5, 1.2614603E-5, 1...|\n", + "|[3.0573912E-5, 3.5561546E-4, 1.5945674E-4, 2.1361349E-4, 8.046549E-5, 1.0269262E-4, 1.3862439E-5,...|\n", + "|[3.3117096E-5, 2.8073433E-4, 1.7961214E-4, 2.020287E-4, 1.3662946E-4, 1.0117796E-4, 3.4090703E-5,...|\n", + "|[4.5728237E-5, 2.8880237E-4, 2.3783019E-4, 2.4589908E-4, 1.2160292E-4, 1.3812551E-4, 1.6343482E-5...|\n", + "|[1.2280059E-4, 2.806991E-4, 6.3642765E-5, 1.02471764E-4, 4.351664E-5, 3.9150563E-4, 8.235125E-6, ...|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "only showing top 20 rows\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "predictions_1.show(truncate=100)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "40799f8e-443e-40ca-919b-391f901cb3f4", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "predictions_1.write.mode(\"overwrite\").parquet(output_file_path + \"_1\")" + ] + }, + { + "cell_type": "markdown", + "id": "e7a69aa9", + "metadata": {}, + "source": [ + "## Inference using Spark DL API\n", + "\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "\n", + "- predict_batch_fn uses Tensorflow APIs to load the model and return a predict function which operates on numpy arrays \n", + "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "dda88b46-6300-4bf7-bc10-7403f4fbbf92", + "metadata": {}, + "outputs": [], + "source": [ + "def predict_batch_fn():\n", + " import tensorflow as tf\n", + " from tensorflow.keras.applications.resnet50 import ResNet50\n", + "\n", + " # Enable GPU memory growth\n", + " gpus = tf.config.experimental.list_physical_devices('GPU')\n", + " if gpus:\n", + " try:\n", + " for gpu in gpus:\n", + " tf.config.experimental.set_memory_growth(gpu, True)\n", + " except RuntimeError as e:\n", + " print(e)\n", + "\n", + " model = ResNet50()\n", + " def predict(inputs):\n", + " inputs = inputs * (2. / 255) - 1\n", + " return model.predict(inputs)\n", + " return predict" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "cff0e851-563d-40b6-9d05-509c22b3b7f9", + "metadata": {}, + "outputs": [], + "source": [ + "classify = predict_batch_udf(predict_batch_fn,\n", + " input_tensor_shapes=[[224, 224, 3]],\n", + " return_type=ArrayType(FloatType()),\n", + " batch_size=50)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "aa7c156f-e2b3-4837-9427-ccf3a5720412", + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.parquet(data_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "80bc50ad-eaf5-4fce-a354-5e17d65e2da5", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 71.1 ms, sys: 32.7 ms, total: 104 ms\n", + "Wall time: 16.6 s\n" + ] + } + ], + "source": [ + "%%time\n", + "# first pass caches model/fn\n", + "predictions_2 = df.select(classify(struct(\"data\")).alias(\"prediction\"))\n", + "results = predictions_2.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "41cace80-7a4b-4929-8e63-9c83f9745e02", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 135 ms, sys: 27.5 ms, total: 163 ms\n", + "Wall time: 15.5 s\n" + ] + } + ], + "source": [ + "%%time\n", + "predictions_2 = df.select(classify(\"data\").alias(\"prediction\"))\n", + "results = predictions_2.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "56a2ec8a-de09-4d7c-9666-1b3c76f10657", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 53 ms, sys: 13.7 ms, total: 66.7 ms\n", + "Wall time: 16.2 s\n" + ] + } + ], + "source": [ + "%%time\n", + "predictions_2 = df.select(classify(col(\"data\")).alias(\"prediction\"))\n", + "results = predictions_2.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "2dcf3791", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 13:===========================================> (3 + 1) / 4]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------+\n", + "| prediction|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "|[1.296447E-4, 2.465122E-4, 6.7463385E-5, 1.2231144E-4, 5.731739E-5, 3.9644213E-4, 7.0297688E-6, 4...|\n", + "|[4.4481887E-5, 3.526653E-4, 4.683818E-5, 8.1168495E-5, 3.178377E-5, 1.9188467E-4, 7.885617E-6, 1....|\n", + "|[1.05946536E-4, 2.2744355E-4, 3.0219735E-5, 6.548672E-5, 2.3649674E-5, 3.7177472E-4, 3.353236E-6,...|\n", + "|[2.0392703E-5, 2.2817637E-4, 7.840744E-5, 6.9875685E-5, 4.702542E-5, 9.8244425E-5, 5.5829764E-6, ...|\n", + "|[1.1312391E-4, 2.31244E-4, 5.279228E-5, 1.0859927E-4, 4.0202678E-5, 3.721753E-4, 5.563934E-6, 3.4...|\n", + "|[9.126345E-5, 2.0679034E-4, 4.5165678E-5, 7.679106E-5, 3.234611E-5, 3.3994843E-4, 3.84E-6, 4.1930...|\n", + "|[1.07930486E-4, 3.7741542E-4, 7.613175E-5, 1.2414041E-4, 4.7409427E-5, 3.332554E-4, 1.05853915E-5...|\n", + "|[2.2216762E-5, 2.7354853E-4, 3.8192928E-5, 6.2340725E-5, 1.7952003E-5, 1.7253387E-4, 6.020507E-6,...|\n", + "|[1.10480236E-4, 2.89734E-4, 4.239379E-5, 1.0727814E-4, 3.047985E-5, 4.7992737E-4, 6.4530495E-6, 3...|\n", + "|[9.6864875E-5, 2.0573521E-4, 7.4498465E-5, 1.1323085E-4, 4.6088306E-5, 2.8680824E-4, 5.604823E-6,...|\n", + "|[7.4198484E-5, 3.2886668E-4, 1.3441108E-4, 1.7755068E-4, 8.469927E-5, 2.2534095E-4, 1.3617541E-5,...|\n", + "|[8.7561886E-5, 2.7312653E-4, 3.5959012E-5, 7.7946424E-5, 2.3565723E-5, 3.6881721E-4, 3.5630535E-6...|\n", + "|[9.743975E-5, 2.7615853E-4, 5.74148E-5, 1.10329434E-4, 3.83045E-5, 3.500394E-4, 6.167429E-6, 4.42...|\n", + "|[6.9320704E-5, 2.53287E-4, 5.0612853E-5, 1.14936556E-4, 3.0210098E-5, 2.7870742E-4, 5.031114E-6, ...|\n", + "|[4.2203726E-5, 2.4911022E-4, 1.2378568E-4, 1.4274308E-4, 7.32259E-5, 1.6058519E-4, 7.9425035E-6, ...|\n", + "|[2.7190901E-5, 3.8381666E-4, 1.2918573E-4, 1.570463E-4, 7.310112E-5, 8.554618E-5, 1.2614603E-5, 1...|\n", + "|[3.0573912E-5, 3.5561546E-4, 1.5945674E-4, 2.1361349E-4, 8.046549E-5, 1.0269262E-4, 1.3862439E-5,...|\n", + "|[3.3117096E-5, 2.8073433E-4, 1.7961214E-4, 2.020287E-4, 1.3662946E-4, 1.0117796E-4, 3.4090703E-5,...|\n", + "|[4.5728237E-5, 2.8880237E-4, 2.3783019E-4, 2.4589908E-4, 1.2160292E-4, 1.3812551E-4, 1.6343482E-5...|\n", + "|[1.2280059E-4, 2.806991E-4, 6.3642765E-5, 1.02471764E-4, 4.351664E-5, 3.9150563E-4, 8.235125E-6, ...|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "only showing top 20 rows\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "predictions_2.show(truncate=100)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "fc511eae", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "predictions_2.write.mode(\"overwrite\").parquet(output_file_path + \"_2\")" + ] + }, + { + "cell_type": "markdown", + "id": "878ca7fb", + "metadata": {}, + "source": [ + "## Using Triton Inference Server\n", + "In this section, we demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. \n", + "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server. \n", + "\n", + "The process looks like this:\n", + "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n", + "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n", + "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n", + "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n", + "\n", + "\"drawing\"" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2605d134-ef75-4d94-9b16-2c6d85f29bef", + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial" + ] + }, + { + "cell_type": "markdown", + "id": "cdded12d", + "metadata": {}, + "source": [ + "Import the utility functions from pytriton_utils.py:" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "a2475d41", + "metadata": {}, + "outputs": [], + "source": [ + "sc.addPyFile(\"pytriton_utils.py\")\n", + "\n", + "from pytriton_utils import (\n", + " use_stage_level_scheduling,\n", + " find_ports,\n", + " start_triton,\n", + " stop_triton\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "1f6701dc", + "metadata": {}, + "source": [ + "Define the Triton Server function:" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "8c8c0744-0558-4dac-bbfe-8bdde4b2af2d", + "metadata": {}, + "outputs": [], + "source": [ + "def triton_server(ports):\n", + " import time\n", + " import signal\n", + " import numpy as np\n", + " import tensorflow as tf\n", + " from tensorflow.keras.applications import ResNet50\n", + " from pytriton.decorators import batch\n", + " from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n", + " from pytriton.triton import Triton, TritonConfig\n", + " from pyspark import TaskContext\n", + "\n", + " print(f\"SERVER: Initializing ResNet on worker {TaskContext.get().partitionId()}.\")\n", + "\n", + " # Enable GPU memory growth\n", + " gpus = tf.config.experimental.list_physical_devices('GPU')\n", + " if gpus:\n", + " try:\n", + " for gpu in gpus:\n", + " tf.config.experimental.set_memory_growth(gpu, True)\n", + " except RuntimeError as e:\n", + " print(e)\n", + " \n", + " model = ResNet50()\n", + " normalization_layer = tf.keras.layers.Rescaling(scale=2./255, offset=-1)\n", + "\n", + " @batch\n", + " def _infer_fn(**inputs):\n", + " images = inputs[\"images\"]\n", + " normalized_images = normalization_layer(images)\n", + " return {\n", + " \"preds\": model.predict(normalized_images),\n", + " }\n", + "\n", + " workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n", + " triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n", + " with Triton(config=triton_conf, workspace=workspace_path) as triton:\n", + " triton.bind(\n", + " model_name=\"ResNet50\",\n", + " infer_func=_infer_fn,\n", + " inputs=[\n", + " Tensor(name=\"images\", dtype=np.float32, shape=(224, 224, 3)),\n", + " ],\n", + " outputs=[\n", + " Tensor(name=\"preds\", dtype=np.float32, shape=(-1,)),\n", + " ],\n", + " config=ModelConfig(\n", + " max_batch_size=100,\n", + " batcher=DynamicBatcher(max_queue_delay_microseconds=5000), # 5ms\n", + " ),\n", + " strict=True,\n", + " )\n", + "\n", + " def _stop_triton(signum, frame):\n", + " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", + " triton.stop()\n", + "\n", + " signal.signal(signal.SIGTERM, _stop_triton)\n", + "\n", + " print(\"SERVER: Serving inference\")\n", + " triton.serve()" + ] + }, + { + "cell_type": "markdown", + "id": "d74f7037", + "metadata": {}, + "source": [ + "#### Start Triton servers" + ] + }, + { + "cell_type": "markdown", + "id": "44a387dc", + "metadata": {}, + "source": [ + "**Specify the number of nodes in the cluster.** \n", + "Following the README, the example standalone cluster uses 1 node. The example Databricks/Dataproc cluster scripts use 4 nodes by default. " + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "132fbfed", + "metadata": {}, + "outputs": [], + "source": [ + "# Change based on cluster setup\n", + "num_nodes = 1 if on_standalone else 4" + ] + }, + { + "cell_type": "markdown", + "id": "f5810c77", + "metadata": {}, + "source": [ + "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "2309a55c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + } + ], + "source": [ + "sc = spark.sparkContext\n", + "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" + ] + }, + { + "cell_type": "markdown", + "id": "533e2a89", + "metadata": {}, + "source": [ + "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "dfc8834a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using ports [7000, 7001, 7002]\n" + ] + } + ], + "source": [ + "model_name = \"ResNet50\"\n", + "ports = find_ports()\n", + "assert len(ports) == 3\n", + "print(f\"Using ports {ports}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "ad24bc52", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 15:> (0 + 1) / 1]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Triton Server PIDs:\n", + " {\n", + " \"cb4ae00-lcedt\": 3077881\n", + "}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", + " ports=ports,\n", + " model_name=model_name)).collectAsMap()\n", + "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + ] + }, + { + "cell_type": "markdown", + "id": "e49ebdbe", + "metadata": {}, + "source": [ + "#### Define client function" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "aa34bebb", + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{ports[0]}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "a5ab49bb", + "metadata": {}, + "outputs": [], + "source": [ + "def triton_fn(url, model_name):\n", + " import numpy as np\n", + " from pytriton.client import ModelClient\n", + "\n", + " print(f\"CLIENT: Connecting to {model_name} at {url}\")\n", + "\n", + " def infer_batch(inputs):\n", + " with ModelClient(url, model_name, inference_timeout_s=240) as client:\n", + " result_data = client.infer_batch(inputs)\n", + " return result_data[\"preds\"]\n", + " \n", + " return infer_batch" + ] + }, + { + "cell_type": "markdown", + "id": "fcd2328e", + "metadata": {}, + "source": [ + "#### Load DataFrame" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "bbfc9009", + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.parquet(data_path)" + ] + }, + { + "cell_type": "markdown", + "id": "8c07365c-0a14-49b3-9bd8-cfb35f48b089", + "metadata": {}, + "source": [ + "#### Run inference" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "9fabcaeb-5a44-42bb-8097-5dbc2d0cee3e", + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial\n", + "\n", + "classify = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", + " input_tensor_shapes=[[224, 224, 3]],\n", + " return_type=ArrayType(FloatType()),\n", + " batch_size=50)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "e595473d-1a5d-46a6-a6ba-89d2ea903de9", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 53 ms, sys: 17.7 ms, total: 70.7 ms\n", + "Wall time: 18.4 s\n" + ] + } + ], + "source": [ + "%%time\n", + "# first pass caches model/fn\n", + "predictions_3 = df.select(classify(struct(\"data\")).alias(\"prediction\"))\n", + "results = predictions_3.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "5f66d468-e0b1-4589-8606-b3848063a823", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 55.5 ms, sys: 20.2 ms, total: 75.7 ms\n", + "Wall time: 12.2 s\n" + ] + } + ], + "source": [ + "%%time\n", + "predictions_3 = df.select(classify(\"data\").alias(\"prediction\"))\n", + "results = predictions_3.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "632c4c3a-fa52-4c3d-b71e-7526286e353a", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 50.6 ms, sys: 18.2 ms, total: 68.8 ms\n", + "Wall time: 12.1 s\n" + ] + } + ], + "source": [ + "%%time\n", + "predictions_3 = df.select(classify(col(\"data\")).alias(\"prediction\"))\n", + "results = predictions_3.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "49870e39", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 21:===========================================> (3 + 1) / 4]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------+\n", + "| prediction|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "|[1.2964063E-4, 2.4653607E-4, 6.7508765E-5, 1.2236452E-4, 5.7346635E-5, 3.9642912E-4, 7.033199E-6,...|\n", + "|[4.4486973E-5, 3.5260408E-4, 4.684452E-5, 8.12069E-5, 3.179397E-5, 1.9187202E-4, 7.887208E-6, 1.3...|\n", + "|[1.059436E-4, 2.2737762E-4, 3.0225037E-5, 6.550149E-5, 2.3658315E-5, 3.7172026E-4, 3.353684E-6, 2...|\n", + "|[2.0393689E-5, 2.2818097E-4, 7.841931E-5, 6.991323E-5, 4.704759E-5, 9.822018E-5, 5.5858673E-6, 2....|\n", + "|[1.13108545E-4, 2.3128217E-4, 5.283139E-5, 1.0866656E-4, 4.0229144E-5, 3.7223354E-4, 5.5677583E-6...|\n", + "|[9.1271184E-5, 2.0681013E-4, 4.5193243E-5, 7.6812066E-5, 3.2361808E-5, 3.399333E-4, 3.8415465E-6,...|\n", + "|[1.0792112E-4, 3.7743401E-4, 7.618583E-5, 1.24259E-4, 4.7426664E-5, 3.3307416E-4, 1.0592865E-5, 9...|\n", + "|[2.2220212E-5, 2.7357432E-4, 3.8200575E-5, 6.235621E-5, 1.7954999E-5, 1.7249273E-4, 6.021971E-6, ...|\n", + "|[1.1044029E-4, 2.8961376E-4, 4.2384647E-5, 1.0728626E-4, 3.0468744E-5, 4.796082E-4, 6.4537376E-6,...|\n", + "|[9.68494E-5, 2.0567125E-4, 7.450887E-5, 1.13256065E-4, 4.609738E-5, 2.8675792E-4, 5.603957E-6, 5....|\n", + "|[7.420906E-5, 3.2883475E-4, 1.3444667E-4, 1.7758778E-4, 8.4717096E-5, 2.2534849E-4, 1.3623082E-5,...|\n", + "|[8.755989E-5, 2.7312606E-4, 3.59614E-5, 7.7967066E-5, 2.3571063E-5, 3.6875304E-4, 3.5629025E-6, 3...|\n", + "|[9.7425895E-5, 2.7611412E-4, 5.74094E-5, 1.1035101E-4, 3.8303257E-5, 3.4981826E-4, 6.167147E-6, 4...|\n", + "|[6.92996E-5, 2.5326438E-4, 5.063317E-5, 1.1494952E-4, 3.0212495E-5, 2.7857954E-4, 5.0324948E-6, 5...|\n", + "|[4.2184765E-5, 2.4904116E-4, 1.237565E-4, 1.4271903E-4, 7.3208634E-5, 1.6054673E-4, 7.938735E-6, ...|\n", + "|[2.719573E-5, 3.8372327E-4, 1.291892E-4, 1.5711001E-4, 7.3108524E-5, 8.553368E-5, 1.2617156E-5, 1...|\n", + "|[3.0565643E-5, 3.55542E-4, 1.5949155E-4, 2.1368133E-4, 8.043127E-5, 1.02662845E-4, 1.3859853E-5, ...|\n", + "|[3.311506E-5, 2.8069926E-4, 1.7956384E-4, 2.0205336E-4, 1.3665091E-4, 1.0115404E-4, 3.409792E-5, ...|\n", + "|[4.573667E-5, 2.888326E-4, 2.3792271E-4, 2.460216E-4, 1.2164583E-4, 1.3814335E-4, 1.6352218E-5, 2...|\n", + "|[1.2279079E-4, 2.8073761E-4, 6.365874E-5, 1.0251792E-4, 4.3527238E-5, 3.914249E-4, 8.236801E-6, 6...|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "only showing top 20 rows\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "predictions_3.show(truncate=100)" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "86cd59f9", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "predictions_3.write.mode(\"overwrite\").parquet(output_file_path + \"_3\")" + ] + }, + { + "cell_type": "markdown", + "id": "4dc06b7e-f750-40b5-9208-a035db11d937", + "metadata": { + "tags": [] + }, + "source": [ + "#### Stop Triton Server on each executor" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "bbfcaa51-3b9f-43ff-a4a8-4b46766115b8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/plain": [ + "[True]" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", + "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "0d88639b-d934-4eb4-ae2f-cc13b9b10456", + "metadata": {}, + "outputs": [], + "source": [ + "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n", + " spark.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df8cc28a-34d7-479c-be7e-9a380d39e25e", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "spark-dl-tf", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/feature_columns/1/model.py b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/feature_columns/1/model.py deleted file mode 100644 index a2fa9635..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/feature_columns/1/model.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import numpy as np -import json -import tensorflow as tf - -# triton_python_backend_utils is available in every Triton Python model. You -# need to use this module to create inference requests and responses. It also -# contains some utility functions for extracting information from model_config -# and converting Triton input/output types to numpy types. -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """Your Python model must use the same class name. Every Python model - that is created must have "TritonPythonModel" as the class name. - """ - - def initialize(self, args): - """`initialize` is called only once when the model is being loaded. - Implementing `initialize` function is optional. This function allows - the model to intialize any state associated with this model. - - Parameters - ---------- - args : dict - Both keys and values are strings. The dictionary keys and values are: - * model_config: A JSON string containing the model configuration - * model_instance_kind: A string containing model instance kind - * model_instance_device_id: A string containing model instance device ID - * model_repository: Model repository path - * model_version: Model version - * model_name: Model name - """ - print("tf: {}".format(tf.__version__)) - gpus = tf.config.list_physical_devices('GPU') - if gpus: - try: - # Currently, memory growth needs to be the same across GPUs - for gpu in gpus: - tf.config.experimental.set_memory_growth(gpu, True) - logical_gpus = tf.config.list_logical_devices('GPU') - print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs") - except RuntimeError as e: - # Memory growth must be set before GPUs have been initialized - print(e) - - self.model = tf.keras.models.load_model("/my_pet_classifier.keras") - - # You must parse model_config. JSON string is not parsed here - self.model_config = model_config = json.loads(args['model_config']) - - # Get output configuration - pred_config = pb_utils.get_output_config_by_name(model_config, "pred") - - # Convert Triton types to numpy types - self.pred_dtype = pb_utils.triton_string_to_numpy(pred_config['data_type']) - - def execute(self, requests): - """`execute` MUST be implemented in every Python model. `execute` - function receives a list of pb_utils.InferenceRequest as the only - argument. This function is called when an inference request is made - for this model. Depending on the batching configuration (e.g. Dynamic - Batching) used, `requests` may contain multiple requests. Every - Python model, must create one pb_utils.InferenceResponse for every - pb_utils.InferenceRequest in `requests`. If there is an error, you can - set the error argument when creating a pb_utils.InferenceResponse - - Parameters - ---------- - requests : list - A list of pb_utils.InferenceRequest - - Returns - ------- - list - A list of pb_utils.InferenceResponse. The length of this list must - be the same as `requests` - """ - - pred_dtype = self.pred_dtype - - responses = [] - - def decode(input_tensor): - return tf.convert_to_tensor([[s[0].decode('utf-8')] for s in input_tensor.as_numpy()]) - - def identity(input_tensor): - return tf.convert_to_tensor(input_tensor.as_numpy()) - - input_transforms = { - "Type": decode, - "Age": identity, - "Breed1": decode, - "Gender": decode, - "Color1": decode, - "Color2": decode, - "MaturitySize": decode, - "FurLength": decode, - "Vaccinated": decode, - "Sterilized": decode, - "Health": decode, - "Fee": identity, - "PhotoAmt": identity - } - - # Every Python backend must iterate over everyone of the requests - # and create a pb_utils.InferenceResponse for each of them. - for request in requests: - # Get input numpy - inputs = {name: transform(pb_utils.get_input_tensor_by_name(request, name)) for name, transform in input_transforms.items()} - - pred = self.model.predict(inputs, verbose=0) - - # Create output tensors. You need pb_utils.Tensor - # objects to create pb_utils.InferenceResponse. - pred_tensor = pb_utils.Tensor("pred", np.squeeze(pred).astype(pred_dtype)) - - # Create InferenceResponse. You can set an error here in case - # there was a problem with handling this inference request. - # Below is an example of how you can set errors in inference - # response: - # - # pb_utils.InferenceResponse( - # output_tensors=..., TritonError("An error occured")) - inference_response = pb_utils.InferenceResponse(output_tensors=[pred_tensor]) - responses.append(inference_response) - - # You should return a list of pb_utils.InferenceResponse. Length - # of this list must match the length of `requests` list. - return responses - - def finalize(self): - """`finalize` is called only once when the model is being unloaded. - Implementing `finalize` function is OPTIONAL. This function allows - the model to perform any necessary clean ups before exit. - """ - print('Cleaning up...') diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/feature_columns/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/feature_columns/config.pbtxt deleted file mode 100644 index 93a7cf04..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/feature_columns/config.pbtxt +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "feature_columns" -backend: "python" -max_batch_size: 8192 - -input [ - { - name: "Type" - data_type: TYPE_STRING - dims: [1] - }, - { - name: "Age" - data_type: TYPE_INT64 - dims: [1] - }, - { - name: "Breed1" - data_type: TYPE_STRING - dims: [1] - }, - { - name: "Gender" - data_type: TYPE_STRING - dims: [1] - }, - { - name: "Color1" - data_type: TYPE_STRING - dims: [1] - }, - { - name: "Color2" - data_type: TYPE_STRING - dims: [1] - }, - { - name: "MaturitySize" - data_type: TYPE_STRING - dims: [1] - }, - { - name: "FurLength" - data_type: TYPE_STRING - dims: [1] - }, - { - name: "Vaccinated" - data_type: TYPE_STRING - dims: [1] - }, - { - name: "Sterilized" - data_type: TYPE_STRING - dims: [1] - }, - { - name: "Health" - data_type: TYPE_STRING - dims: [1] - }, - { - name: "Fee" - data_type: TYPE_FP32 - dims: [1] - }, - { - name: "PhotoAmt" - data_type: TYPE_FP32 - dims: [1] - } -] -output [ - { - name: "pred" - data_type: TYPE_FP32 - dims: [1] - } -] - -instance_group [{ kind: KIND_GPU }] - -parameters: { - key: "EXECUTION_ENV_PATH", - value: {string_value: "$$TRITON_MODEL_DIRECTORY/../tf-gpu.tar.gz"} -} diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/mnist_model/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/mnist_model/config.pbtxt deleted file mode 100644 index cc9172f4..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/mnist_model/config.pbtxt +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -platform: "tensorflow_savedmodel" -max_batch_size: 8192 - diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/resnet50/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/resnet50/config.pbtxt deleted file mode 100644 index cc9172f4..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/resnet50/config.pbtxt +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -platform: "tensorflow_savedmodel" -max_batch_size: 8192 - diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/text_classification/1/model.py b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/text_classification/1/model.py deleted file mode 100644 index 1bdef0b9..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/text_classification/1/model.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import numpy as np -import json -import tensorflow as tf - -# triton_python_backend_utils is available in every Triton Python model. You -# need to use this module to create inference requests and responses. It also -# contains some utility functions for extracting information from model_config -# and converting Triton input/output types to numpy types. -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """Your Python model must use the same class name. Every Python model - that is created must have "TritonPythonModel" as the class name. - """ - - def initialize(self, args): - """`initialize` is called only once when the model is being loaded. - Implementing `initialize` function is optional. This function allows - the model to intialize any state associated with this model. - - Parameters - ---------- - args : dict - Both keys and values are strings. The dictionary keys and values are: - * model_config: A JSON string containing the model configuration - * model_instance_kind: A string containing model instance kind - * model_instance_device_id: A string containing model instance device ID - * model_repository: Model repository path - * model_version: Model version - * model_name: Model name - """ - import re - import string - from tensorflow.keras import layers - - print("tf: {}".format(tf.__version__)) - - def custom_standardization(input_data): - lowercase = tf.strings.lower(input_data) - stripped_html = tf.strings.regex_replace(lowercase, "
", " ") - return tf.strings.regex_replace( - stripped_html, "[%s]" % re.escape(string.punctuation), "" - ) - - max_features = 10000 - sequence_length = 250 - - vectorize_layer = layers.TextVectorization( - standardize=custom_standardization, - max_tokens=max_features, - output_mode="int", - output_sequence_length=sequence_length, - ) - - custom_objects = {"vectorize_layer": vectorize_layer, - "custom_standardization": custom_standardization} - with tf.keras.utils.custom_object_scope(custom_objects): - self.model = tf.keras.models.load_model( - "/text_model_cleaned.keras", compile=False - ) - - # You must parse model_config. JSON string is not parsed here - self.model_config = model_config = json.loads(args['model_config']) - - # Get output configuration - pred_config = pb_utils.get_output_config_by_name(model_config, "pred") - - # Convert Triton types to numpy types - self.pred_dtype = pb_utils.triton_string_to_numpy(pred_config['data_type']) - - def execute(self, requests): - """`execute` MUST be implemented in every Python model. `execute` - function receives a list of pb_utils.InferenceRequest as the only - argument. This function is called when an inference request is made - for this model. Depending on the batching configuration (e.g. Dynamic - Batching) used, `requests` may contain multiple requests. Every - Python model, must create one pb_utils.InferenceResponse for every - pb_utils.InferenceRequest in `requests`. If there is an error, you can - set the error argument when creating a pb_utils.InferenceResponse - - Parameters - ---------- - requests : list - A list of pb_utils.InferenceRequest - - Returns - ------- - list - A list of pb_utils.InferenceResponse. The length of this list must - be the same as `requests` - """ - - pred_dtype = self.pred_dtype - - responses = [] - - # Every Python backend must iterate over everyone of the requests - # and create a pb_utils.InferenceResponse for each of them. - for request in requests: - # Get input numpy - sentence_input = pb_utils.get_input_tensor_by_name(request, "sentence") - sentences = sentence_input.as_numpy() - sentences = np.squeeze(sentences).tolist() - sentences = [s.decode('utf-8') for s in sentences] - sentences = tf.convert_to_tensor(sentences) - - pred = self.model.predict(sentences, verbose=0) - - # Create output tensors. You need pb_utils.Tensor - # objects to create pb_utils.InferenceResponse. - pred_tensor = pb_utils.Tensor("pred", pred.astype(pred_dtype)) - - # Create InferenceResponse. You can set an error here in case - # there was a problem with handling this inference request. - # Below is an example of how you can set errors in inference - # response: - # - # pb_utils.InferenceResponse( - # output_tensors=..., TritonError("An error occured")) - inference_response = pb_utils.InferenceResponse(output_tensors=[pred_tensor]) - responses.append(inference_response) - - # You should return a list of pb_utils.InferenceResponse. Length - # of this list must match the length of `requests` list. - return responses - - def finalize(self): - """`finalize` is called only once when the model is being unloaded. - Implementing `finalize` function is OPTIONAL. This function allows - the model to perform any necessary clean ups before exit. - """ - print('Cleaning up...') diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/text_classification/config.pbtxt b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/text_classification/config.pbtxt deleted file mode 100644 index 44b21bf3..00000000 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/models_config/text_classification/config.pbtxt +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "text_classification" -backend: "python" -max_batch_size: 8192 - -input [ - { - name: "sentence" - data_type: TYPE_STRING - dims: [1] - } -] -output [ - { - name: "pred" - data_type: TYPE_FP32 - dims: [1] - } -] - -instance_group [{ kind: KIND_GPU }] - -parameters: { - key: "EXECUTION_ENV_PATH", - value: {string_value: "$$TRITON_MODEL_DIRECTORY/../tf-gpu.tar.gz"} -} diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/pytriton_utils.py b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/pytriton_utils.py new file mode 120000 index 00000000..330ea4a5 --- /dev/null +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/pytriton_utils.py @@ -0,0 +1 @@ +../pytriton_utils.py \ No newline at end of file diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/text_classification_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/text_classification_tf.ipynb index 63499611..29230de5 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/text_classification_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/text_classification_tf.ipynb @@ -5,9 +5,12 @@ "id": "2cd2accf-5877-4136-a243-7a33a13ce2b4", "metadata": {}, "source": [ + "\n", + "\n", "# Pyspark TensorFlow Inference\n", "\n", - "## Text classification\n", + "### Text Classification\n", + "In this notebook, we demonstrate training a model to perform sentiment analysis, and using the trained model for distributed inference. \n", "Based on: https://www.tensorflow.org/tutorials/keras/text_classification" ] }, @@ -16,9 +19,7 @@ "id": "bc72d0ed", "metadata": {}, "source": [ - "### Using TensorFlow\n", - "Note that cuFFT/cuDNN/cuBLAS registration errors are expected with `tf=2.17.0` and will not affect behavior, as noted in [this issue.](https://github.com/tensorflow/tensorflow/issues/62075) \n", - "This notebook does not demonstrate inference with TensorRT, as [TF-TRT](https://docs.nvidia.com/deeplearning/tensorrt/release-notes/index.html#tensorrt-10) does not yet support `tf=2.17.0`. See the `pytorch` notebooks for TensorRT demos." + "Note that cuFFT/cuDNN/cuBLAS registration errors are expected (as of `tf=2.17.0`) and will not affect behavior, as noted in [this issue.](https://github.com/tensorflow/tensorflow/issues/62075) " ] }, { @@ -31,13 +32,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-24 16:15:43.020721: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2024-10-24 16:15:43.028070: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2024-10-24 16:15:43.035674: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2024-10-24 16:15:43.037910: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2024-10-24 16:15:43.044256: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "2025-01-07 17:55:03.625173: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2025-01-07 17:55:03.632499: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2025-01-07 17:55:03.640392: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2025-01-07 17:55:03.642797: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2025-01-07 17:55:03.648973: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2024-10-24 16:15:43.368732: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" + "2025-01-07 17:55:04.012978: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" ] } ], @@ -46,8 +47,8 @@ "import re\n", "import shutil\n", "import string\n", - "\n", "import matplotlib.pyplot as plt\n", + "\n", "import tensorflow as tf\n", "from tensorflow.keras import layers, losses" ] @@ -67,16 +68,8 @@ } ], "source": [ - "print(tf.__version__)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "57b1d71f", - "metadata": {}, - "outputs": [], - "source": [ + "print(tf.__version__)\n", + "\n", "# Enable GPU memory growth\n", "gpus = tf.config.experimental.list_physical_devices('GPU')\n", "if gpus:\n", @@ -88,131 +81,91 @@ ] }, { - "cell_type": "code", - "execution_count": 10, - "id": "d229c1b6-3967-46b5-9ea8-68f4b42dd211", + "cell_type": "markdown", + "id": "b64bb471", "metadata": {}, - "outputs": [], "source": [ - "import pathlib\n", - "url = \"https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\"\n", - "\n", - "dataset = tf.keras.utils.get_file(\n", - " fname=\"aclImdb\", origin=url, untar=True,\n", - ")\n", - "\n", - "dataset_dir = pathlib.Path(dataset)" + "### Download and explore the dataset" ] }, { "cell_type": "code", - "execution_count": 11, - "id": "bfa5177f", + "execution_count": 3, + "id": "d229c1b6-3967-46b5-9ea8-68f4b42dd211", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/home/rishic/.keras/datasets/aclImdb\n", - "/home/rishic/.keras/datasets/aclImdb\n" - ] - } - ], + "outputs": [], "source": [ - "print(dataset_dir)\n", - "# aclImdb might be created as a directory containing a single directory aclImdb. Check if this is the case:\n", - "if os.path.exists(dataset_dir / \"aclImdb\"):\n", - " dataset_dir = dataset_dir / \"aclImdb\"\n", - "print(dataset_dir)" + "from datasets import load_dataset\n", + "dataset = load_dataset(\"imdb\")" ] }, { "cell_type": "code", - "execution_count": 6, - "id": "1f8038ae-8bc1-46bf-ae4c-6da08886c473", + "execution_count": 4, + "id": "88f9a92e", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['README', 'imdb.vocab', 'test', 'train', 'imdbEr.txt']" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "os.listdir(dataset_dir)" + "# Create directories for our data\n", + "base_dir = \"spark-dl-datasets/imdb\"\n", + "if os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False):\n", + " # For databricks, use the driver disk rather than Workspace (much faster)\n", + " base_dir = \"/local_disk0/\" + base_dir\n", + "\n", + "train_dir = base_dir + \"/train\"\n", + "test_dir = base_dir + \"/test\"" ] }, { "cell_type": "code", - "execution_count": 13, - "id": "12faaa3f-3441-4361-b9eb-4317e8c2c2f7", + "execution_count": 5, + "id": "3f984d5a", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['pos',\n", - " 'labeledBow.feat',\n", - " 'urls_pos.txt',\n", - " 'neg',\n", - " 'urls_unsup.txt',\n", - " 'unsupBow.feat',\n", - " 'urls_neg.txt',\n", - " 'unsup']" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "train_dir = os.path.join(dataset_dir, \"train\")\n", - "test_dir = os.path.join(dataset_dir, \"test\")\n", - "os.listdir(train_dir)" + "# Create directories for positive (1) and negative (0) reviews\n", + "for split in [\"train\", \"test\"]:\n", + " split_dir = os.path.join(base_dir, split)\n", + " pos_dir = split_dir + \"/pos\"\n", + " neg_dir = split_dir + \"/neg\"\n", + "\n", + " os.makedirs(pos_dir, exist_ok=True)\n", + " os.makedirs(neg_dir, exist_ok=True)" ] }, { "cell_type": "code", - "execution_count": 14, - "id": "152cc0cc-65d0-4e17-9ee8-222390df45b5", + "execution_count": 6, + "id": "6cd2328a", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Rachel Griffiths writes and directs this award winning short film. A heartwarming story about coping with grief and cherishing the memory of those we've loved and lost. Although, only 15 minutes long, Griffiths manages to capture so much emotion and truth onto film in the short space of time. Bud Tingwell gives a touching performance as Will, a widower struggling to cope with his wife's death. Will is confronted by the harsh reality of loneliness and helplessness as he proceeds to take care of Ruth's pet cow, Tulip. The film displays the grief and responsibility one feels for those they have loved and lost. Good cinematography, great direction, and superbly acted. It will bring tears to all those who have lost a loved one, and survived.\n" - ] - } - ], + "outputs": [], "source": [ - "sample_file = os.path.join(train_dir, \"pos/1181_9.txt\")\n", - "with open(sample_file) as f:\n", - " print(f.read())" + "def write_reviews_to_files(dataset_split, split_name):\n", + " for idx, example in enumerate(dataset_split):\n", + " label_dir = \"pos\" if example[\"label\"] == 1 else \"neg\"\n", + " dir_path = os.path.join(base_dir, split_name, label_dir)\n", + "\n", + " file_path = dir_path + f\"/review_{idx}.txt\"\n", + " with open(file_path, \"w\", encoding=\"utf-8\") as f:\n", + " f.write(example[\"text\"])\n", + "\n", + "# Write train and test sets\n", + "write_reviews_to_files(dataset[\"train\"], \"train\")\n", + "write_reviews_to_files(dataset[\"test\"], \"test\")" ] }, { - "cell_type": "code", - "execution_count": 15, - "id": "b2277f58-78c8-4a12-bc98-5103e7c81a35", + "cell_type": "markdown", + "id": "b02fde64", "metadata": {}, - "outputs": [], "source": [ - "remove_dir = os.path.join(train_dir, \"unsup\")\n", - "shutil.rmtree(remove_dir)" + "There are 25,000 examples in the training folder, of which we will use 80% (or 20,000) for training, and 5,000 for validation." ] }, { "cell_type": "code", - "execution_count": 17, - "id": "ed83de92-ebb3-4170-b2bf-25265c6a6942", + "execution_count": 7, + "id": "5c357f22", "metadata": {}, "outputs": [ { @@ -227,7 +180,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-24 02:18:45.343343: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 46446 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" + "2025-01-07 17:55:15.035387: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 45468 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 25000 files belonging to 2 classes.\n", + "Using 5000 files for validation.\n", + "Found 25000 files belonging to 2 classes.\n" ] } ], @@ -236,29 +198,50 @@ "seed = 42\n", "\n", "raw_train_ds = tf.keras.utils.text_dataset_from_directory(\n", - " train_dir,\n", + " str(train_dir),\n", " batch_size=batch_size,\n", " validation_split=0.2,\n", " subset=\"training\",\n", " seed=seed,\n", + ")\n", + "\n", + "raw_val_ds = tf.keras.utils.text_dataset_from_directory(\n", + " str(train_dir),\n", + " batch_size=batch_size,\n", + " validation_split=0.2,\n", + " subset=\"validation\",\n", + " seed=seed,\n", + ")\n", + "\n", + "raw_test_ds = tf.keras.utils.text_dataset_from_directory(\n", + " str(test_dir),\n", + " batch_size=batch_size\n", ")" ] }, + { + "cell_type": "markdown", + "id": "02994994", + "metadata": {}, + "source": [ + "We can take a look at a sample of the dataset (note that OUT_OF_RANGE errors are safe to ignore: https://github.com/tensorflow/tensorflow/issues/62963):" + ] + }, { "cell_type": "code", - "execution_count": 11, - "id": "57c30568-daa8-4b2b-b30a-577c984a8af5", + "execution_count": 8, + "id": "1d528a95", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Review b'\"Pandemonium\" is a horror movie spoof that comes off more stupid than funny. Believe me when I tell you, I love comedies. Especially comedy spoofs. \"Airplane\", \"The Naked Gun\" trilogy, \"Blazing Saddles\", \"High Anxiety\", and \"Spaceballs\" are some of my favorite comedies that spoof a particular genre. \"Pandemonium\" is not up there with those films. Most of the scenes in this movie had me sitting there in stunned silence because the movie wasn\\'t all that funny. There are a few laughs in the film, but when you watch a comedy, you expect to laugh a lot more than a few times and that\\'s all this film has going for it. Geez, \"Scream\" had more laughs than this film and that was more of a horror film. How bizarre is that?

*1/2 (out of four)'\n", + "Review b'I was really, really disappointed with this movie. it started really well, and built up some great atmosphere and suspense, but when it finally got round to revealing the \"monster\"...it turned out to be just some psycho with skin problems......again. Whoop-de-do. Yet another nutjob movie...like we don\\'t already have enough of them.

To be fair, the \"creep\" is genuinely unsettling to look at, and the way he moves and the strange sounds he makes are pretty creepy, but I\\'m sick of renting film like this only to discover that the monster is human, albeit a twisted, demented, freakish one. When I saw all the tell-tale rats early on I was hoping for some kind of freaky rat-monster hybrid thing...it was such a let down when the Creep was revealed.

On top of this, some of the stuff in this movie makes no sense. (Spoiler)

Why the hell does the Creep kill the security Guard? Whats the point, apart from sticking a great honking sign up that says \"HI I\\'m A PSYCHO AND I LIVE DOWN HERE!\"? Its stupid, and only seems to happen to prevent Franka Potente\\'s character from getting help.

what the hells he been eating down there? I got the impression he was effectively walled in, and only the unexpected opening into that tunnel section let him loose...so has he been munching rats all that time, and if so why do they hang around him so much? Why is he so damn hard to kill? He\\'s thin, malnourished and not exactly at peak performance...but seems to keep going despite injuries that are equivalent to those that .cripple the non-psycho characters in the film.

The DVD commentary says we are intended to empathise with Creep, but I just find him loathsome. Its an effective enough movie, but it wasted so many opportunities that it makes me sick.'\n", "Label 0\n", - "Review b\"David Mamet is a very interesting and a very un-equal director. His first movie 'House of Games' was the one I liked best, and it set a series of films with characters whose perspective of life changes as they get into complicated situations, and so does the perspective of the viewer.

So is 'Homicide' which from the title tries to set the mind of the viewer to the usual crime drama. The principal characters are two cops, one Jewish and one Irish who deal with a racially charged area. The murder of an old Jewish shop owner who proves to be an ancient veteran of the Israeli Independence war triggers the Jewish identity in the mind and heart of the Jewish detective.

This is were the flaws of the film are the more obvious. The process of awakening is theatrical and hard to believe, the group of Jewish militants is operatic, and the way the detective eventually walks to the final violent confrontation is pathetic. The end of the film itself is Mamet-like smart, but disappoints from a human emotional perspective.

Joe Mantegna and William Macy give strong performances, but the flaws of the story are too evident to be easily compensated.\"\n", + "Review b\"This has the absolute worst performance from Robert Duval who sounds just like William Buckley throughout the entire film. His hammy melodramatic acting takes away from any dramatic interest. I'm not sure if this was deliberate scene stealing or inadvertent but it's the only thing I can recall from a truly forgettable film. This picture should be shown in every amateur acting class of an example of what not to do. Thank God, Duvall went on to bigger and better things and stopped trying to effect a cultured accent. He is a good character actor but that's about it. Klaus is so much better. His performance is muted and noteworthy.\"\n", "Label 0\n", - "Review b'Great documentary about the lives of NY firefighters during the worst terrorist attack of all time.. That reason alone is why this should be a must see collectors item.. What shocked me was not only the attacks, but the\"High Fat Diet\" and physical appearance of some of these firefighters. I think a lot of Doctors would agree with me that,in the physical shape they were in, some of these firefighters would NOT of made it to the 79th floor carrying over 60 lbs of gear. Having said that i now have a greater respect for firefighters and i realize becoming a firefighter is a life altering job. The French have a history of making great documentary\\'s and that is what this is, a Great Documentary.....'\n", + "Review b'A long time ago, in a galaxy far, far away.....There was a boy who was only two years old when the original \"Star Wars\" film was released. He doesn\\'t remember first seeing the movie, but he also doesn\\'t remember life before it. He does remember the first \"Star Wars\" themed gift he got...a shoebox full of action figures from the original set. He was too young to fully appreciate how special that gift would be. But years later, he would get what to this day goes down as one of the best gifts he\\'s ever received: another box full of action figures, ten of the final twelve he needed to complete his collection. It\\'s now legendary in this boy\\'s family how the last action figure he needed, Anakin Skywalker, stopped being produced and carried in stores, and how this boy went for about ten years (until he got into college) trying to track one down and finally bought it from someone on his dorm floor for a bag of beer nuggets (don\\'t ask...it\\'s a Northern Illinois University thing).

I can\\'t review \"Star Wars\" as a movie. It represents absolutely everything good, fun and magical about my childhood. There\\'s no separating it in my mind from Christmases, birthdays, summers and winters growing up. In the winter, my friends and I would build snow forts and pretend we were on Hoth (I was always Han Solo). My friends\\' dad built them a kick-ass tree house, and that served as the Ewok village. They also had a huge pine tree whose bottom branches were high enough to create a sort of cave underneath it, and this made a great spot to pretend we were in Yoda\\'s home. I am unabashedly dorky when it comes to \"Star Wars\" and I think people either just understand that or they don\\'t. I don\\'t get the appeal of \"Lord of the Rings\" or \"Star Trek\" but I understand the rabid flocks of fans that follow them because I am a rabid fan of George Lucas\\'s films.

I feel no need to defend my opinion of these movies as some of the greatest of all time. Every time I put them in the DVD player, I feel like I\\'m eight years old again, when life was simple and the biggest problem I had was figuring out how I was going to track down a figure of Anakin Skywalker.

Grade (for the entire trilogy): A+'\n", "Label 1\n" ] }, @@ -266,7 +249,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-03 17:44:08.132892: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + "2025-01-07 17:55:21.572943: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" ] } ], @@ -278,73 +261,47 @@ ] }, { - "cell_type": "code", - "execution_count": 12, - "id": "1e863eb6-4bd7-4da0-b10d-d951b5ee52bd", + "cell_type": "markdown", + "id": "4bca98b1", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Label 0 corresponds to neg\n", - "Label 1 corresponds to pos\n" - ] - } - ], "source": [ - "print(\"Label 0 corresponds to\", raw_train_ds.class_names[0])\n", - "print(\"Label 1 corresponds to\", raw_train_ds.class_names[1])" + "Notice the reviews contain raw text (with punctuation and occasional HTML tags like \\
\\). We will show how to handle these in the following section." ] }, { "cell_type": "code", - "execution_count": 13, - "id": "1593e2e5-df51-4fbf-b4be-c786e740ddab", + "execution_count": 9, + "id": "f8921ed2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Found 25000 files belonging to 2 classes.\n", - "Using 5000 files for validation.\n" + "Label 0 corresponds to neg\n", + "Label 1 corresponds to pos\n" ] } ], "source": [ - "raw_val_ds = tf.keras.utils.text_dataset_from_directory(\n", - " train_dir,\n", - " batch_size=batch_size,\n", - " validation_split=0.2,\n", - " subset=\"validation\",\n", - " seed=seed,\n", - ")" + "print(\"Label 0 corresponds to\", raw_train_ds.class_names[0])\n", + "print(\"Label 1 corresponds to\", raw_train_ds.class_names[1])" ] }, { - "cell_type": "code", - "execution_count": 14, - "id": "944fd61d-3926-4296-889a-b2a375a1b039", + "cell_type": "markdown", + "id": "f6cf0e47", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found 25000 files belonging to 2 classes.\n" - ] - } - ], "source": [ - "raw_test_ds = tf.keras.utils.text_dataset_from_directory(\n", - " test_dir, batch_size=batch_size\n", - ")" + "### Prepare the dataset for training\n", + "\n", + "Next, we will standardize, tokenize, and vectorize the data using the tf.keras.layers.TextVectorization layer. \n", + "We will write a custom standardization function to remove the HTML." ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 10, "id": "cb141709-fcc1-4cee-bc98-9c89aaba8648", "metadata": {}, "outputs": [], @@ -357,9 +314,17 @@ " )" ] }, + { + "cell_type": "markdown", + "id": "b35e36a2", + "metadata": {}, + "source": [ + "Next, we will create a TextVectorization layer to standardize, tokenize, and vectorize our data." + ] + }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 11, "id": "d4e80ea9-536a-4ebc-8b35-1eca73dbba7d", "metadata": {}, "outputs": [], @@ -375,9 +340,17 @@ ")" ] }, + { + "cell_type": "markdown", + "id": "879fbc3f", + "metadata": {}, + "source": [ + "Next, we will call adapt to fit the state of the preprocessing layer to the dataset. This will cause the model to build an index of strings to integers." + ] + }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 12, "id": "ad1e5d81-7dae-4b08-b520-ca45501b9510", "metadata": {}, "outputs": [ @@ -385,7 +358,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-10-03 17:44:10.225130: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + "2025-01-07 17:55:35.387277: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" ] } ], @@ -395,9 +368,17 @@ "vectorize_layer.adapt(train_text)" ] }, + { + "cell_type": "markdown", + "id": "ad1e5d81-7dae-4b08-b520-ca45501b9510", + "metadata": {}, + "source": [ + "Let's create a function to see the result of using this layer to preprocess some data." + ] + }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 13, "id": "80f243f5-edd3-4e1c-bddc-abc1cc6673ef", "metadata": {}, "outputs": [], @@ -409,7 +390,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 14, "id": "8f37e95c-515c-4edb-a1ee-fc47be5df4b9", "metadata": {}, "outputs": [ @@ -417,32 +398,32 @@ "name": "stdout", "output_type": "stream", "text": [ - "Review tf.Tensor(b'Silent Night, Deadly Night 5 is the very last of the series, and like part 4, it\\'s unrelated to the first three except by title and the fact that it\\'s a Christmas-themed horror flick.

Except to the oblivious, there\\'s some obvious things going on here...Mickey Rooney plays a toymaker named Joe Petto and his creepy son\\'s name is Pino. Ring a bell, anyone? Now, a little boy named Derek heard a knock at the door one evening, and opened it to find a present on the doorstep for him. Even though it said \"don\\'t open till Christmas\", he begins to open it anyway but is stopped by his dad, who scolds him and sends him to bed, and opens the gift himself. Inside is a little red ball that sprouts Santa arms and a head, and proceeds to kill dad. Oops, maybe he should have left well-enough alone. Of course Derek is then traumatized by the incident since he watched it from the stairs, but he doesn\\'t grow up to be some killer Santa, he just stops talking.

There\\'s a mysterious stranger lurking around, who seems very interested in the toys that Joe Petto makes. We even see him buying a bunch when Derek\\'s mom takes him to the store to find a gift for him to bring him out of his trauma. And what exactly is this guy doing? Well, we\\'re not sure but he does seem to be taking these toys apart to see what makes them tick. He does keep his landlord from evicting him by promising him to pay him in cash the next day and presents him with a \"Larry the Larvae\" toy for his kid, but of course \"Larry\" is not a good toy and gets out of the box in the car and of course, well, things aren\\'t pretty.

Anyway, eventually what\\'s going on with Joe Petto and Pino is of course revealed, and as with the old story, Pino is not a \"real boy\". Pino is probably even more agitated and naughty because he suffers from \"Kenitalia\" (a smooth plastic crotch) so that could account for his evil ways. And the identity of the lurking stranger is revealed too, and there\\'s even kind of a happy ending of sorts. Whee.

A step up from part 4, but not much of one. Again, Brian Yuzna is involved, and Screaming Mad George, so some decent special effects, but not enough to make this great. A few leftovers from part 4 are hanging around too, like Clint Howard and Neith Hunter, but that doesn\\'t really make any difference. Anyway, I now have seeing the whole series out of my system. Now if I could get some of it out of my brain. 4 out of 5.', shape=(), dtype=string)\n", + "Review tf.Tensor(b\"To describe this film as garbage is unfair. At least rooting through garbage can be an absorbing hobby. This flick was neither absorbing nor entertaining.

Kevin Bacon can act superbly given the chance, so no doubt had an IRS bill to settle when he agreed to this dire screenplay. The mad scientist story of 'Hollow Man' has been told before, been told better, and been told without resorting to so many ludicrously expensive special effects.

Most of those special effects seem to be built around the transparent anatomical dolls of men, women and dogs you could buy in the early seventies. In the UK they were marketed as 'The Transparent Man (/Woman/Dog)' which is maybe where they got the title for this film.

Clever special effects, dire script, non-existent plot.

\", shape=(), dtype=string)\n", "Label neg\n", "Vectorized review (, )\n" + "array([[ 6, 1507, 11, 19, 14, 1184, 7, 5230, 30, 217, 5821,\n", + " 139, 1184, 68, 26, 33, 6676, 1, 11, 512, 13, 1078,\n", + " 6676, 888, 439, 1727, 5292, 68, 503, 3597, 333, 2, 558,\n", + " 37, 56, 797, 64, 33, 8270, 978, 6, 3956, 51, 27,\n", + " 4531, 6, 11, 3756, 907, 2, 1106, 1660, 63, 5, 3514,\n", + " 134, 43, 74, 566, 155, 74, 566, 122, 3, 74, 566,\n", + " 204, 1, 6, 37, 106, 1, 3152, 307, 293, 88, 5,\n", + " 143, 307, 293, 294, 6, 26, 2250, 183, 2, 7541, 1,\n", + " 4379, 5, 352, 362, 3, 2312, 22, 99, 756, 8, 2,\n", + " 402, 3887, 8, 2, 2142, 34, 65, 1, 14, 2, 7541,\n", + " 134, 1, 61, 7, 271, 111, 34, 182, 2, 409, 15,\n", + " 11, 19, 1066, 307, 293, 3756, 223, 2939, 112, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0]])>, )\n" ] } ], @@ -455,9 +436,17 @@ "print(\"Vectorized review\", vectorize_text(first_review, first_label))" ] }, + { + "cell_type": "markdown", + "id": "680f53bb", + "metadata": {}, + "source": [ + "We can lookup the token (string) that each integer corresponds to by calling .get_vocabulary() on the layer." + ] + }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 15, "id": "60c9208a-39ac-4e6c-a603-61038cdf3d10", "metadata": {}, "outputs": [ @@ -465,8 +454,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "1287 ---> silent\n", - " 313 ---> night\n", + "1287 ---> nowhere\n", + " 313 ---> house\n", "Vocabulary size: 10000\n" ] } @@ -479,7 +468,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 16, "id": "3cf90d4b-8dae-44b2-b32b-80cb0092c430", "metadata": {}, "outputs": [], @@ -489,9 +478,23 @@ "test_ds = raw_test_ds.map(vectorize_text)" ] }, + { + "cell_type": "markdown", + "id": "b3db3f77", + "metadata": {}, + "source": [ + "### Configure the dataset for performance\n", + "\n", + "These are two important methods you should use when loading data to make sure that I/O does not become blocking.\n", + "\n", + "`.cache()` keeps data in memory after it's loaded off disk. This will ensure the dataset does not become a bottleneck while training your model. If your dataset is too large to fit into memory, you can also use this method to create a performant on-disk cache, which is more efficient to read than many small files.\n", + "\n", + "`.prefetch()` overlaps data preprocessing and model execution while training." + ] + }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 17, "id": "115a5aba-8a00-458f-be25-0aae9f55de22", "metadata": {}, "outputs": [], @@ -503,9 +506,17 @@ "test_ds = test_ds.cache().prefetch(buffer_size=AUTOTUNE)" ] }, + { + "cell_type": "markdown", + "id": "0d6d6692", + "metadata": {}, + "source": [ + "### Create the model" + ] + }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 18, "id": "d64f4495-102d-4244-9b42-1ba9976a366e", "metadata": {}, "outputs": [], @@ -515,7 +526,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 19, "id": "3dc95d22-935f-4091-b0ee-da95174eb9a0", "metadata": {}, "outputs": [ @@ -624,7 +635,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 20, "id": "d9059b93-7666-46db-bf15-517c4c205df9", "metadata": {}, "outputs": [], @@ -634,9 +645,17 @@ " metrics=[tf.metrics.BinaryAccuracy(threshold=0.5)])" ] }, + { + "cell_type": "markdown", + "id": "f8b66d33", + "metadata": {}, + "source": [ + "#### Train model" + ] + }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 21, "id": "b1d5959f-1bd8-48da-9815-8239599519b2", "metadata": {}, "outputs": [ @@ -652,49 +671,49 @@ "output_type": "stream", "text": [ "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", - "I0000 00:00:1727977450.773487 1857915 service.cc:146] XLA service 0xac47fb0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n", - "I0000 00:00:1727977450.773523 1857915 service.cc:154] StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n", - "2024-10-03 17:44:10.785495: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", - "2024-10-03 17:44:10.838694: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8907\n" + "I0000 00:00:1736272546.613950 3675377 service.cc:146] XLA service 0x70ca64001f30 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n", + "I0000 00:00:1736272546.613963 3675377 service.cc:154] StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n", + "2025-01-07 17:55:46.627250: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", + "2025-01-07 17:55:46.680543: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8907\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m 88/625\u001b[0m \u001b[32m━━\u001b[0m\u001b[37m━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - binary_accuracy: 0.5075 - loss: 0.6925 " + "\u001b[1m 89/625\u001b[0m \u001b[32m━━\u001b[0m\u001b[37m━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - binary_accuracy: 0.5183 - loss: 0.6920" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "I0000 00:00:1727977451.426198 1857915 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n" + "I0000 00:00:1736272547.264048 3675377 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step - binary_accuracy: 0.5797 - loss: 0.6821 - val_binary_accuracy: 0.7248 - val_loss: 0.6141\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step - binary_accuracy: 0.5778 - loss: 0.6821 - val_binary_accuracy: 0.7072 - val_loss: 0.6182\n", "Epoch 2/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 699us/step - binary_accuracy: 0.7588 - loss: 0.5810 - val_binary_accuracy: 0.8092 - val_loss: 0.4989\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 469us/step - binary_accuracy: 0.7568 - loss: 0.5817 - val_binary_accuracy: 0.8002 - val_loss: 0.4987\n", "Epoch 3/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 477us/step - binary_accuracy: 0.8197 - loss: 0.4674 - val_binary_accuracy: 0.8282 - val_loss: 0.4282\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 453us/step - binary_accuracy: 0.8323 - loss: 0.4656 - val_binary_accuracy: 0.8352 - val_loss: 0.4233\n", "Epoch 4/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 449us/step - binary_accuracy: 0.8514 - loss: 0.3946 - val_binary_accuracy: 0.8402 - val_loss: 0.3870\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 449us/step - binary_accuracy: 0.8535 - loss: 0.3954 - val_binary_accuracy: 0.8520 - val_loss: 0.3789\n", "Epoch 5/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 408us/step - binary_accuracy: 0.8673 - loss: 0.3488 - val_binary_accuracy: 0.8494 - val_loss: 0.3608\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 439us/step - binary_accuracy: 0.8708 - loss: 0.3502 - val_binary_accuracy: 0.8590 - val_loss: 0.3506\n", "Epoch 6/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 406us/step - binary_accuracy: 0.8809 - loss: 0.3174 - val_binary_accuracy: 0.8498 - val_loss: 0.3477\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 433us/step - binary_accuracy: 0.8805 - loss: 0.3183 - val_binary_accuracy: 0.8668 - val_loss: 0.3309\n", "Epoch 7/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 466us/step - binary_accuracy: 0.8899 - loss: 0.2913 - val_binary_accuracy: 0.8554 - val_loss: 0.3325\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 447us/step - binary_accuracy: 0.8908 - loss: 0.2935 - val_binary_accuracy: 0.8698 - val_loss: 0.3170\n", "Epoch 8/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 410us/step - binary_accuracy: 0.8977 - loss: 0.2703 - val_binary_accuracy: 0.8580 - val_loss: 0.3232\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 451us/step - binary_accuracy: 0.9002 - loss: 0.2714 - val_binary_accuracy: 0.8714 - val_loss: 0.3088\n", "Epoch 9/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 397us/step - binary_accuracy: 0.9057 - loss: 0.2539 - val_binary_accuracy: 0.8580 - val_loss: 0.3208\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 427us/step - binary_accuracy: 0.9057 - loss: 0.2557 - val_binary_accuracy: 0.8712 - val_loss: 0.3032\n", "Epoch 10/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 784us/step - binary_accuracy: 0.9084 - loss: 0.2405 - val_binary_accuracy: 0.8638 - val_loss: 0.3121\n" + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 404us/step - binary_accuracy: 0.9113 - loss: 0.2414 - val_binary_accuracy: 0.8766 - val_loss: 0.2969\n" ] } ], @@ -706,9 +725,17 @@ " epochs=epochs)" ] }, + { + "cell_type": "markdown", + "id": "4c8d8f2a", + "metadata": {}, + "source": [ + "#### Evaluate the model" + ] + }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 22, "id": "656afe07-354f-4ff2-8e3e-d02bad6c5958", "metadata": {}, "outputs": [ @@ -716,16 +743,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m 1/782\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m21s\u001b[0m 28ms/step - binary_accuracy: 0.9062 - loss: 0.2768" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 705us/step - binary_accuracy: 0.8542 - loss: 0.3343\n", - "Loss: 0.33130890130996704\n", - "Accuracy: 0.856440007686615\n" + "\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 747us/step - binary_accuracy: 0.8727 - loss: 0.3141\n", + "Loss: 0.3163740932941437\n", + "Accuracy: 0.8708400130271912\n" ] } ], @@ -736,9 +756,17 @@ "print(\"Accuracy: \", accuracy)" ] }, + { + "cell_type": "markdown", + "id": "b2a307ce", + "metadata": {}, + "source": [ + "Create a plot of accuracy and loss over time:" + ] + }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 23, "id": "a01d0f13-d0b8-4d78-9ddc-ede5ed402446", "metadata": {}, "outputs": [ @@ -748,7 +776,7 @@ "dict_keys(['binary_accuracy', 'loss', 'val_binary_accuracy', 'val_loss'])" ] }, - "execution_count": 28, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -760,13 +788,13 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 24, "id": "1f7484c3-3cdf-46d5-b95d-80316f0e6240", "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABVv0lEQVR4nO3dd3gU1f7H8fcmIQ2S0JNAQkLvTUAEpKgoWFCkGBSlqHhFqggClyLlAgqooAiIVwFBEcWAIB0EpSlcKSJiAKVJRyShBtjM74/5ZcmSENIn2f28nmcfZmdnZ75LovvhnDPn2AzDMBARERFxER5WFyAiIiKSlRRuRERExKUo3IiIiIhLUbgRERERl6JwIyIiIi5F4UZERERcisKNiIiIuBSFGxEREXEpCjciIiLiUhRuRCzQpUsXIiMjM/TeESNGYLPZsragXObQoUPYbDZmzZqVo9ddv349NpuN9evXO/al9WeVXTVHRkbSpUuXLD1nWsyaNQubzcahQ4dy/NoimaVwI5KEzWZL0yPpl59IZm3evJkRI0Zw/vx5q0sRcQleVhcgkpvMmTPH6fmnn37K6tWrk+2vXLlypq7z0UcfkZCQkKH3Dh06lEGDBmXq+pJ2mflZpdXmzZsZOXIkXbp0oWDBgk6vxcTE4OGhf4eKpIfCjUgSzz77rNPzH3/8kdWrVyfbf6vLly/j7++f5uvky5cvQ/UBeHl54eWl/3RzSmZ+VlnBx8fH0uuL5EX654BIOjVr1oxq1arx888/06RJE/z9/fn3v/8NwDfffMOjjz5KiRIl8PHxoWzZsowePRq73e50jlvHcSSO15g4cSIzZsygbNmy+Pj4UK9ePbZt2+b03pTG3NhsNnr27MmiRYuoVq0aPj4+VK1alRUrViSrf/369dStWxdfX1/Kli3Lhx9+mOZxPBs2bKB9+/aUKlUKHx8fwsPDefXVV7ly5Uqyz1egQAGOHTtG69atKVCgAMWKFaN///7J/i7Onz9Ply5dCAoKomDBgnTu3DlN3TP/+9//sNlszJ49O9lrK1euxGaz8e233wJw+PBhXnnlFSpWrIifnx9FihShffv2aRpPktKYm7TW/Msvv9ClSxfKlCmDr68vISEhPP/88/z999+OY0aMGMGAAQMAKF26tKPrM7G2lMbc/Pnnn7Rv357ChQvj7+/PPffcw9KlS52OSRw/9OWXXzJmzBjCwsLw9fXlgQce4MCBA3f83LczdepUqlatio+PDyVKlKBHjx7JPvv+/ftp27YtISEh+Pr6EhYWRocOHYiNjXUcs3r1au69914KFixIgQIFqFixouO/I5HM0j//RDLg77//5uGHH6ZDhw48++yzBAcHA+YgzAIFCtCvXz8KFCjAd999x/Dhw4mLi2PChAl3PO/nn3/OhQsX+Ne//oXNZmP8+PG0adOGP//8844tCBs3biQ6OppXXnmFgIAA3nvvPdq2bcuRI0coUqQIADt27KBly5aEhoYycuRI7HY7o0aNolixYmn63F999RWXL1+me/fuFClShK1bt/L+++/z119/8dVXXzkda7fbadGiBfXr12fixImsWbOGt99+m7Jly9K9e3cADMPgiSeeYOPGjbz88stUrlyZhQsX0rlz5zvWUrduXcqUKcOXX36Z7Pj58+dTqFAhWrRoAcC2bdvYvHkzHTp0ICwsjEOHDjFt2jSaNWvGb7/9lq5Wt/TUvHr1av7880+6du1KSEgIe/bsYcaMGezZs4cff/wRm81GmzZt2LdvH/PmzePdd9+laNGiALf9mZw6dYqGDRty+fJlevfuTZEiRZg9ezaPP/44CxYs4Mknn3Q6/s0338TDw4P+/fsTGxvL+PHj6dixIz/99FOaP3OiESNGMHLkSJo3b0737t2JiYlh2rRpbNu2jU2bNpEvXz6uXbtGixYtiI+Pp1evXoSEhHDs2DG+/fZbzp8/T1BQEHv27OGxxx6jRo0ajBo1Ch8fHw4cOMCmTZvSXZNIigwRua0ePXoYt/5n0rRpUwMwpk+fnuz4y5cvJ9v3r3/9y/D39zeuXr3q2Ne5c2cjIiLC8fzgwYMGYBQpUsQ4d+6cY/8333xjAMaSJUsc+954441kNQGGt7e3ceDAAce+Xbt2GYDx/vvvO/a1atXK8Pf3N44dO+bYt3//fsPLyyvZOVOS0ucbN26cYbPZjMOHDzt9PsAYNWqU07G1a9c26tSp43i+aNEiAzDGjx/v2Hfjxg2jcePGBmDMnDkz1XoGDx5s5MuXz+nvLD4+3ihYsKDx/PPPp1r3li1bDMD49NNPHfvWrVtnAMa6deucPkvSn1V6ak7puvPmzTMA44cffnDsmzBhggEYBw8eTHZ8RESE0blzZ8fzvn37GoCxYcMGx74LFy4YpUuXNiIjIw273e70WSpXrmzEx8c7jp08ebIBGLt37052raRmzpzpVNPp06cNb29v46GHHnJcwzAMY8qUKQZgfPLJJ4ZhGMaOHTsMwPjqq69ue+53333XAIwzZ86kWoNIRqlbSiQDfHx86Nq1a7L9fn5+ju0LFy5w9uxZGjduzOXLl/n999/veN6oqCgKFSrkeN64cWPA7Ia4k+bNm1O2bFnH8xo1ahAYGOh4r91uZ82aNbRu3ZoSJUo4jitXrhwPP/zwHc8Pzp/v0qVLnD17loYNG2IYBjt27Eh2/Msvv+z0vHHjxk6fZdmyZXh5eTlacgA8PT3p1atXmuqJiori+vXrREdHO/atWrWK8+fPExUVlWLd169f5++//6ZcuXIULFiQ7du3p+laGak56XWvXr3K2bNnueeeewDSfd2k17/77ru59957HfsKFCjASy+9xKFDh/jtt9+cju/atSve3t6O5+n5nUpqzZo1XLt2jb59+zoNcO7WrRuBgYGObrGgoCDA7Bq8fPlyiudKHDT9zTffZPtgbXFPCjciGVCyZEmnL4xEe/bs4cknnyQoKIjAwECKFSvmGIycdLzB7ZQqVcrpeWLQ+eeff9L93sT3J7739OnTXLlyhXLlyiU7LqV9KTly5AhdunShcOHCjnE0TZs2BZJ/Pl9f32RdK0nrAXMsTGhoKAUKFHA6rmLFimmqp2bNmlSqVIn58+c79s2fP5+iRYty//33O/ZduXKF4cOHEx4ejo+PD0WLFqVYsWKcP38+TT+XpNJT87lz5+jTpw/BwcH4+flRrFgxSpcuDaTt9+F210/pWol38B0+fNhpf2Z+p269LiT/nN7e3pQpU8bxeunSpenXrx///e9/KVq0KC1atOCDDz5w+rxRUVE0atSIF198keDgYDp06MCXX36poCNZRmNuRDIg6b/IE50/f56mTZsSGBjIqFGjKFu2LL6+vmzfvp2BAwem6X/cnp6eKe43DCNb35sWdrudBx98kHPnzjFw4EAqVapE/vz5OXbsGF26dEn2+W5XT1aLiopizJgxnD17loCAABYvXszTTz/tdEdZr169mDlzJn379qVBgwYEBQVhs9no0KFDtn6hPvXUU2zevJkBAwZQq1YtChQoQEJCAi1btsyxL/Ls/r1Iydtvv02XLl345ptvWLVqFb1792bcuHH8+OOPhIWF4efnxw8//MC6detYunQpK1asYP78+dx///2sWrUqx353xHUp3IhkkfXr1/P3338THR1NkyZNHPsPHjxoYVU3FS9eHF9f3xTvlEnL3TO7d+9m3759zJ49m06dOjn2r169OsM1RUREsHbtWi5evOjUEhITE5Pmc0RFRTFy5Ei+/vprgoODiYuLo0OHDk7HLFiwgM6dO/P222879l29ejVDk+alteZ//vmHtWvXMnLkSIYPH+7Yv3///mTnTM+M0xERESn+/SR2e0ZERKT5XOmReN6YmBjKlCnj2H/t2jUOHjxI8+bNnY6vXr061atXZ+jQoWzevJlGjRoxffp0/vOf/wDg4eHBAw88wAMPPMA777zD2LFjGTJkCOvWrUt2LpH0UreUSBZJ/Ndm0n8RX7t2jalTp1pVkhNPT0+aN2/OokWLOH78uGP/gQMHWL58eZreD86fzzAMJk+enOGaHnnkEW7cuMG0adMc++x2O++//36az1G5cmWqV6/O/PnzmT9/PqGhoU7hMrH2W1sq3n///WS3pWdlzSn9fQFMmjQp2Tnz588PkKaw9cgjj7B161a2bNni2Hfp0iVmzJhBZGQkVapUSetHSZfmzZvj7e3Ne++95/SZPv74Y2JjY3n00UcBiIuL48aNG07vrV69Oh4eHsTHxwNmd92tatWqBeA4RiQz1HIjkkUaNmxIoUKF6Ny5M71798ZmszFnzpxsbf5PrxEjRrBq1SoaNWpE9+7dsdvtTJkyhWrVqrFz585U31upUiXKli1L//79OXbsGIGBgXz99dfpHruRVKtWrWjUqBGDBg3i0KFDVKlShejo6HSPR4mKimL48OH4+vrywgsvJJvR97HHHmPOnDkEBQVRpUoVtmzZwpo1axy3yGdHzYGBgTRp0oTx48dz/fp1SpYsyapVq1JsyatTpw4AQ4YMoUOHDuTLl49WrVo5Qk9SgwYNYt68eTz88MP07t2bwoULM3v2bA4ePMjXX3+dbbMZFytWjMGDBzNy5EhatmzJ448/TkxMDFOnTqVevXqOsWXfffcdPXv2pH379lSoUIEbN24wZ84cPD09adu2LQCjRo3ihx9+4NFHHyUiIoLTp08zdepUwsLCnAZKi2SUwo1IFilSpAjffvstr732GkOHDqVQoUI8++yzPPDAA475VqxWp04dli9fTv/+/Rk2bBjh4eGMGjWKvXv33vFurnz58rFkyRLH+AlfX1+efPJJevbsSc2aNTNUj4eHB4sXL6Zv377MnTsXm83G448/zttvv03t2rXTfJ6oqCiGDh3K5cuXne6SSjR58mQ8PT357LPPuHr1Ko0aNWLNmjUZ+rmkp+bPP/+cXr168cEHH2AYBg899BDLly93ulsNoF69eowePZrp06ezYsUKEhISOHjwYIrhJjg4mM2bNzNw4EDef/99rl69So0aNViyZImj9SS7jBgxgmLFijFlyhReffVVChcuzEsvvcTYsWMd8zDVrFmTFi1asGTJEo4dO4a/vz81a9Zk+fLljjvFHn/8cQ4dOsQnn3zC2bNnKVq0KE2bNmXkyJGOu61EMsNm5KZ/VoqIJVq3bs2ePXtSHA8iIpLXaMyNiJu5damE/fv3s2zZMpo1a2ZNQSIiWUwtNyJuJjQ01LHe0eHDh5k2bRrx8fHs2LGD8uXLW12eiEimacyNiJtp2bIl8+bN4+TJk/j4+NCgQQPGjh2rYCMiLkMtNyIiIuJSNOZGREREXIrCjYiIiLgUtxtzk5CQwPHjxwkICEjXlOciIiJiHcMwuHDhAiVKlLjjZJVuF26OHz9OeHi41WWIiIhIBhw9epSwsLBUj3G7cBMQEACYfzmBgYEWVyMiIiJpERcXR3h4uON7PDVuF24Su6ICAwMVbkRERPKYtAwp0YBiERERcSkKNyIiIuJSFG5ERETEpbjdmBsREcladrud69evW12GuABvb+873uadFgo3IiKSIYZhcPLkSc6fP291KeIiPDw8KF26NN7e3pk6j8KNiIhkSGKwKV68OP7+/poYVTIlcZLdEydOUKpUqUz9PinciIhIutntdkewKVKkiNXliIsoVqwYx48f58aNG+TLly/D59GAYhERSbfEMTb+/v4WVyKuJLE7ym63Z+o8CjciIpJh6oqSrJRVv0/qlsoidjts2AAnTkBoKDRuDJ6eVlclIiLiftRykwWioyEyEu67D555xvwzMtLcLyIiri8yMpJJkyal+fj169djs9my/U6zWbNmUbBgwWy9Rm6kcJNJ0dHQrh389Zfz/mPHzP0KOCIiqbPbYf16mDfP/DOTwy1SZbPZUn2MGDEiQ+fdtm0bL730UpqPb9iwISdOnCAoKChD15PUqVsqE+x26NMHDCP5a4YBNhv07QtPPKEuKhGRlERHm/8fTfoPxLAwmDwZ2rTJ+uudOHHCsT1//nyGDx9OTEyMY1+BAgUc24ZhYLfb8fK681dlsWLF0lWHt7c3ISEh6XqPpJ1abjJhw4bkLTZJGQYcPWoeJyIizqxo+Q4JCXE8goKCsNlsjue///47AQEBLF++nDp16uDj48PGjRv5448/eOKJJwgODqZAgQLUq1ePNWvWOJ331m4pm83Gf//7X5588kn8/f0pX748ixcvdrx+a7dUYvfRypUrqVy5MgUKFKBly5ZOYezGjRv07t2bggULUqRIEQYOHEjnzp1p3bp1uv4Opk2bRtmyZfH29qZixYrMmTPH8ZphGIwYMYJSpUrh4+NDiRIl6N27t+P1qVOnUr58eXx9fQkODqZdu3bpunZOUbjJhCS/c1lynIiIu7hTyzeYLd/Z2UV1O4MGDeLNN99k79691KhRg4sXL/LII4+wdu1aduzYQcuWLWnVqhVHjhxJ9TwjR47kqaee4pdffuGRRx6hY8eOnDt37rbHX758mYkTJzJnzhx++OEHjhw5Qv/+/R2vv/XWW3z22WfMnDmTTZs2ERcXx6JFi9L12RYuXEifPn147bXX+PXXX/nXv/5F165dWbduHQBff/017777Lh9++CH79+9n0aJFVK9eHYD//e9/9O7dm1GjRhETE8OKFSto0qRJuq6fYww3ExsbawBGbGxsps+1bp1hmP8Zpv5Yty7TlxIRyVWuXLli/Pbbb8aVK1cy9P7c8P/PmTNnGkFBQUlqWmcAxqJFi+743qpVqxrvv/++43lERITx7rvvOp4DxtChQx3PL168aADG8uXLna71zz//OGoBjAMHDjje88EHHxjBwcGO58HBwcaECRMcz2/cuGGUKlXKeOKJJ9L8GRs2bGh069bN6Zj27dsbjzzyiGEYhvH2228bFSpUMK5du5bsXF9//bURGBhoxMXF3fZ6mZXa71V6vr/VcpMJjRubfcO3uy3fZoPwcPM4ERG5KTe3fNetW9fp+cWLF+nfvz+VK1emYMGCFChQgL17996x5aZGjRqO7fz58xMYGMjp06dve7y/vz9ly5Z1PA8NDXUcHxsby6lTp7j77rsdr3t6elKnTp10fba9e/fSqFEjp32NGjVi7969ALRv354rV65QpkwZunXrxsKFC7lx4wYADz74IBEREZQpU4bnnnuOzz77jMuXL6fr+jlF4SYTPD3NQW+QPOAkPp80SYOJRURuFRqatcdlpfz58zs979+/PwsXLmTs2LFs2LCBnTt3Ur16da5du5bqeW5dPsBms5GQkJCu442U+u2yUXh4ODExMUydOhU/Pz9eeeUVmjRpwvXr1wkICGD79u3MmzeP0NBQhg8fTs2aNXPlwqkKN5nUpg0sWAAlSzrvDwsz92fHaH8RkbwuL7V8b9q0iS5duvDkk09SvXp1QkJCOHToUI7WEBQURHBwMNu2bXPss9vtbN++PV3nqVy5Mps2bXLat2nTJqpUqeJ47ufnR6tWrXjvvfdYv349W7ZsYffu3QB4eXnRvHlzxo8fzy+//MKhQ4f47rvvMvHJsoduBc8CbdqYt3trhmIRkbRJbPlu184MMkkbKHJby3f58uWJjo6mVatW2Gw2hg0blmoLTHbp1asX48aNo1y5clSqVIn333+ff/75J11LFgwYMICnnnqK2rVr07x5c5YsWUJ0dLTj7q9Zs2Zht9upX78+/v7+zJ07Fz8/PyIiIvj222/5888/adKkCYUKFWLZsmUkJCRQsWLF7PrIGaZwk0U8PaFZM6urEBHJOxJbvlOa52bSpNzT8v3OO+/w/PPP07BhQ4oWLcrAgQOJi4vL8ToGDhzIyZMn6dSpE56enrz00ku0aNECz3QkwNatWzN58mQmTpxInz59KF26NDNnzqTZ/3+BFSxYkDfffJN+/fpht9upXr06S5YsoUiRIhQsWJDo6GhGjBjB1atXKV++PPPmzaNq1arZ9IkzzmbkdIeexeLi4ggKCiI2NpbAwECryxERyZOuXr3KwYMHKV26NL6+vpk6l9bmy5iEhAQqV67MU089xejRo60uJ0uk9nuVnu9vtdyIiIil1PKdNocPH2bVqlU0bdqU+Ph4pkyZwsGDB3nmmWesLi3X0YBiERGRPMDDw4NZs2ZRr149GjVqxO7du1mzZg2VK1e2urRcRy03IiIieUB4eHiyO50kZWq5EREREZeicCMiIiIuReFGREREXIrCjYiIiLgUhRsRERFxKQo3IiIi4lIUbkRERNKpWbNm9O3b1/E8MjKSSZMmpfoem83GokWLMn3trDpPakaMGEGtWrWy9RrZSeFGRETcRqtWrWjZsmWKr23YsAGbzcYvv/yS7vNu27aNl156KbPlObldwDhx4gQPP/xwll7L1SjciIiI23jhhRdYvXo1fyVdqfP/zZw5k7p161KjRo10n7dYsWL4+/tnRYl3FBISgo+PT45cK69SuBEREbfx2GOPUaxYMWbNmuW0/+LFi3z11Ve88MIL/P333zz99NOULFkSf39/qlevzrx581I9763dUvv376dJkyb4+vpSpUoVVq9enew9AwcOpEKFCvj7+1OmTBmGDRvG9evXAZg1axYjR45k165d2Gw2bDabo+Zbu6V2797N/fffj5+fH0WKFOGll17i4sWLjte7dOlC69atmThxIqGhoRQpUoQePXo4rpUWCQkJjBo1irCwMHx8fKhVqxYrVqxwvH7t2jV69uxJaGgovr6+REREMG7cOAAMw2DEiBGUKlUKHx8fSpQoQe/evdN87YzQ8gsiIpIlDAMuX7bm2v7+YLPd+TgvLy86derErFmzGDJkCLb/f9NXX32F3W7n6aef5uLFi9SpU4eBAwcSGBjI0qVLee655yhbtix33333Ha+RkJBAmzZtCA4O5qeffiI2NtZpfE6igIAAZs2aRYkSJdi9ezfdunUjICCA119/naioKH799VdWrFjBmjVrAAgKCkp2jkuXLtGiRQsaNGjAtm3bOH36NC+++CI9e/Z0CnDr1q0jNDSUdevWceDAAaKioqhVqxbdunW7818aMHnyZN5++20+/PBDateuzSeffMLjjz/Onj17KF++PO+99x6LFy/myy+/pFSpUhw9epSjR48C8PXXX/Puu+/yxRdfULVqVU6ePMmuXbvSdN0MM9xMbGysARixsbFWlyIikmdduXLF+O2334wrV6449l28aBhmxMn5x8WLaa997969BmCsW7fOsa9x48bGs88+e9v3PProo8Zrr73meN60aVOjT58+jucRERHGu+++axiGYaxcudLw8vIyjh075nh9+fLlBmAsXLjwtteYMGGCUadOHcfzN954w6hZs2ay45KeZ8aMGUahQoWMi0n+ApYuXWp4eHgYJ0+eNAzDMDp37mxEREQYN27ccBzTvn17Iyoq6ra13HrtEiVKGGPGjHE6pl69esYrr7xiGIZh9OrVy7j//vuNhISEZOd6++23jQoVKhjXrl277fUSpfR7lSg939/qlhIREbdSqVIlGjZsyCeffALAgQMH2LBhAy+88AIAdrud0aNHU716dQoXLkyBAgVYuXIlR44cSdP59+7dS3h4OCVKlHDsa9CgQbLj5s+fT6NGjQgJCaFAgQIMHTo0zddIeq2aNWuSP39+x75GjRqRkJBATEyMY1/VqlXx9PR0PA8NDeX06dNpukZcXBzHjx+nUaNGTvsbNWrE3r17AbPra+fOnVSsWJHevXuzatUqx3Ht27fnypUrlClThm7durFw4UJu3LiRrs+ZXgo3IiKSJfz94eJFax7pHcv7wgsv8PXXX3PhwgVmzpxJ2bJladq0KQATJkxg8uTJDBw4kHXr1rFz505atGjBtWvXsuzvasuWLXTs2JFHHnmEb7/9lh07djBkyJAsvUZS+fLlc3pus9lISEjIsvPfddddHDx4kNGjR3PlyhWeeuop2rVrB5irmcfExDB16lT8/Px45ZVXaNKkSbrG/KSXxtyIiEiWsNkgSQNCrvbUU0/Rp08fPv/8cz799FO6d+/uGH+zadMmnnjiCZ599lnAHEOzb98+qlSpkqZzV65cmaNHj3LixAlCQ0MB+PHHH52O2bx5MxEREQwZMsSx7/Dhw07HeHt7Y7fb73itWbNmcenSJUfrzaZNm/Dw8KBixYppqvdOAgMDKVGiBJs2bXIEwMTrJB2DFBgYSFRUFFFRUbRr146WLVty7tw5ChcujJ+fH61ataJVq1b06NGDSpUqsXv3bu66664sqfFWCjciIuJ2ChQoQFRUFIMHDyYuLo4uXbo4XitfvjwLFixg8+bNFCpUiHfeeYdTp06lOdw0b96cChUq0LlzZyZMmEBcXJxTiEm8xpEjR/jiiy+oV68eS5cuZeHChU7HREZGcvDgQXbu3ElYWBgBAQHJbgHv2LEjb7zxBp07d2bEiBGcOXOGXr168dxzzxEcHJyxv5wUDBgwgDfeeIOyZctSq1YtZs6cyc6dO/nss88AeOeddwgNDaV27dp4eHjw1VdfERISQsGCBZk1axZ2u5369evj7+/P3Llz8fPzIyIiIsvqu5W6pURExC298MIL/PPPP7Ro0cJpfMzQoUO56667aNGiBc2aNSMkJITWrVun+bweHh4sXLiQK1eucPfdd/Piiy8yZswYp2Mef/xxXn31VXr27EmtWrXYvHkzw4YNczqmbdu2tGzZkvvuu49ixYqleDu6v78/K1eu5Ny5c9SrV4927drxwAMPMGXKlPT9ZdxB79696devH6+99hrVq1dnxYoVLF68mPLlywPmnV/jx4+nbt261KtXj0OHDrFs2TI8PDwoWLAgH330EY0aNaJGjRqsWbOGJUuWUKRIkSytMSmbYRhGtp09F4qLiyMoKIjY2FgCAwOtLkdEJE+6evUqBw8epHTp0vj6+lpdjriI1H6v0vP9rZYbERERcSkKNyIiIuJSFG5ERETEpSjciIiIiEtRuBERkQxzs3tSJJtl1e+Two2IiKRb4oy3l61aKVNcUuIMzUmXisgITeInIiLp5unpScGCBR3rE/n7+ztm+BXJiISEBM6cOYO/vz9eXpmLJwo3IiKSISEhIQBpXoBR5E48PDwoVapUpoOywk0Wu3EDMhk4RUTyBJvNRmhoKMWLF8/WRRDFfXh7e+PhkfkRM/oaziIxMfDaa1C2LEyebHU1IiI5x9PTM9NjJESykgYUZ5GjR2HpUpg6FQ4csLoaERER96Vwk0WaN4eWLc1uqX//2+pqRERE3JfCTRYaPx48POCrr+DHH62uRkRExD0p3GSh6tWhSxdzu39/0NxWIiIiOU/hJouNGgV+frBpEyxaZHU1IiIi7kfhJouVLGneNQUwcCDo7kgREZGcpXCTDQYMgGLFYP9+mDHD6mpERETci+Xh5oMPPiAyMhJfX1/q16/P1q1bUz3+/Pnz9OjRg9DQUHx8fKhQoQLLli3LoWrTJjAQRowwt0eOhLg4S8sRERFxK5aGm/nz59OvXz/eeOMNtm/fTs2aNWnRosVtp/K+du0aDz74IIcOHWLBggXExMTw0UcfUbJkyRyu/M66dYMKFeDMGfMuKhEREckZNsPC9err169PvXr1mDJlCmAumhUeHk6vXr0YNGhQsuOnT5/OhAkT+P333x0r0qZXXFwcQUFBxMbGEhgYmKn672TRInjySXOA8f795ngcERERSb/0fH9b1nJz7do1fv75Z5o3b36zGA8PmjdvzpYtW1J8z+LFi2nQoAE9evQgODiYatWqMXbsWOx2+22vEx8fT1xcnNMjpzzxBNx7L1y5AsOG5dhlRURE3Jpl4ebs2bPY7XaCg4Od9gcHB3Py5MkU3/Pnn3+yYMEC7HY7y5YtY9iwYbz99tv85z//ue11xo0bR1BQkOMRHh6epZ8jNTYbTJxobs+aBb/8kmOXFhERcVuWDyhOj4SEBIoXL86MGTOoU6cOUVFRDBkyhOnTp9/2PYMHDyY2NtbxOHr0aA5WDPXrQ/v25oR+r7+eo5cWERFxS5aFm6JFi+Lp6cmpU6ec9p86dYqQkJAU3xMaGkqFChWcVp+tXLkyJ0+e5Nq1aym+x8fHh8DAQKdHThs3DvLlg5UrYfXqHL+8iIiIW7Es3Hh7e1OnTh3Wrl3r2JeQkMDatWtp0KBBiu9p1KgRBw4cICEhwbFv3759hIaG4u3tne01Z1TZsvDKK+b2gAGQyhAhERERySRLu6X69evHRx99xOzZs9m7dy/du3fn0qVLdO3aFYBOnToxePBgx/Hdu3fn3Llz9OnTh3379rF06VLGjh1Ljx49rPoIaTZsGAQFwa5d8NlnVlcjIiLiurysvHhUVBRnzpxh+PDhnDx5klq1arFixQrHIOMjR47g4XEzf4WHh7Ny5UpeffVVatSoQcmSJenTpw8DBw606iOkWZEi8O9/m0syDB1qjsPx87O6KhEREddj6Tw3VsjJeW5udfUqVKwIR46Y43BSmMpHREREUpAn5rlxR76+MGaMuT1unDl7sYiIiGQthZsc9swzULu2ud7U6NFWVyMiIuJ6FG5ymIcHTJhgbk+bZi7LICIiIllH4cYCDzwADz8MN25AkpvBREREJAso3Fhk/HizFefrr+E2S2mJiIhIBijcWKRaNfj/6Xzo399cnkFEREQyT+HGQqNGgb8/bN4MCxdaXY2IiIhrULixUIkS8Npr5vagQXD9urX1iIiIuAKFG4sNGADFi5t3TX34odXViIiI5H0KNxYLCIARI8ztkSMhNtbSckRERPI8hZtc4MUXzWUZzp6Ft96yuhoREZG8TeEmF8iX72aoefddOHrU2npERETyMoWbXOLxx6FxY3NxzeHDra5GREQk71K4ySVsNpg40dyePRt27bK2HhERkbxK4SYXuftuiIoyJ/R7/XWrqxEREcmbFG5ymbFjzTE4q1aZDxEREUkfhZtcpkwZ6NHD3B4wAOx2a+sRERHJaxRucqGhQyEoCH75BebMsboaERGRvEXhJhcqUgSGDDG3hw6Fy5etrUdERCQvUbjJpXr1gogIOHYMJk+2uhoREZG8Q+Eml/L1hTFjzO1x4+DMGWvrERERySsUbnKxp5+Gu+6CCxdg1CirqxEREckbFG5yMQ8PmDDB3J4+Hfbts7YeERGRvEDhJpe7/3545BG4cQMGD7a6GhERkdxP4SYPGD/ebMWJjoZNm6yuRkREJHdTuMkDqlaF5583twcMMJdnEBERkZQp3OQRo0aBvz9s2WK24IiIiEjKFG7yiNBQ6N/f3B40CK5ds7YeERGR3ErhJg/p3x+KF4cDB+DDD7PnGnY7rF8P8+aZf2ptKxERyWsUbvKQgAAYOdLcHjkSYmOz9vzR0RAZCffdB888Y/4ZGaluMBERyVsUbvKYF1+ESpXg77/hzTez7rzR0dCuHfz1l/P+Y8fM/Qo4IiKSVyjc5DFeXvDWW+b2pElw9Gjmz2m3Q58+Kd+Flbivb191UYmISN6gcJMHtWoFTZrA1avmquGZtWFD8habpAzDDFEbNmT+WiIiItlN4SYPstlg4kRze84c2Lkzc+c7cSJrjxMREbGSwk0eVa8edOhgtqq8/nrmzhUamrXHiYiIWEnhJg8bMwby5YPVq2Hlyoyfp3FjCAszW4RSYrNBeLh5nIiISG6ncJOHlSkDPXua2wMGZHzAr6cnTJ5sbt8acBKfT5pkHiciIpLbKdzkcUOHQsGCsHs3fPppxs/Tpg0sWAAlSzrvDwsz97dpk6kyRUREcozNMNxrGca4uDiCgoKIjY0lMDDQ6nKyxMSJZstNiRKwf7+5BlVG2e3mXVEnTphjbBo3VouNiIhYLz3f32q5cQE9e0JEBBw/Du++m7lzeXpCs2bw9NPmnwo2IiKS1yjcuABfXxg71tx+6y04fdraekRERKykcOMiOnSAOnXgwgUYNcrqakRERKyjcOMiPDxgwgRz+8MPYd8+a+sRERGxisKNC7nvPnj0UbhxAwYNsroaERERayjcuJjx481WnIULYeNGq6sRERHJeQo3LqZKFXjhBXN7wICUV/oWERFxZQo3LmjkSMifH3780ZyAT0RExJ0o3Lig0FDo39/cHjwYrl2zth4REZGcpHDjovr3h+Bg+OMPmD7d6mpERERyjsKNiypQwOyeAnPem/PnLS1HREQkxyjcuLAXXoDKleHvv+HNN62uRkREJGco3LgwLy9zOQaASZPgyBFLyxEREckRCjcu7rHHoGlTiI+HoUOtrkZERCT7Kdy4OJsNJk40t+fOhR07rK1HREQkuyncuIG6deHpp80J/TSxn4iIuDqFGzcxZgx4e8PatbBypdXViIiIZB+FGzdRujT07GluDxgAdru19YiIiGQXhRs3MmQIFCwIv/4Ks2dbXY2IiEj2ULhxI4UL37xjatgwuHTJ2npERESyg8KNm+nZEyIj4fhxePddq6sRERHJego3bsbHB8aONbffegtOnbK2HhERkaymcOOGoqLM28MvXry5/pSIiIirULhxQx4eMGGCuT1jBsTEWFuPiIhIVlK4cVPNmplLM9jtMGiQ1dWIiIhkHYUbN/bWW2YrzqJFsGGD1dWIiIhkDYUbN1alCrz4ormtZRlERMRVKNy4uZEjIX9++Okn+Oorq6sRERHJPIUbNxcSYrbaAAweDPHx1tYjIiKSWbki3HzwwQdERkbi6+tL/fr12bp1622PnTVrFjabzenh6+ubg9W6ntdeM0POn3/CtGlWVyMiIpI5loeb+fPn069fP9544w22b99OzZo1adGiBadPn77tewIDAzlx4oTjcfjw4Rys2PUUKHBzvpvRo+H8eUvLERERyRTLw80777xDt27d6Nq1K1WqVGH69On4+/vzySef3PY9NpuNkJAQxyM4ODgHK3ZNzz8PlSvDuXMwbpzV1YiIiGScpeHm2rVr/PzzzzRv3tyxz8PDg+bNm7Nly5bbvu/ixYtEREQQHh7OE088wZ49e3KiXJfm5QXjx5vbkyeDGsNERCSvsjTcnD17FrvdnqzlJTg4mJMnT6b4nooVK/LJJ5/wzTffMHfuXBISEmjYsCF//fVXisfHx8cTFxfn9JCUPfqoOblffLw5uFhERCQvsrxbKr0aNGhAp06dqFWrFk2bNiU6OppixYrx4Ycfpnj8uHHjCAoKcjzCw8NzuOK8w2aDiRPN7XnztGq4iIjkTZaGm6JFi+Lp6cmpW5amPnXqFCEhIWk6R758+ahduzYHDhxI8fXBgwcTGxvreBw9ejTTdbuyOnXgzTfN7X794IsvrK1HREQkvSwNN97e3tSpU4e1a9c69iUkJLB27VoaNGiQpnPY7XZ2795NaGhoiq/7+PgQGBjo9JDUvf469OplbnfqBEl+PCIiIrme5d1S/fr146OPPmL27Nns3buX7t27c+nSJbp27QpAp06dGJxkAMioUaNYtWoVf/75J9u3b+fZZ5/l8OHDvJi4joBkms1mdkm1awfXr8OTT8LOnVZXJSIikjZeVhcQFRXFmTNnGD58OCdPnqRWrVqsWLHCMcj4yJEjeHjczGD//PMP3bp14+TJkxQqVIg6deqwefNmqlSpYtVHcEmenjBnDpw5A99/Dw8/DFu2QGSk1ZWJiIikzmYY7rVcYlxcHEFBQcTGxqqLKg3On4fGjeHXX6FiRdi4EYoWtboqERFxN+n5/ra8W0pyt4IFYcUKCA+HmBho1QouX7a6KhERkdtTuJE7KlnSDDiFCsGPP0JUFNy4YXVVIiIiKVO4kTSpUgWWLAFfX/j2W+jeHdyrQ1NERPIKhRtJs0aNzMn9PDzgv/+9udimiIhIbqJwI+nSujV88IG5PXIk3GZiaBEREcso3Ei6vfwyDB1qbr/yCnzzjbX1iIiIJKVwIxkyahQ8/zwkJECHDrB5s9UViYiImBRuJENsNrNL6tFH4epV8xbxvXutrkpEREThRjLBywvmz4e774Zz56BlSzh+3OqqRETE3SncSKbkzw9Ll0KFCnDkiLlMQ2ys1VWJiIg7U7iRTCta1JzkLyQEfvnFvKMqPt7qqkRExF0p3EiWKF0ali2DgABYvx46dTIHG4uIiOQ0hRvJMrVrQ3Q05MsHX34J/fppFmMREcl5CjeSpZo3h9mzze3Jk2HiRGvrERER96NwI1nu6advhprXX4e5c62tR0RE3IvCjWSL116DV181t7t2hVWrrK1HRETch8KNZJuJE83Zi2/cgLZtYft2qysSERF3oHAj2cbDA2bNgvvvh4sXzTlw/vzT6qpERMTVKdxItvLxgYULoWZNOH0aWrQw/xQREckuCjeS7QIDYflyiIiAAwfgscfMlhwREZHskKFwc/ToUf766y/H861bt9K3b19mzJiRZYWJawkNhZUroUgR2LYNnnoKrl+3uioREXFFGQo3zzzzDOvWrQPg5MmTPPjgg2zdupUhQ4YwatSoLC1QXEfFivDtt+DnZ7bkvPSSJvkTEZGsl6Fw8+uvv3L33XcD8OWXX1KtWjU2b97MZ599xqxZs7KyPnEx99xjriSeONh42DCrKxIREVeToXBz/fp1fHx8AFizZg2PP/44AJUqVeLEiRNZV524pFat4MMPze0xY2DqVGvrERER15KhcFO1alWmT5/Ohg0bWL16NS1btgTg+PHjFClSJEsLFNf04oswcqS53bOnuSaViIhIVshQuHnrrbf48MMPadasGU8//TQ1a9YEYPHixY7uKpE7GTbs5ribZ56BDRusrkhERFyBzTAyNqTTbrcTFxdHoUKFHPsOHTqEv78/xYsXz7ICs1pcXBxBQUHExsYSGBhodTlu78YNaNcOvvkGChaEjRuhalWrqxIRkdwmPd/fGWq5uXLlCvHx8Y5gc/jwYSZNmkRMTEyuDjaS+3h5wbx50LAhnD8PLVvC0aNWVyUiInlZhsLNE088waeffgrA+fPnqV+/Pm+//TatW7dm2rRpWVqguD4/P1iyBCpVgr/+Mpdp+Ocfq6sSEZG8KkPhZvv27TRu3BiABQsWEBwczOHDh/n000957733srRAcQ+FC8OKFVCiBOzZA61bw9WrVlclIiJ5UYbCzeXLlwkICABg1apVtGnTBg8PD+655x4OHz6cpQWK+4iIMCf3CwyEH36AZ58Fuz1j57LbYf16s8tr/fqMn0dERPKeDIWbcuXKsWjRIo4ePcrKlSt56KGHADh9+rQG6Uqm1KhhDi729oavv4Y+fdI/i3F0NERGwn33mXdh3Xef+Vy3m4uIuIcMhZvhw4fTv39/IiMjufvuu2nQoAFgtuLUrl07SwsU99OsGcyZAzYbfPABvPlm2t8bHW3efZVk6TMAjh0z9yvgiIi4vgzfCn7y5ElOnDhBzZo18fAwM9LWrVsJDAykUqVKWVpkVtKt4HnHe++ZLTdgLtXQuXPqx9vtZgvNrcEmkc0GYWFw8CB4emZlpSIikt3S8/2d4XCTKHF18LCwsMycJsco3OQtAwfC+PFmGFmyxLyT6nbWrze7oO5k3TqzdUhERPKObJ/nJiEhgVGjRhEUFERERAQREREULFiQ0aNHk5CQkKGiRVIybtzNgcXt2sG2bbc/Nq3Lmmn5MxER1+aVkTcNGTKEjz/+mDfffJNGjRoBsHHjRkaMGMHVq1cZM2ZMlhYp7svDAz7+GE6fhlWr4NFHYfNmKFcu+bGhoWk7Z1qPExGRvClD3VIlSpRg+vTpjtXAE33zzTe88sorHDt2LMsKzGrqlsqbLlwwu5x+/hnKlDEDTnCw8zGJY26OHUv5DiuNuRERybuyvVvq3LlzKQ4arlSpEufOncvIKUVSFRAAS5eawebPP80WnAsXnI/x9ITJk81tm835tcTnkyYp2IiIuLoMhZuaNWsyZcqUZPunTJlCjRo1Ml2USEqCg81ZjIsWNVtw2rWDa9ecj2nTBhYsgJIlnfeHhZn727TJuXpFRMQaGeqW+v7773n00UcpVaqUY46bLVu2cPToUZYtW+ZYmiE3UrdU3rdtm3m30+XL8NxzMHt28pYaux02bDAHD4eGQuPGarEREcnLsr1bqmnTpuzbt48nn3yS8+fPc/78edq0acOePXuYM2dOhooWSat69cxWGE9Pc7K/wYOTH+PpaQagp582/1SwERFxH5me5yapXbt2cdddd2HPxQv5qOXGdcyaBV27mtuTJ0Pv3paWIyIi2SjbW25EcoMuXWDsWHO7b1/48ksrqxERkdxC4UbytEGDoEcP89bv554zZykWERH3pnAjeZrNZnZJtW1r3jnVujXs3m11VSIiYqV0zVDc5g730Z4/fz4ztYhkiKcnzJ1rzmK8YQO0bAlbtkCpUlZXJiIiVkhXuAkKCrrj6506dcpUQSIZ4esL33xj3vK9Z48ZcDZuhMKFra5MRERyWpbeLZUX6G4p1/bXX9Cggflnw4awZg34+VldlYiIZJbulhK3FRZmzmJcsKC5/lS7dqAVQURE3IvCjbicqlVh8WLw8YFly6BKFfj6a6urEhGRnKJwIy6pcWP4/nuoXBlOnTJbcNq0MZdjEBER16ZwIy6rfn3YsQOGDQMvL1i40GzFmTnTnBdHRERck8KNuDQfHxg1ylxFvE4dOH8enn8eHnoIDh60ujoREckOCjfiFmrUgB9/hAkTzNvG16yBatXMCQBz8VJoIiKSAQo34ja8vKB/f3MG46ZN4fJlc02qe++F336zujoREckqCjfidsqVg+++g+nTISDAbNGpXRtGjzaXcBARkbxN4UbckocH/OtfZovNY4+ZoWb4cKhbF7Zts7o6ERHJDIUbcWthYeacOJ9/DkWLml1W99wDAwaY3VYiIpL3KNyI27PZ4OmnzVacZ56BhASYONEchLx+vdXViYhIeinciPy/YsXgs89gyRIoWRL++APuu8/svoqNtbo6ERFJK4UbkVs89pjZivPyy+bzGTPMyf+WLLG2LhERSRuFG5EUBAbCtGlmt1S5cnD8ODz+uNltdeaM1dWJiEhqFG5EUtG0KfzyC7z+unmH1bx55npVn32mJRxERHIrhRuRO/Dzg7fegp9+MgcZ//03PPsstGoFR49aXZ2IiNxK4UYkjerWhf/9D/7zH/D2hqVLoWpVczLAhASrqxMRkUQKNyLpkC8fDBkCO3dCgwZw4QJ0727eVbVvn9XViYgIKNyIZEjlyrBhg7nwpr8//PAD1KwJ48fDjRtWVyci4t5yRbj54IMPiIyMxNfXl/r167N169Y0ve+LL77AZrPRunXr7C1QJAWentC7N+zZAw8+CFevwsCBUL8+7NpldXUiIu7L8nAzf/58+vXrxxtvvMH27dupWbMmLVq04PTp06m+79ChQ/Tv35/GjRvnUKUiKYuMhJUrYeZMKFgQtm83x+cMHWoGHhERyVmWh5t33nmHbt260bVrV6pUqcL06dPx9/fnk08+ue177HY7HTt2ZOTIkZQpUyYHqxVJmc0GXbrA3r3Qtq3ZNTVmjLna+ObNVlcnIuJeLA03165d4+eff6Z58+aOfR4eHjRv3pwtW7bc9n2jRo2iePHivPDCCzlRpkiahYTAggXmIzgYfv8d7r3X7L66eNHq6kRE3IOl4ebs2bPY7XaCg4Od9gcHB3Py5MkU37Nx40Y+/vhjPvroozRdIz4+nri4OKeHSHZr29Zsxena1Zzs7/33oVo1s/tKRESyl+XdUulx4cIFnnvuOT766COKFi2apveMGzeOoKAgxyM8PDybqxQxFSoEn3xiBprISDh8GFq2NLuvzp2zujoREddlabgpWrQonp6enDp1ymn/qVOnCAkJSXb8H3/8waFDh2jVqhVeXl54eXnx6aefsnjxYry8vPjjjz+SvWfw4MHExsY6Hkc1pazksIcegt27oU8fc2zO7NnmQpxff211ZSIirsnScOPt7U2dOnVYu3atY19CQgJr166lQYMGyY6vVKkSu3fvZufOnY7H448/zn333cfOnTtTbJXx8fEhMDDQ6SGS0woUgEmTYNMmc46cU6egXTto0wZOnLC6OhER12J5t1S/fv346KOPmD17Nnv37qV79+5cunSJrl27AtCpUycGDx4MgK+vL9WqVXN6FCxYkICAAKpVq4a3t7eVH0Xkjho0gB07YNgw8PKChQvNVpyZM7UQp4hIVrE83ERFRTFx4kSGDx9OrVq12LlzJytWrHAMMj5y5Agn9E9bcSE+PjBqlLlOVZ06cP48PP+82X118KDV1YmI5H02w3Cvfy/GxcURFBREbGysuqjEcjduwLvvwvDh5oR//v4wdiz07GnOgCwiIqb0fH9b3nIj4s68vGDAAPjlF2jaFC5fhr59zblxfvvN6upERPImhRuRXKB8efjuO5g+HQIC4McfzdmNR46E1ath3jxYvx7sdqsrFRHJ/dQtJZLL/PUXvPwyLF2a/LWwMHMl8jZtcr4uERErqVtKJA8LCzNnNk7JX3+Zsx9HR+dsTSIieYnCjUguY7eb425S07WrOeOxiIgkp3Ajksts2GC20KQmLg7KlDFbcdau1Rw5IiJJKdyI5DJpndYpIcHsnmre3Jz1ePJkc84cERF3p3AjksuEhqbtuE8+gVdeMZd2iIkxu7JKloRu3WDnzuysUEQkd1O4EcllGjc2BxXbbCm/brNBeDh06gQffADHj8PUqVC1qjlPzn//a95G3rAhzJ1rTg4oIuJOFG5EchlPT7OLCZIHnMTnkybdnME4IAC6dzdXHv/hB4iKMicH3LIFnnvODEKDB8OhQzn1CURErKVwI5ILtWkDCxaY3UxJhYWZ+1Oa58ZmM1t9vvgCjh6F0aPN48+ehTffNAcgt2oFy5eb43VERFyVJvETycXsdvPuqRMnzLE4jRunb82pGzfg22/NbqvVq2/uL1PGbO3p2hWKFMn6ukVEslp6vr8VbkTcREyMubzDzJkQG2vu8/GBDh3Mgcn16t1+nI+IiNU0Q7GIJFOxorkC+bFjNwcdx8fD7NlQv74Zbj75xByULCKSlynciLiZ/PnhhRfg559vDjr29jafv/CCOU7ntddg/36rKxURyRiFGxE3ZbPBPffAp5+aMyK/9RZERsI//8A770CFCtCiBXzzjVYjF5G8ReFGRChWDF5/HQ4cMFcjf+QRM/ysWgWtW5sDkMeOhVOnrK5UROTOFG5ExMHT0ww2S5eaQef11827qY4cgSFDzDlznnkGNm7UelYiknsp3IhIisqUMbuq/vrL7Lq65x64fh3mzTNvSa9Z07z76uJFqysVEXGmcCMiqfL1NQcdb9lyc9Cxn585I3L37lCiBPTqBb/9ZnWlIiImhRsRSbO77jJvIz92zLytvHx5uHABpkwx17a67z5zBuXr162uVETcmcKNiKRboULmKuS//27OfPzkk+DhAevXQ/v2EBEBI0aYIUhEJKcp3IhIhnl4QPPmEB1tLsw5dCgEB5vLRYwcaYacdu1g3ToNQBaRnKPlF0QkS127BgsXmutZ/fDDzf2VKsGLL8K995qDkX19ratRRPIerS2VCoUbkZyzezdMmwZz5jjfVZUvH1Svbi75kPioUgW8vKyrVURyN4WbVCjciOS8uDiYO9dcoXzbNjh7Nvkxfn7mgOV69aBuXfPPcuXMri8REYWbVCjciFjLMODwYTPk/O9/N/+8cCH5sUFBN4NO4iMsTKuXi7gjhZtUKNyI5D4JCbBvnxl0Eh87dpirlt8qONg57NSrB0WL5nzNIpKzFG5SoXAjkjdcvw6//uoceH79NeVFPCMjnVt46tQB/ect4loUblKhcCOSd12+DLt2OQeemJjkx9lsULGic+tOrVq6Q0skL1O4SYXCjYhriY01l4VIGniOHEl+nJdX8ju0qlbVHVoieYXCTSoUbkRc3+nTzmFn2zY4cyb5cb6+ULu2c+ApX153aInkRgo3qVC4EXE/hmG25tx6h1ZcXPJjg4LMMTtJA094uO7QErGawk0qFG5EBMw7tPbvT36H1tWryY8tXtx5wPLdd0OxYjlfs4g7U7hJhcKNiNzO9euwZ49z4Nm9O+U7tCpUgMaNzUeTJuYdW2rdEck+CjepULgRyXl2O2zYYC6oGRpqBgJPT6urSpsrV5LfofX778mPK1nyZthp3NgcrKyxOyJZR+EmFQo3IjkrOhr69IG//rq5LywMJk+GNm2sqyszzp2DTZvMwLZhgzl+58YN52MKFTIXCU0MO3fdBd7e1tQr4goUblKhcCOSc6KjoV07c0BvUondNwsW5N2Ak9Tly/DTT2bQ+eEH2LLF3JeUnx/cc8/Nbqx77oH8+a2pVyQvUrhJhcKNSM6w281xKElbbJKy2cwWnIMH804XVVpdv24OTk5s2dm4Ef7+2/kYLy+zNSexZefee6FIEWvqFckLFG5SoXAjkjPWr4f77rvzcevWQbNm2V2NtRISzHE6P/xwM/AcPZr8uKpVncfthIfnfK0iuVV6vr81N6eIZIsTJ7L2uLzMwwOqVDEfL79s7jt8+GbQ+eEHM/zs2WM+pk83j4mIMLuwEsNOxYq6I0skLRRuRCRbhIZm7XGuJiLCfDz7rPn8zBmz+yox8OzYYQagOXPMB5hz6yQOUm7SBGrW1PIRIilRt5SIZIvEMTfHjiUfUAyuPeYmK1y4AD/+eLMr66efkk8wWKAANGx4s2Xn7rvNgcsirkhjblKhcCOScxLvlgLngONqd0vlhPh4c4HQpIOUY2Odj/H2NmdQTgw7jRqZy0mIuAKFm1Qo3IjkrJTmuQkPh0mTFGwyw26HX3+9GXYSJ0lMymaDGjWcx+2EhFhTr0hmKdykQuFGJOfl5RmK8wrDgD//vDlAecMGOHAg+XHlyjnffl6mjH4Wkjco3KRC4UZE3MWJE86DlHftSj7+ydvbDDwVKkD58s5/hoTo7izJPRRuUqFwIyLu6vx52LzZedmI+PjbH1+gwM2gc2v4KVw4x8oWARRuUqVwIyJistvhyBHYvx/27TMfiduHDpmTD95OkSLOgSdxu3x5LSsh2UPhJhUKNyIidxYfb96mnzTwJG4fO5b6e0uUSLm1p0wZ8PHJmfrF9SjcpELhRkQkcy5dMgcr39ras29f8jW0kvLwMOc+Sqmrq1QpDWyW1CncpELhRkQk+5w7Z4adlLq6Ll68/fu8vaFs2ZQHNoeGamCzKNykSuFGRCTnGQacPHkz6CQNP3/8kfrA5vz5bz+wWSupuw+Fm1Qo3IiI5C52u7lKekrjew4eTH1gc+HC5q3sQUHm0hP+/uYjo9uJf6qLLPdRuEmFwo2ISN5x7drNgc1Jw8/+/c6zXmc1b++sCUspBaek2z4+6nJLq/R8f2s9WRERybW8vaFiRfNxq8SBzQcPmtuXL5uPK1fSt534/MqVm+e+ds183Lp+V1az2W4Gnfz5zcVky5Y1H+XK3dwuUkQhKD3UciMiIoLZ/XX1avrDUUbClN2evtqCglIOPWXLQsmS5p1ork4tNyIiIunk4XGz2yi7Bypfv5489Fy4AIcPmwOs//jDbJX64w84ftxsQdq+3XzcysfHnEPo1uBTrhxERJitX+5GLTciImmkBUDFCpcvm4uiJoaepMHn0KHUW4E8PMw5hFIKPmXKmEts5BUaUJwKhRsRyYjoaOjTx3kQa1gYTJ4MbdpYV5e4txs3zCU0EsNO0uDzxx/O44hSEhycPPTk1nE+CjepULgRkfSKjoZ27ZKvqJ34P/4FCxRwJPdJnFsoadhJGn7OnUv9/YGBtw8+VozzUbhJhcKNiKSH3W4uGXC7245tNrMF5+BBdVFJ3vLPP86hJ2nwudP6YUnH+dwafCIjs2ecj8JNKhRuRCQ91q+H++6783Hr1kGzZtldjUjOuHLFDOwpdXcdOmR2h92Oh4c5Hm39+qytSXdLiYhkkRMnsvY4kbzAzw+qVDEft7pxw5xR+nbjfC5ftn71d4UbEZFUhIZm7XEieZ2XF5QubT4efND5NcOAU6fMgGMlhRsRkVQ0bmyOqTl2LPmAYrg55qZx45yvTSS3sdkgJMTqKsAN5jQUEck4T0/zdm9Iflts4vNJkzSYWCQ3UbgREbmDNm3M271LlnTeHxam28BFcqNcEW4++OADIiMj8fX1pX79+mzduvW2x0ZHR1O3bl0KFixI/vz5qVWrFnPmzMnBakXEHbVpY94lsm4dfP65+efBgwo2IrmR5WNu5s+fT79+/Zg+fTr169dn0qRJtGjRgpiYGIoXL57s+MKFCzNkyBAqVaqEt7c33377LV27dqV48eK0aNHCgk8gIu7C01O3e4vkBZbPc1O/fn3q1avHlClTAEhISCA8PJxevXoxaNCgNJ3jrrvu4tFHH2X06NF3PFbz3IiIiOQ96fn+trRb6tq1a/z88880b97csc/Dw4PmzZuzZcuWO77fMAzWrl1LTEwMTZo0yc5SRUREJI+wtFvq7Nmz2O12goODnfYHBwfz+++/3/Z9sbGxlCxZkvj4eDw9PZk6dSoP3nqz/f+Lj48nPj7e8TwuLi5rihcREZFcyfIxNxkREBDAzp07uXjxImvXrqVfv36UKVOGZil0ho8bN46RI0fmfJEiIiJiCUvDTdGiRfH09OTUqVNO+0+dOkVIKrMAeXh4UK5cOQBq1arF3r17GTduXIrhZvDgwfTr18/xPC4ujvDw8Kz5ACIiIpLrWDrmxtvbmzp16rB27VrHvoSEBNauXUuDBg3SfJ6EhASnrqekfHx8CAwMdHqIiIiI67K8W6pfv3507tyZunXrcvfddzNp0iQuXbpE165dAejUqRMlS5Zk3LhxgNnNVLduXcqWLUt8fDzLli1jzpw5TJs2zcqPISIiIrmE5eEmKiqKM2fOMHz4cE6ePEmtWrVYsWKFY5DxkSNH8PC42cB06dIlXnnlFf766y/8/PyoVKkSc+fOJSoqyqqPICKSp9jtsGGDuZJ5aKi5LpaWjxBXYvk8NzlN89yIiDuLjoY+feCvv27uCwsz18/SbMuSm+WZeW5ERCTnREdDu3bOwQbMFc/btTNfF3EFCjciIm7AbjdbbFJqq0/c17eveZxIXqdwIyLiBjZsSN5ik5RhwNGj5nEieZ3CjYiIGzhxImuPE8nNFG5ERNxAaGjWHieSmynciIi4gcaNzbuibLaUX7fZIDzcPE4kr1O4ERFxA56e5u3ekDzgJD6fNEnz3YhrULgREXETbdrAggVQsqTz/rAwc7/muRFXYfkMxSIiknPatIEnntAMxeLaFG5ERNyMpyc0a2Z1FSLZR91SIiIi4lIUbkRERMSlKNyIiIiIS1G4EREREZeiAcUiIpJn2e2680uSU7gREZE8KTraXOk86YKgYWHmZIWas8e9qVtKRETynOhoaNcu+Urnx46Z+6OjralLcgeFGxERyVPsdrPFxjCSv5a4r29f8zhxTwo3IiKSp2zYkLzFJinDgKNHzePEPSnciIhInnLiRNYeJ65H4UZERPKU0NCsPU5cj8KNiIjkKY0bm3dF2Wwpv26zQXi4eZy4J4UbERHJUzw9zdu9IXnASXw+aZLmu3FnCjciIpLntGkDCxZAyZLO+8PCzP2a58a9aRI/ERHJk9q0gSee0AzFkpzCjYiI5FmentCsmdVVSG6jbikRERFxKWq5ERERsZgWAM1aCjciIiIW0gKgWU/dUiIiIhbRAqDZQ+FGRETEAloANPso3IiIiFhAC4BmH4UbERERC2gB0OyjcCMiImIBLQCafRRuRERELKAFQLOPwo2IiIgFtABo9lG4ERERsYgWAM0emsRPRETEQq60AGhumWlZ4UZERMRirrAAaG6aaVndUiIiIpIpuW2mZYUbERERybDcONOywo2IiIhkWG6caVnhRkRERDIsN860rHAjIiIiGZYbZ1pWuBEREZEMy40zLSvciIiISIblxpmWFW5EREQkU3LbTMuaxE9EREQyLTfNtKxwIyIiIlkit8y0rG4pERERcSkKNyIiIuJSFG5ERETEpSjciIiIiEtRuBERERGXonAjIiIiLkXhRkRERFyKwo2IiIi4FIUbERERcSluN0OxYRgAxMXFWVyJiIiIpFXi93bi93hq3C7cXLhwAYDw8HCLKxEREZH0unDhAkFBQakeYzPSEoFcSEJCAsePHycgIADbrWuzC2Cm4/DwcI4ePUpgYKDV5bg9/TxyF/08ch/9THKX7Pp5GIbBhQsXKFGiBB4eqY+qcbuWGw8PD8LCwqwuI08IDAzU/yhyEf08chf9PHIf/Uxyl+z4edypxSaRBhSLiIiIS1G4EREREZeicCPJ+Pj48MYbb+Dj42N1KYJ+HrmNfh65j34muUtu+Hm43YBiERERcW1quRERERGXonAjIiIiLkXhRkRERFyKwo2IiIi4FIUbcRg3bhz16tUjICCA4sWL07p1a2JiYqwuS4A333wTm81G3759rS7FrR07doxnn32WIkWK4OfnR/Xq1fnf//5ndVluyW63M2zYMEqXLo2fnx9ly5Zl9OjRaVp3SDLvhx9+oFWrVpQoUQKbzcaiRYucXjcMg+HDhxMaGoqfnx/Nmzdn//79OVafwo04fP/99/To0YMff/yR1atXc/36dR566CEuXbpkdWlubdu2bXz44YfUqFHD6lLc2j///EOjRo3Ily8fy5cv57fffuPtt9+mUKFCVpfmlt566y2mTZvGlClT2Lt3L2+99Rbjx4/n/ffft7o0t3Dp0iVq1qzJBx98kOLr48eP57333mP69On89NNP5M+fnxYtWnD16tUcqU+3gsttnTlzhuLFi/P999/TpEkTq8txSxcvXuSuu+5i6tSp/Oc//6FWrVpMmjTJ6rLc0qBBg9i0aRMbNmywuhQBHnvsMYKDg/n4448d+9q2bYufnx9z5861sDL3Y7PZWLhwIa1btwbMVpsSJUrw2muv0b9/fwBiY2MJDg5m1qxZdOjQIdtrUsuN3FZsbCwAhQsXtrgS99WjRw8effRRmjdvbnUpbm/x4sXUrVuX9u3bU7x4cWrXrs1HH31kdVluq2HDhqxdu5Z9+/YBsGvXLjZu3MjDDz9scWVy8OBBTp486fT/raCgIOrXr8+WLVtypAa3WzhT0iYhIYG+ffvSqFEjqlWrZnU5bumLL75g+/btbNu2zepSBPjzzz+ZNm0a/fr149///jfbtm2jd+/eeHt707lzZ6vLczuDBg0iLi6OSpUq4enpid1uZ8yYMXTs2NHq0tzeyZMnAQgODnbaHxwc7HgtuyncSIp69OjBr7/+ysaNG60uxS0dPXqUPn36sHr1anx9fa0uRzADf926dRk7diwAtWvX5tdff2X69OkKNxb48ssv+eyzz/j888+pWrUqO3fupG/fvpQoUUI/D1G3lCTXs2dPvv32W9atW0dYWJjV5biln3/+mdOnT3PXXXfh5eWFl5cX33//Pe+99x5eXl7Y7XarS3Q7oaGhVKlSxWlf5cqVOXLkiEUVubcBAwYwaNAgOnToQPXq1Xnuued49dVXGTdunNWlub2QkBAATp065bT/1KlTjteym8KNOBiGQc+ePVm4cCHfffcdpUuXtrokt/XAAw+we/dudu7c6XjUrVuXjh07snPnTjw9Pa0u0e00atQo2dQI+/btIyIiwqKK3Nvly5fx8HD+CvP09CQhIcGiiiRR6dKlCQkJYe3atY59cXFx/PTTTzRo0CBHalC3lDj06NGDzz//nG+++YaAgABH32hQUBB+fn4WV+deAgICko11yp8/P0WKFNEYKIu8+uqrNGzYkLFjx/LUU0+xdetWZsyYwYwZM6wuzS21atWKMWPGUKpUKapWrcqOHTt45513eP75560uzS1cvHiRAwcOOJ4fPHiQnTt3UrhwYUqVKkXfvn35z3/+Q/ny5SldujTDhg2jRIkSjjuqsp0h8v+AFB8zZ860ujQxDKNp06ZGnz59rC7DrS1ZssSoVq2a4ePjY1SqVMmYMWOG1SW5rbi4OKNPnz5GqVKlDF9fX6NMmTLGkCFDjPj4eKtLcwvr1q1L8fuic+fOhmEYRkJCgjFs2DAjODjY8PHxMR544AEjJiYmx+rTPDciIiLiUjTmRkRERFyKwo2IiIi4FIUbERERcSkKNyIiIuJSFG5ERETEpSjciIiIiEtRuBERERGXonAjIm7JZrOxaNEiq8sQkWygcCMiOa5Lly7YbLZkj5YtW1pdmoi4AK0tJSKWaNmyJTNnznTa5+PjY1E1IuJK1HIjIpbw8fEhJCTE6VGoUCHA7DKaNm0aDz/8MH5+fpQpU4YFCxY4vX/37t3cf//9+Pn5UaRIEV566SUuXrzodMwnn3xC1apV8fHxITQ0lJ49ezq9fvbsWZ588kn8/f0pX748ixcvdrz2zz//0LFjR4oVK4afnx/ly5dPFsZEJHdSuBGRXGnYsGG0bduWXbt20bFjRzp06MDevXsBuHTpEi1atKBQoUJs27aNr776ijVr1jiFl2nTptGjRw9eeukldu/ezeLFiylXrpzTNUaOHMlTTz3FL7/8wiOPPELHjh05d+6c4/q//fYby5cvZ+/evUybNo2iRYvm3F+AiGRcji3RKSLy/zp37mx4enoa+fPnd3qMGTPGMAxzhfqXX37Z6T3169c3unfvbhiGYcyYMcMoVKiQcfHiRcfrS5cuNTw8PIyTJ08ahmEYJUqUMIYMGXLbGgBj6NChjucXL140AGP58uWGYRhGq1atjK5du2bNBxaRHKUxNyJiifvuu49p06Y57StcuLBju0GDBk6vNWjQgJ07dwKwd+9eatasSf78+R2vN2rUiISEBGJiYrDZbBw/fpwHHngg1Rpq1Kjh2M6fPz+BgYGcPn0agO7du9O2bVu2b9/OQw89ROvWrWnYsGGGPquI5CyFGxGxRP78+ZN1E2UVPz+/NB2XL18+p+c2m42EhAQAHn74YQ4fPsyyZctYvXo1DzzwAD169GDixIlZXq+IZC2NuRGRXOnHH39M9rxy5coAVK5cmV27dnHp0iXH65s2bcLDw4OKFSsSEBBAZGQka9euzVQNxYoVo3PnzsydO5dJkyYxY8aMTJ1PRHKGWm5ExBLx8fGcPHnSaZ+Xl5dj0O5XX31F3bp1uffee/nss8/YunUrH3/8MQAdO3bkjTfeoHPnzowYMYIzZ87Qq1cvnnvuOYKDgwEYMWIEL7/8MsWLF+fhhx/mwoULbNq0iV69eqWpvuHDh1OnTh2qVq1KfHw83377rSNciUjupnAjIpZYsWIFoaGhTvsqVqzI77//Dph3Mn3xxRe88sorhIaGMm/ePKpUqQKAv78/K1eupE+fPtSrVw9/f3/atm3LO++84zhX586duXr1Ku+++y79+/enaNGitGvXLs31eXt7M3jwYA4dOoSfnx+NGzfmiy++yIJPLiLZzWYYhmF1ESIiSdlsNhYuXEjr1q2tLkVE8iCNuRERERGXonAjIiIiLkVjbkQk11FvuYhkhlpuRERExKUo3IiIiIhLUbgRERERl6JwIyIiIi5F4UZERERcisKNiIiIuBSFGxEREXEpCjciIiLiUhRuRERExKX8H9OYtlmx9fX1AAAAAElFTkSuQmCC", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABWv0lEQVR4nO3dd3gU1f7H8fcmIQ1I6EmAEIpI701AioKChSICQVECFq5IFfEC0gIIqAiCIlUFFUUEQ5cuXBFQuFJERJArTSAUgYQe2Mzvj/llSUghfZLdz+t59tnds7Mz302i++HMmXNshmEYiIiIiDgJN6sLEBEREclMCjciIiLiVBRuRERExKko3IiIiIhTUbgRERERp6JwIyIiIk5F4UZEREScisKNiIiIOBWFGxEREXEqCjciFujevTulS5dO13vDw8Ox2WyZW1AOc/ToUWw2G/PmzcvW427evBmbzcbmzZsdban9XWVVzaVLl6Z79+6Zus/UmDdvHjabjaNHj2b7sUUySuFGJB6bzZaqW/wvP5GM2rZtG+Hh4Vy6dMnqUkScgofVBYjkJF988UWC559//jnr169P1F6pUqUMHWfOnDnExsam673Dhw9nyJAhGTq+pF5GfleptW3bNkaPHk337t0pUKBAgtcOHjyIm5v+HSqSFgo3IvE899xzCZ7/9NNPrF+/PlH73a5du4avr2+qj5MnT5501Qfg4eGBh4f+080uGfldZQYvLy9Ljy+SG+mfAyJp1Lx5c6pWrcovv/xC06ZN8fX15c033wRg2bJlPPHEExQvXhwvLy/KlSvH2LFjsdvtCfZx9ziOuPEa7733HrNnz6ZcuXJ4eXlRr149du7cmeC9SY25sdls9OnTh6VLl1K1alW8vLyoUqUKa9asSVT/5s2bqVu3Lt7e3pQrV45Zs2alehzPli1b6NSpE6VKlcLLy4vg4GBee+01rl+/nujz5cuXj5MnT9K+fXvy5ctH0aJFGTRoUKKfxaVLl+jevTv+/v4UKFCAsLCwVJ2e+e9//4vNZuOzzz5L9NratWux2WysXLkSgGPHjvHqq69SoUIFfHx8KFy4MJ06dUrVeJKkxtyktuZff/2V7t27U7ZsWby9vQkMDOSFF17gn3/+cWwTHh7OG2+8AUCZMmUcpz7jaktqzM1ff/1Fp06dKFSoEL6+vjzwwAOsWrUqwTZx44e++eYbxo0bR8mSJfH29qZFixYcPnz4np87OdOnT6dKlSp4eXlRvHhxevfuneiz//nnnzz99NMEBgbi7e1NyZIl6dKlC1FRUY5t1q9fz4MPPkiBAgXIly8fFSpUcPx3JJJR+uefSDr8888/PPbYY3Tp0oXnnnuOgIAAwByEmS9fPgYOHEi+fPn4/vvvGTlyJNHR0UycOPGe+/3qq6+4fPky//rXv7DZbLz77rt06NCBv/766549CD/++CMRERG8+uqr5M+fnw8++ICnn36a48ePU7hwYQB2795N69atCQoKYvTo0djtdsaMGUPRokVT9bkXLVrEtWvX6NWrF4ULF2bHjh18+OGH/P333yxatCjBtna7nVatWtGgQQPee+89NmzYwKRJkyhXrhy9evUCwDAM2rVrx48//sgrr7xCpUqVWLJkCWFhYfespW7dupQtW5Zvvvkm0fYLFy6kYMGCtGrVCoCdO3eybds2unTpQsmSJTl69CgzZsygefPm/P7772nqdUtLzevXr+evv/6iR48eBAYGsn//fmbPns3+/fv56aefsNlsdOjQgUOHDrFgwQLef/99ihQpApDs7+TMmTM0atSIa9eu0a9fPwoXLsxnn31G27ZtWbx4MU899VSC7d9++23c3NwYNGgQUVFRvPvuu3Tt2pWff/451Z85Tnh4OKNHj6Zly5b06tWLgwcPMmPGDHbu3MnWrVvJkycPMTExtGrVips3b9K3b18CAwM5efIkK1eu5NKlS/j7+7N//36efPJJqlevzpgxY/Dy8uLw4cNs3bo1zTWJJMkQkWT17t3buPs/k2bNmhmAMXPmzETbX7t2LVHbv/71L8PX19e4ceOGoy0sLMwICQlxPD9y5IgBGIULFzYuXLjgaF+2bJkBGCtWrHC0jRo1KlFNgOHp6WkcPnzY0bZ3714DMD788ENHW5s2bQxfX1/j5MmTjrY///zT8PDwSLTPpCT1+SZMmGDYbDbj2LFjCT4fYIwZMybBtrVq1TLq1KnjeL506VIDMN59911H2+3bt40mTZoYgDF37twU6xk6dKiRJ0+eBD+zmzdvGgUKFDBeeOGFFOvevn27ARiff/65o23Tpk0GYGzatCnBZ4n/u0pLzUkdd8GCBQZg/PDDD462iRMnGoBx5MiRRNuHhIQYYWFhjucDBgwwAGPLli2OtsuXLxtlypQxSpcubdjt9gSfpVKlSsbNmzcd206dOtUAjH379iU6Vnxz585NUNPZs2cNT09P49FHH3UcwzAMY9q0aQZgfPrpp4ZhGMbu3bsNwFi0aFGy+37//fcNwDh37lyKNYikl05LiaSDl5cXPXr0SNTu4+PjeHz58mXOnz9PkyZNuHbtGn/88cc99xsaGkrBggUdz5s0aQKYpyHupWXLlpQrV87xvHr16vj5+Tnea7fb2bBhA+3bt6d48eKO7e677z4ee+yxe+4fEn6+q1evcv78eRo1aoRhGOzevTvR9q+88kqC502aNEnwWb777js8PDwcPTkA7u7u9O3bN1X1hIaGcuvWLSIiIhxt69at49KlS4SGhiZZ961bt/jnn3+47777KFCgALt27UrVsdJTc/zj3rhxg/Pnz/PAAw8ApPm48Y9fv359HnzwQUdbvnz56NmzJ0ePHuX3339PsH2PHj3w9PR0PE/L31R8GzZsICYmhgEDBiQY4Pzyyy/j5+fnOC3m7+8PmKcGr127luS+4gZNL1u2LMsHa4trUrgRSYcSJUok+MKIs3//fp566in8/f3x8/OjaNGijsHI8ccbJKdUqVIJnscFnYsXL6b5vXHvj3vv2bNnuX79Ovfdd1+i7ZJqS8rx48fp3r07hQoVcoyjadasGZD483l7eyc6tRK/HjDHwgQFBZEvX74E21WoUCFV9dSoUYOKFSuycOFCR9vChQspUqQIDz/8sKPt+vXrjBw5kuDgYLy8vChSpAhFixbl0qVLqfq9xJeWmi9cuED//v0JCAjAx8eHokWLUqZMGSB1fw/JHT+pY8VdwXfs2LEE7Rn5m7r7uJD4c3p6elK2bFnH62XKlGHgwIF8/PHHFClShFatWvHRRx8l+LyhoaE0btyYl156iYCAALp06cI333yjoCOZRmNuRNIh/r/I41y6dIlmzZrh5+fHmDFjKFeuHN7e3uzatYvBgwen6n/c7u7uSbYbhpGl700Nu93OI488woULFxg8eDAVK1Ykb968nDx5ku7duyf6fMnVk9lCQ0MZN24c58+fJ3/+/CxfvpxnnnkmwRVlffv2Ze7cuQwYMICGDRvi7++PzWajS5cuWfqF2rlzZ7Zt28Ybb7xBzZo1yZcvH7GxsbRu3Trbvsiz+u8iKZMmTaJ79+4sW7aMdevW0a9fPyZMmMBPP/1EyZIl8fHx4YcffmDTpk2sWrWKNWvWsHDhQh5++GHWrVuXbX874rwUbkQyyebNm/nnn3+IiIigadOmjvYjR45YWNUdxYoVw9vbO8krZVJz9cy+ffs4dOgQn332Gd26dXO0r1+/Pt01hYSEsHHjRq5cuZKgJ+TgwYOp3kdoaCijR4/m22+/JSAggOjoaLp06ZJgm8WLFxMWFsakSZMcbTdu3EjXpHmprfnixYts3LiR0aNHM3LkSEf7n3/+mWifaZlxOiQkJMmfT9xpz5CQkFTvKy3i9nvw4EHKli3raI+JieHIkSO0bNkywfbVqlWjWrVqDB8+nG3bttG4cWNmzpzJW2+9BYCbmxstWrSgRYsWTJ48mfHjxzNs2DA2bdqUaF8iaaXTUiKZJO5fm/H/RRwTE8P06dOtKikBd3d3WrZsydKlSzl16pSj/fDhw6xevTpV74eEn88wDKZOnZrumh5//HFu377NjBkzHG12u50PP/ww1fuoVKkS1apVY+HChSxcuJCgoKAE4TKu9rt7Kj788MNEl6VnZs1J/bwApkyZkmifefPmBUhV2Hr88cfZsWMH27dvd7RdvXqV2bNnU7p0aSpXrpzaj5ImLVu2xNPTkw8++CDBZ/rkk0+IioriiSeeACA6Oprbt28neG+1atVwc3Pj5s2bgHm67m41a9YEcGwjkhHquRHJJI0aNaJgwYKEhYXRr18/bDYbX3zxRZZ2/6dVeHg469ato3HjxvTq1Qu73c60adOoWrUqe/bsSfG9FStWpFy5cgwaNIiTJ0/i5+fHt99+m+axG/G1adOGxo0bM2TIEI4ePUrlypWJiIhI83iU0NBQRo4cibe3Ny+++GKiGX2ffPJJvvjiC/z9/alcuTLbt29nw4YNjkvks6JmPz8/mjZtyrvvvsutW7coUaIE69atS7Inr06dOgAMGzaMLl26kCdPHtq0aeMIPfENGTKEBQsW8Nhjj9GvXz8KFSrEZ599xpEjR/j222+zbDbjokWLMnToUEaPHk3r1q1p27YtBw8eZPr06dSrV88xtuz777+nT58+dOrUifvvv5/bt2/zxRdf4O7uztNPPw3AmDFj+OGHH3jiiScICQnh7NmzTJ8+nZIlSyYYKC2SXgo3IpmkcOHCrFy5ktdff53hw4dTsGBBnnvuOVq0aOGYb8VqderUYfXq1QwaNIgRI0YQHBzMmDFjOHDgwD2v5sqTJw8rVqxwjJ/w9vbmqaeeok+fPtSoUSNd9bi5ubF8+XIGDBjA/PnzsdlstG3blkmTJlGrVq1U7yc0NJThw4dz7dq1BFdJxZk6dSru7u58+eWX3Lhxg8aNG7Nhw4Z0/V7SUvNXX31F3759+eijjzAMg0cffZTVq1cnuFoNoF69eowdO5aZM2eyZs0aYmNjOXLkSJLhJiAggG3btjF48GA+/PBDbty4QfXq1VmxYoWj9ySrhIeHU7RoUaZNm8Zrr71GoUKF6NmzJ+PHj3fMw1SjRg1atWrFihUrOHnyJL6+vtSoUYPVq1c7rhRr27YtR48e5dNPP+X8+fMUKVKEZs2aMXr0aMfVViIZYTNy0j8rRcQS7du3Z//+/UmOBxERyW005kbExdy9VMKff/7Jd999R/Pmza0pSEQkk6nnRsTFBAUFOdY7OnbsGDNmzODmzZvs3r2b8uXLW12eiEiGacyNiItp3bo1CxYsIDIyEi8vLxo2bMj48eMVbETEaajnRkRERJyKxtyIiIiIU1G4EREREaficmNuYmNjOXXqFPnz50/TlOciIiJiHcMwuHz5MsWLF7/nZJUuF25OnTpFcHCw1WWIiIhIOpw4cYKSJUumuI3LhZv8+fMD5g/Hz8/P4mpEREQkNaKjowkODnZ8j6fE5cJN3KkoPz8/hRsREZFcJjVDSjSgWERERJyKwo2IiIg4FYUbERERcSouN+ZGREQyl91u59atW1aXIU7A09Pznpd5p4bCjYiIpIthGERGRnLp0iWrSxEn4ebmRpkyZfD09MzQfhRuREQkXeKCTbFixfD19dXEqJIhcZPsnj59mlKlSmXo70nhRkRE0sxutzuCTeHCha0uR5xE0aJFOXXqFLdv3yZPnjzp3o8GFIuISJrFjbHx9fW1uBJxJnGno+x2e4b2o3AjIiLpplNRkpky6+9Jp6Uyid0OW7bA6dMQFARNmoC7u9VViYiIuB713GSCiAgoXRoeegiefda8L13abBcREedXunRppkyZkurtN2/ejM1my/IrzebNm0eBAgWy9Bg5kcJNBkVEQMeO8PffCdtPnjTbFXBERFJmt8PmzbBggXmfweEWKbLZbCnewsPD07XfnTt30rNnz1Rv36hRI06fPo2/v3+6jicp02mpDLDboX9/MIzErxkG2GwwYAC0a6dTVCIiSYmIMP8/Gv8fiCVLwtSp0KFD5h/v9OnTjscLFy5k5MiRHDx40NGWL18+x2PDMLDb7Xh43PursmjRommqw9PTk8DAwDS9R1JPPTcZsGVL4h6b+AwDTpwwtxMRkYSs6PkODAx03Pz9/bHZbI7nf/zxB/nz52f16tXUqVMHLy8vfvzxR/73v//Rrl07AgICyJcvH/Xq1WPDhg0J9nv3aSmbzcbHH3/MU089ha+vL+XLl2f58uWO1+8+LRV3+mjt2rVUqlSJfPny0bp16wRh7Pbt2/Tr148CBQpQuHBhBg8eTFhYGO3bt0/Tz2DGjBmUK1cOT09PKlSowBdffOF4zTAMwsPDKVWqFF5eXhQvXpx+/fo5Xp8+fTrly5fH29ubgIAAOnbsmKZjZxeFmwyI9zeXKduJiLiKe/V8g9nznZWnqJIzZMgQ3n77bQ4cOED16tW5cuUKjz/+OBs3bmT37t20bt2aNm3acPz48RT3M3r0aDp37syvv/7K448/TteuXblw4UKy21+7do333nuPL774gh9++IHjx48zaNAgx+vvvPMOX375JXPnzmXr1q1ER0ezdOnSNH22JUuW0L9/f15//XV+++03/vWvf9GjRw82bdoEwLfffsv777/PrFmz+PPPP1m6dCnVqlUD4L///S/9+vVjzJgxHDx4kDVr1tC0adM0HT/bGC4mKirKAIyoqKgM72vTJsMw/zNM+bZpU4YPJSKSo1y/ft34/fffjevXr6fr/Tnh/59z5841/P3949W0yQCMpUuX3vO9VapUMT788EPH85CQEOP99993PAeM4cOHO55fuXLFAIzVq1cnONbFixcdtQDG4cOHHe/56KOPjICAAMfzgIAAY+LEiY7nt2/fNkqVKmW0a9cu1Z+xUaNGxssvv5xgm06dOhmPP/64YRiGMWnSJOP+++83YmJiEu3r22+/Nfz8/Izo6Ohkj5dRKf1dpeX7Wz03GdCkiXluOLnL8m02CA42txMRkTtycs933bp1Ezy/cuUKgwYNolKlShQoUIB8+fJx4MCBe/bcVK9e3fE4b968+Pn5cfbs2WS39/X1pVy5co7nQUFBju2joqI4c+YM9evXd7zu7u5OnTp10vTZDhw4QOPGjRO0NW7cmAMHDgDQqVMnrl+/TtmyZXn55ZdZsmQJt2/fBuCRRx4hJCSEsmXL8vzzz/Pll19y7dq1NB0/uyjcZIC7uznoDRIHnLjnU6ZoMLGIyN2CgjJ3u8yUN2/eBM8HDRrEkiVLGD9+PFu2bGHPnj1Uq1aNmJiYFPdz9/IBNpuN2NjYNG1vJHXeLgsFBwdz8OBBpk+fjo+PD6+++ipNmzbl1q1b5M+fn127drFgwQKCgoIYOXIkNWrUyJELpyrcZFCHDrB4MZQokbC9ZEmzPStG+4uI5Ha5qed769atdO/enaeeeopq1aoRGBjI0aNHs7UGf39/AgIC2Llzp6PNbreza9euNO2nUqVKbN26NUHb1q1bqVy5suO5j48Pbdq04YMPPmDz5s1s376dffv2AeDh4UHLli159913+fXXXzl69Cjff/99Bj5Z1tCl4JmgQwfzcm/NUCwikjpxPd8dO5pBJn4HRU7r+S5fvjwRERG0adMGm83GiBEjUuyBySp9+/ZlwoQJ3HfffVSsWJEPP/yQixcvpmnJgjfeeIPOnTtTq1YtWrZsyYoVK4iIiHBc/TVv3jzsdjsNGjTA19eX+fPn4+PjQ0hICCtXruSvv/6iadOmFCxYkO+++47Y2FgqVKiQVR853RRuMom7OzRvbnUVIiK5R1zPd1Lz3EyZknN6vidPnswLL7xAo0aNKFKkCIMHDyY6Ojrb6xg8eDCRkZF069YNd3d3evbsSatWrXBPQwJs3749U6dO5b333qN///6UKVOGuXPn0vz/v8AKFCjA22+/zcCBA7Hb7VSrVo0VK1ZQuHBhChQoQEREBOHh4dy4cYPy5cuzYMECqlSpkkWfOP1sRnaf0LNYdHQ0/v7+REVF4efnZ3U5IiK50o0bNzhy5AhlypTB29s7Q/vS2nzpExsbS6VKlejcuTNjx461upxMkdLfVVq+v9VzIyIillLPd+ocO3aMdevW0axZM27evMm0adM4cuQIzz77rNWl5TgaUCwiIpILuLm5MW/ePOrVq0fjxo3Zt28fGzZsoFKlSlaXluOo50ZERCQXCA4OTnSlkyRNPTciIiLiVBRuRERExKko3IiIiIhTUbgRERERp6JwIyIiIk5F4UZEREScisKNiIhIGjVv3pwBAwY4npcuXZopU6ak+B6bzcbSpUszfOzM2k9KwsPDqVmzZpYeIysp3IiIiMto06YNrVu3TvK1LVu2YLPZ+PXXX9O83507d9KzZ8+MlpdAcgHj9OnTPPbYY5l6LGejcCMiIi7jxRdfZP369fwdf6XO/zd37lzq1q1L9erV07zfokWL4uvrmxkl3lNgYCBeXl7ZcqzcSuFGRERcxpNPPknRokWZN29egvYrV66waNEiXnzxRf755x+eeeYZSpQoga+vL9WqVWPBggUp7vfu01J//vknTZs2xdvbm8qVK7N+/fpE7xk8eDD3338/vr6+lC1blhEjRnDr1i0A5s2bx+jRo9m7dy82mw2bzeao+e7TUvv27ePhhx/Gx8eHwoUL07NnT65cueJ4vXv37rRv35733nuPoKAgChcuTO/evR3HSo3Y2FjGjBlDyZIl8fLyombNmqxZs8bxekxMDH369CEoKAhvb29CQkKYMGECAIZhEB4eTqlSpfDy8qJ48eL069cv1cdODy2/ICIimcIw4No1a47t6ws227238/DwoFu3bsybN49hw4Zh+/83LVq0CLvdzjPPPMOVK1eoU6cOgwcPxs/Pj1WrVvH8889Trlw56tevf89jxMbG0qFDBwICAvj555+JiopKMD4nTv78+Zk3bx7Fixdn3759vPzyy+TPn59///vfhIaG8ttvv7FmzRo2bNgAgL+/f6J9XL16lVatWtGwYUN27tzJ2bNneemll+jTp0+CALdp0yaCgoLYtGkThw8fJjQ0lJo1a/Lyyy/f+4cGTJ06lUmTJjFr1ixq1arFp59+Stu2bdm/fz/ly5fngw8+YPny5XzzzTeUKlWKEydOcOLECQC+/fZb3n//fb7++muqVKlCZGQke/fuTdVx081wMVFRUQZgREVFWV2KiEiudf36deP33383rl+/7mi7csUwzIiT/bcrV1Jf+4EDBwzA2LRpk6OtSZMmxnPPPZfse5544gnj9ddfdzxv1qyZ0b9/f8fzkJAQ4/333zcMwzDWrl1reHh4GCdPnnS8vnr1agMwlixZkuwxJk6caNSpU8fxfNSoUUaNGjUSbRd/P7NnzzYKFixoXIn3A1i1apXh5uZmREZGGoZhGGFhYUZISIhx+/ZtxzadOnUyQkNDk63l7mMXL17cGDduXIJt6tWrZ7z66quGYRhG3759jYcfftiIjY1NtK9JkyYZ999/vxETE5Ps8eIk9XcVJy3f3zotJSIiLqVixYo0atSITz/9FIDDhw+zZcsWXnzxRQDsdjtjx46lWrVqFCpUiHz58rF27VqOHz+eqv0fOHCA4OBgihcv7mhr2LBhou0WLlxI48aNCQwMJF++fAwfPjzVx4h/rBo1apA3b15HW+PGjYmNjeXgwYOOtipVquDu7u54HhQUxNmzZ1N1jOjoaE6dOkXjxo0TtDdu3JgDBw4A5qmvPXv2UKFCBfr168e6desc23Xq1Inr169TtmxZXn75ZZYsWcLt27fT9DnTSuFGREQyha8vXLlizS2tY3lffPFFvv32Wy5fvszcuXMpV64czZo1A2DixIlMnTqVwYMHs2nTJvbs2UOrVq2IiYnJtJ/V9u3b6dq1K48//jgrV65k9+7dDBs2LFOPEV+ePHkSPLfZbMTGxmba/mvXrs2RI0cYO3Ys169fp3PnznTs2BEwVzM/ePAg06dPx8fHh1dffZWmTZumacxPWmnMjYiIZAqbDeJ1IORonTt3pn///nz11Vd8/vnn9OrVyzH+ZuvWrbRr147nnnsOMMfQHDp0iMqVK6dq35UqVeLEiROcPn2aoKAgAH766acE22zbto2QkBCGDRvmaDt27FiCbTw9PbHb7fc81rx587h69aqj92br1q24ublRoUKFVNV7L35+fhQvXpytW7c6AmDcceKPQfLz8yM0NJTQ0FA6duxI69atuXDhAoUKFcLHx4c2bdrQpk0bevfuTcWKFdm3bx+1a9fOlBrvpnAjIiIuJ1++fISGhjJ06FCio6Pp3r2747Xy5cuzePFitm3bRsGCBZk8eTJnzpxJdbhp2bIl999/P2FhYUycOJHo6OgEISbuGMePH+frr7+mXr16rFq1iiVLliTYpnTp0hw5coQ9e/ZQsmRJ8ufPn+gS8K5duzJq1CjCwsIIDw/n3Llz9O3bl+eff56AgID0/XCS8MYbbzBq1CjKlStHzZo1mTt3Lnv27OHLL78EYPLkyQQFBVGrVi3c3NxYtGgRgYGBFChQgHnz5mG322nQoAG+vr7Mnz8fHx8fQkJCMq2+u+m0lIiIuKQXX3yRixcv0qpVqwTjY4YPH07t2rVp1aoVzZs3JzAwkPbt26d6v25ubixZsoTr169Tv359XnrpJcaNG5dgm7Zt2/Laa6/Rp08fatasybZt2xgxYkSCbZ5++mlat27NQw89RNGiRZO8HN3X15e1a9dy4cIF6tWrR8eOHWnRogXTpk1L2w/jHvr168fAgQN5/fXXqVatGmvWrGH58uWUL18eMK/8evfdd6lbty716tXj6NGjfPfdd7i5uVGgQAHmzJlD48aNqV69Ohs2bGDFihUULlw4U2uMz2YYhpFle8+BoqOj8ff3JyoqCj8/P6vLERHJlW7cuMGRI0coU6YM3t7eVpcjTiKlv6u0fH+r50ZEREScisKNiIiIOBWFGxEREXEqCjciIiLiVBRuREQk3VzsmhTJYpn196RwIyIiaRY34+01q1bKFKcUN0Nz/KUi0kOT+GWimBiIjoYiRayuREQka7m7u1OgQAHH+kS+vr6OGX5F0iM2NpZz587h6+uLh0fG4onCTSbZvBleeAFq14bFi62uRkQk6wUGBgKkegFGkXtxc3OjVKlSGQ7KCjeZpGhROHoUjhyBnTuhXj2rKxIRyVo2m42goCCKFSuWpYsgiuvw9PTEzS3jI2YUbjJJlSrQrRt89hkMHQobNlhdkYhI9nB3d8/wGAmRzKQBxZlo9Gjw9ISNG2H9equrERERcU0KN5koJARefdV8PHQoxMZaW4+IiIgrUrjJZG++CfnywS+/aGCxiIiIFRRuMlnRojBokPl4+HDQGDsREZHspXCTBQYONEPOn3/C3LlWVyMiIuJaFG6yQP78Zq8NQHg4aAJPERGR7KNwk0X+9S8oXRpOn4YPP7S6GhEREdehcJNFvLxgzBjz8dtvw8WL1tYjIiLiKiwPNx999BGlS5fG29ubBg0asGPHjhS3v3TpEr179yYoKAgvLy/uv/9+vvvuu2yqNm2efRaqVoVLl+Cdd6yuRkRExDVYGm4WLlzIwIEDGTVqFLt27aJGjRq0atUq2XVKYmJieOSRRzh69CiLFy/m4MGDzJkzhxIlSmRz5anj7g7jx5uPp06FkyetrUdERMQV2AzDMKw6eIMGDahXrx7Tpk0DzBVBg4OD6du3L0OGDEm0/cyZM5k4cSJ//PEHefLkSdcxo6Oj8ff3JyoqCj8/vwzVnxqGAU2awNat5jicmTOz/JAiIiJOJy3f35b13MTExPDLL7/QsmXLO8W4udGyZUu2b9+e5HuWL19Ow4YN6d27NwEBAVStWpXx48djt9uTPc7NmzeJjo5OcMtONps55gbg44/h0KFsPbyIiIjLsSzcnD9/HrvdTkBAQIL2gIAAIiMjk3zPX3/9xeLFi7Hb7Xz33XeMGDGCSZMm8dZbbyV7nAkTJuDv7++4BQcHZ+rnSI0HH4QnnwS7HUaMyPbDi4iIuBTLBxSnRWxsLMWKFWP27NnUqVOH0NBQhg0bxswUzvUMHTqUqKgox+3EiRPZWPEd48aZvTjffGMuzSAiIiJZw7JwU6RIEdzd3Tlz5kyC9jNnzhAYGJjke4KCgrj//vtxd3d3tFWqVInIyEhiYmKSfI+Xlxd+fn4JblaoXh26djUfDx1qSQkiIiIuwbJw4+npSZ06ddi4caOjLTY2lo0bN9KwYcMk39O4cWMOHz5MbLzltg8dOkRQUBCenp5ZXnNGjRkDefLA+vUQ72OLiIhIJrL0tNTAgQOZM2cOn332GQcOHKBXr15cvXqVHj16ANCtWzeGxuvm6NWrFxcuXKB///4cOnSIVatWMX78eHr37m3VR0iTMmXglVfMx0OHmldSiYiISObysPLgoaGhnDt3jpEjRxIZGUnNmjVZs2aNY5Dx8ePHcXO7k7+Cg4NZu3Ytr732GtWrV6dEiRL079+fwYMHW/UR0mzYMPj0U9i5EyIi4Omnra5IRETEuVg6z40Vsnuem6SMGmWeoqpQAX77DTwsjZgiIiI5X66Y58aVvf46FC4MBw/CvHlWVyMiIuJcFG4s4Odnnp4CCA+H69ctLUdERMSpKNxYpFcvKFXKXG/qo4+srkZERMR5KNxYxNsbRo82H48fb64cLiIiIhmncGOh55+HypXh4kWYONHqakRERJyDwo2F3N3NXhuA99+H06etrUdERMQZKNxYrG1beOABc1Dx2LFWVyMiIpL7KdxYzGaDt982H8+ZA4cPW1uPiIhIbqdwkwM0awaPPQa3b8OIEVZXIyIikrsp3OQQcWNvvv4adu+2thYREZHcTOEmh6hZE5591nz85puWliIiIpKrKdzkIGPGmOtMrVkDmzdbXY2IiEjupHCTg5QrBz17mo+HDAHXWtJUREQkcyjc5DAjRoCvL/z8MyxbZnU1IiIiuY/CTQ4TGAivvWY+fvNN8woqERERST2FmxzojTegUCE4cAC++MLqakRERHIXhZscyN//zhVTo0bBjRvW1iMiIpKbKNzkUK++CiVLwokTMH261dWIiIjkHgo3OZSPD4SHm4/Hj4eoKEvLERERyTUUbnKwsDCoWBH++Qfee8/qakRERHIHhZsczMMDxo0zH0+eDGfOWFuPiIhIbqBwk8M99RTUrw/XrsFbb1ldjYiISM6ncJPD2Wzw9tvm41mz4K+/rK1HREQkp1O4yQUeeggefRRu3YKRI62uRkREJGdTuMklJkww77/6CvbutbYWERGRnEzhJpeoXRtCQ83FNOMm+BMREZHEFG5ykbFjzSuovvsOfvgha45ht8PmzbBggXlvt2fNcURERLKKwk0uUr48vPSS+XjIELMXJzNFREDp0uYYn2efNe9LlzbbRUREcguFm1xmxAhz9uLt22HFiszbb0QEdOwIf/+dsP3kSbNdAUdERHILhZtcpnhx6N/ffPzmm5lz2shuN/eZVE9QXNuAATpFJSIiuYPCTS40eDAULAj798P8+Rnf35YtiXts4jMMcwHPLVsyfiwREZGspnCTCxUoYI65AXPem5s3M7a/06czdzsRERErKdzkUn36mKeojh+HmTMztq+goMzdTkRExEoKN7mUry+Eh5uP33oLoqPTv68mTaBkSXOph6TYbBAcbG4nIiKS0ync5GI9esD998P58+aq4enl7g5Tp5qP7w44cc+nTDG3ExERyekUbnIxD487K4VPmgRnz6Z/Xx06wOLFUKJEwvaSJc32Dh3Sv28REZHsZDOMzJ4KLmeLjo7G39+fqKgo/Pz8rC4nwwwD6tWDX36Bfv3u9MCkl91uXhV1+rQ5xqZJE/XYiIiI9dLy/a1w4wQ2bIBHHoE8eeDQIXNWYREREWeSlu9vnZZyAi1bQosWcOuWeWm4iIiIK1O4cRITJpj38+fDvn3W1iIiImIlhRsnUa+euQaUYcCwYVZXIyIiYh2FGyfy1lvm4N8VK+DHH62uRkRExBoKN06kQgV44QXz8ZAhSS+EKSIi4uwUbpzMqFHg7Q1bt8J331ldjYiISPZTuHEyJUqY890ADB1qzlsjIiLiShRunNDgweDvb141tWCB1dWIiIhkL4UbJ1SokBlwAEaMgJgYa+sRERHJTgo3Tqp/f3P5hKNHYdYsq6sRERHJPgo3TsrX985sxWPHwuXL1tYjIiKSXRRunNiLL8J998G5c/D++1ZXIyIikj0UbpxYnjzmxH4A771nhhwRERFnp3Dj5Dp1glq1zNNScetPiYiIODOFGyfn5nYn1Hz0ERw7Zm09IiIiWU3hxgU8+ig89JB5SXh4uNXViIiIZC2FGxdgs93pvfn8c9i/39p6REREspLCjYto0AA6dIDYWBg2zOpqREREso7CjQt56y1zDM6yZbB9u9XViIiIZA2FGxdSqRJ0724+HjIEDMPSckRERLKEwo2LCQ8HLy/44QdYs8bqakRERDKfwo2LCQ6GPn3Mx0OHmmNwREREnInCjQsaOhT8/GDvXvj6a6urERERyVwKNy6ocGH497/NxyNGmPPfiIiIOAuFGxfVvz8EBMBff8HHH1tdjYiISOZRuHFR+fKZvTYAY8bAlSvW1iMiIpJZFG5c2MsvQ9mycOYMTJ1qdTUiIiKZQ+HGhXl6wtix5uN334V//rG2HhERkcygcOPiunSBGjUgOvrO+lMiIiK5mcKNi3NzuxNqpk2DEyesrUdERCSjckS4+eijjyhdujTe3t40aNCAHTt2JLvtvHnzsNlsCW7e3t7ZWK3zad0amjaFmzdh9GirqxEREckYy8PNwoULGThwIKNGjWLXrl3UqFGDVq1acfbs2WTf4+fnx+nTpx23Y8eOZWPFzsdmg7ffNh/PnQsHDlhbj4iISEZYHm4mT57Myy+/TI8ePahcuTIzZ87E19eXTz/9NNn32Gw2AgMDHbeAgIBsrNg5NWwI7dqZyzEMH251NSIiIulnabiJiYnhl19+oWXLlo42Nzc3WrZsyfbt25N935UrVwgJCSE4OJh27dqxf//+ZLe9efMm0dHRCW6StHHjzDE4ERHw889WVyMiIpI+loab8+fPY7fbE/W8BAQEEBkZmeR7KlSowKeffsqyZcuYP38+sbGxNGrUiL///jvJ7SdMmIC/v7/jFhwcnOmfw1lUqQLdupmP33gD7HZr6xEREUkPy09LpVXDhg3p1q0bNWvWpFmzZkRERFC0aFFmzZqV5PZDhw4lKirKcTuhy4FSFB4OPj6wZYsZcERERHIbS8NNkSJFcHd358yZMwnaz5w5Q2BgYKr2kSdPHmrVqsXhw4eTfN3Lyws/P78EN0leSAjMm2c+fv99SCYzioiI5FiWhhtPT0/q1KnDxo0bHW2xsbFs3LiRhg0bpmofdrudffv2ERQUlFVlupzOne/MXNy7N2zYYG09IiIiaWH5aamBAwcyZ84cPvvsMw4cOECvXr24evUqPXr0AKBbt24MHTrUsf2YMWNYt24df/31F7t27eK5557j2LFjvPTSS1Z9BKc0bBh07WqOu+nYEf74w+qKREREUsfD6gJCQ0M5d+4cI0eOJDIykpo1a7JmzRrHIOPjx4/j5nYng128eJGXX36ZyMhIChYsSJ06ddi2bRuVK1e26iM4JZsNPv4YjhyBbdvgySfhp5+gSBGrKxMREUmZzTAMw+oislN0dDT+/v5ERUVp/E0qnD0LDRrA0aPQpAmsXw9eXlZXJSIiriYt39+Wn5aSnK1YMVi5Evz8zCuo/vUvcK04LCIiuY3CjdxTlSrwzTfmBH+ffQbvvGN1RSIiIslTuJFUadUKPvjAfDx0qDmLsYiISE6kcCOp1rs39OljPn7uOfjlF2vrERERSYrCjaTJ+++bvTjXr0PbtnDypNUViYiIJKRwI2ni4QELF0LlynDqFLRpA1evWl2ViIjIHQo3kmb+/uYVVEWLwu7d5imq2FirqxIRETEp3Ei6lCkDS5eCp6d5/+abVlckIiJiUriRdGvUCD791Hz8zjswd6619YiIiIDCjWRQ164wYoT5uGdP2LzZ0nJEREQUbiTjwsPNlcRv34ann4Y//7S6IhERcWUKN5Jhbm4wbx7Urw8XLpiLbF68aHVVIiLiqhRuJFP4+MCyZRAcDIcOQceOcOuW1VWJiIgrUriRTBMYaF4ini8ffP+9OZuxFtkUEZHspnAjmap6dViwAGw2mD0bpkyxuiIREXE1CjeS6Z58EiZNMh+//jqsWGFtPSIi4loUbiRLDBhgXhpuGPDMM7B3r9UViYiIq1C4kSxhs8G0afDww+baU23awOnTVlclIiKuQOFGskyePLB4MVSoACdOQLt25mriIiIiWSld4ebEiRP8/fffjuc7duxgwIABzJ49O9MKE+dQsKB5BVWhQrBzJ4SFaZFNERHJWukKN88++yybNm0CIDIykkceeYQdO3YwbNgwxowZk6kFSu53330QEWH25CxaZM5oLCIiklXSFW5+++036tevD8A333xD1apV2bZtG19++SXz5s3LzPrESTRrBrNmmY/HjoX5862tR0REnFe6ws2tW7fw8vICYMOGDbRt2xaAihUrclqjRiUZPXrA4MHm4xdfhK1bra1HREScU7rCTZUqVZg5cyZbtmxh/fr1tG7dGoBTp05RuHDhTC1QnMv48dC+PcTEmPd//WV1RSIi4mzSFW7eeecdZs2aRfPmzXnmmWeoUaMGAMuXL3ecrhJJipubeUqqdm04f968RDwqyuqqRETEmdgMI32r/9jtdqKjoylYsKCj7ejRo/j6+lKsWLFMKzCzRUdH4+/vT1RUFH5+flaX47JOnjRXET91Clq1Mq+o8vCwuioREcmp0vL9na6em+vXr3Pz5k1HsDl27BhTpkzh4MGDOTrYSM5RooS5LIOvL6xdC6+9ZnVFIiLiLNIVbtq1a8fnn38OwKVLl2jQoAGTJk2iffv2zJgxI1MLFOdVu/adq6amTTNvIiIiGZWucLNr1y6aNGkCwOLFiwkICODYsWN8/vnnfPDBB5laoDi3p56Ct982H/fvD2vWWFuPiIjkfukKN9euXSN//vwArFu3jg4dOuDm5sYDDzzAsWPHMrVAcX7//jd0727OXNy5M/z2m9UViYhIbpaucHPfffexdOlSTpw4wdq1a3n00UcBOHv2rAbpSprZbOYEf02bwuXL5hVUZ89mbJ92O2zeDAsWmPd2e2ZUKiIiuUG6ws3IkSMZNGgQpUuXpn79+jRs2BAwe3Fq1aqVqQWKa/D0hG+/hXLl4OhR83TVjRvp21dEBJQuDQ89BM8+a96XLm22i4iI80v3peCRkZGcPn2aGjVq4OZmZqQdO3bg5+dHxYoVM7XIzKRLwXO2P/6Ahg3h0iXo2hW++MLs2UmtiAjo2BHu/quO28fixdChQ6aVKyIi2SQt39/pDjdx4lYHL1myZEZ2k20UbnK+jRvNuW/sdnMdquHDU/c+u93soYm3YH0CNhuULAlHjoC7e6aVKyIi2SDL57mJjY1lzJgx+Pv7ExISQkhICAUKFGDs2LHExsamq2iROC1awPTp5uMRI+Cbb1L3vi1bkg82YPbmnDhhbiciIs4rXXPCDhs2jE8++YS3336bxo0bA/Djjz8SHh7OjRs3GDduXKYWKa6nZ0/zFNX770NYGISEQIMGKb8ntWu2am1XERHnlq5w89lnn/Hxxx87VgMHqF69OiVKlODVV19VuJFMMXEiHDoEq1ZBu3awYweUKpX89kFBqdtvarcTEZHcKV2npS5cuJDkoOGKFSty4cKFDBclAua4mAULoHp1OHPGvET88uXkt2/SxBxTk9wAZJsNgoPN7URExHmlK9zUqFGDaUnMlT9t2jSqV6+e4aJE4uTPb65BFRAAv/5qXtqd3Jw17u4wdar5+O6AE/d8yhQNJhYRcXbpulrqP//5D0888QSlSpVyzHGzfft2Tpw4wXfffedYmiEn0tVSudPPP0Pz5ubcNwMHwqRJyW8bEWEu5RB/cHFwsBlsdBm4iEjulOVXSzVr1oxDhw7x1FNPcenSJS5dukSHDh3Yv38/X3zxRbqKFklJgwbw2Wfm48mTYfbs5Lft0MGcCHDTJvjqK/P+yBEFGxERV5HheW7i27t3L7Vr18aeg+e6V89N7jZ2LIwcCR4e5iKbLVpYXZGIiGSHLO+5EbHK8OHmuJvbt+Hpp83LxUVEROJTuJFcxWaDTz6BRo0gKgqefBL++cfqqkREJCdRuJFcx9sbliwxl1r43//MsTQxMVZXJSIiOUWaJvHrcI8RmZcuXcpILSKpVqwYrFxpLrL5ww/wyitmj05aFtkUERHnlKZw4+/vf8/Xu3XrlqGCRFKrShVz3aknnoC5c6FiRfj3v62uSkRErJapV0vlBrpayvlMmwZ9+5q9Nt9+C089ZXVFIiKS2XS1lLiUPn2gd29z1e/nnoNdu6yuSERErKRwI05hyhR49FG4ds1cg+rkSasrEhERqyjciFPw8DDH31SuDKdOQdu2cPWq1VWJiIgVFG7Eafj7m1dQFSlinpp6/nmIjbW6KhERyW4KN+JUypSBpUvB09OcC+ff/zbH4oiIiOtQuBGn07ixOecNmKuHt25tnqoSERHXoHAjTum552DmTHM243XroGpVWLTI6qpERCQ7KNyI0/rXv8yxN7Vrw8WL0LmzOQ5HE2mLiDg3hRtxapUqwfbtMGwYuLnB/PlQvTps3mx1ZSIiklUUbsTpeXrCW2/Bli1QtiycOAEPPwyDBsGNG1ZXJyIimU3hRlxGo0awdy+89JJ5BdWkSVC/Pvz6q9WViYhIZlK4EZeSLx/MmQPLlkHRorBvH9SrBxMngt1udXUiIpIZFG7EJbVtC7/9Zt7HxJjz4Tz8MBw7ZnVlIiKSUQo34rKKFTMn/JszB/LmhR9+MAcbf/65Jv4TEcnNFG7Epdls5hicvXuhYUOIjoawMOjUCc6ft7o6ERFJD4UbEaBcObPn5q23zEU4v/0WqlWDNWusrkxERNJK4Ubk/3l4mPPh/PQTVKwIkZHw2GPQuzdcu2Z1dSIikloKNyJ3qVPHnNm4b1/z+fTpUKsW7NxpbV0iIpI6CjciSfDxgQ8+gLVroXhxOHTIHJMzZgzcvm11dSIikhKFG5EUPPqoORdO587mPDijRsGDD8Kff1pdmYiIJCdHhJuPPvqI0qVL4+3tTYMGDdixY0eq3vf1119js9lo37591hYoLq1QIfj6a/jyS/D3h59/hpo1YdYsXTIuIpITWR5uFi5cyMCBAxk1ahS7du2iRo0atGrVirNnz6b4vqNHjzJo0CCaNGmSTZWKK7PZ4NlnzaUaHnrIHGD8yivQpo058FhERHIOy8PN5MmTefnll+nRoweVK1dm5syZ+Pr68umnnyb7HrvdTteuXRk9ejRly5bNxmrF1ZUqBRs2wOTJ4OUFq1aZl4wvXWp1ZSIiEsfScBMTE8Mvv/xCy5YtHW1ubm60bNmS7du3J/u+MWPGUKxYMV588cV7HuPmzZtER0cnuIlkhJsbvPYa/Pe/UKOGOdnfU0/BCy/A5ctWVyciIpaGm/Pnz2O32wkICEjQHhAQQGQyff0//vgjn3zyCXPmzEnVMSZMmIC/v7/jFhwcnOG6RQCqVjXH3/z73+Zpq7lzzbDz449WVyYi4tosPy2VFpcvX+b5559nzpw5FClSJFXvGTp0KFFRUY7biRMnsrhKcSVeXvDOO7B5M4SEwJEj0LQpDB1qLsgpIiLZz8PKgxcpUgR3d3fOnDmToP3MmTMEBgYm2v5///sfR48epU2bNo622NhYADw8PDh48CDlypVL8B4vLy+8vLyyoHqRO5o2NQcb9+8P8+bB22+bSzfMnw9VqlhdnYiIa7G058bT05M6deqwceNGR1tsbCwbN26kYcOGibavWLEi+/btY8+ePY5b27Zteeihh9izZ49OOYml/PzMU1PffguFC8OePeZsx1OmwP9ncBERyQaW9twADBw4kLCwMOrWrUv9+vWZMmUKV69epUePHgB069aNEiVKMGHCBLy9valatWqC9xcoUAAgUbuIVTp0MGczfvFFWL3aHHy8cqXZo1OypNXViYg4P8vDTWhoKOfOnWPkyJFERkZSs2ZN1qxZ4xhkfPz4cdzcctXQIBGCgszLxGfNgoEDYeNG85Lx6dPhmWesrk5ExLnZDMO15liNjo7G39+fqKgo/Pz8rC5HXMChQ/Dcc3cW3uzSxQw5BQtaW5eISG6Slu9vdYmIZLH774etWyE8HNzdzaUcqlUzJwMUEZHMp3Ajkg3y5DEX3dy6FcqXh5Mn4ZFHYMAAuH49+ffZ7eZl5gsWmPd2ezYVLCKSiynciGSjBg1g927o1ct8PnUq1K1rtt0tIgJKlzbXsnr2WfO+dGmzXUREkqdwI5LN8uY1x9ysWgUBAfD772bomTDhTs9MRAR07Ah//53wvSdPmu0KOCIiyVO4EbHI44/Db7+Z61LdugVvvgnNmsGff5qTASY11D+ubcAAnaISEUmOwo2IhYoUMSf9mzcP8uc3x+TUqJG4xyY+w4ATJ2DLlmwrU0QkV1G4EbGYzQZhYbB3LzRpkvIA4/hOn87aukREciuFG5EcokwZ2LQJevZM3fZBQVlbj4hIbqVwI5KDuLubg42LFUt+G5sNgoPNXh4REUlM4UYkh3F3hxkzkn/dMMzFON3ds60kEZFcReFGJAfq0MEcaJzcQpsTJ8KiRXD7dvbWJSKSG2htKZEczG43r4o6fRpu3IAff4Qvv4SbN83XQ0LMy8ZffBH05ywiziwt398KNyK5zJkz5mmr6dPh3DmzLX9+ePll6NfPDDwiIs5GC2eKOLGAAHMRzmPHYM4cqFQJLl+GyZOhbFkIDYWff7a6ShER6yjciORSPj7w0kuwfz+sXm0uxBkbC998Aw88AI0bm+N2NJOxiLgahRuRXM5mg9atYd06+PVX6NEDPD1h2zZzHary5c0FOi9ftrpSEZHsoXAj4kSqVYNPPzVPWY0YAYULw5Ej5lpUJUvCG2/A8eNWVykikrUUbkScUGAgjBljrkE1axZUrAjR0fDee+a4nGeegZ07ra5SRCRrKNyIODEfH3M5h/37YdUqaNHCHIPz9ddQv745y/GSJRqXIyLOReFGxAW4ucHjj8OGDbBnj7lQZ5485rw5HTrA/ffDhx/ClStWVyoiknEKNyIupkYNmDfPHJczbBgUKgR//WXOkRMcDIMHw99/W12liEj6KdyIuKigIHjrLXNczowZZu/NpUvw7rvmCuVdu8Ivv1hdpYhI2inciLg4X1945RU4cABWrICHHjLXrPrqK6hbF5o3h+XLzTl0RERyA4UbEQHMcTlPPgnffw+7dsHzz4OHB/znP9CuHVSoYC75cPWq1ZWKiKRM4UZEEqlVCz7/HI4ehaFDoWBBOHwYevc2x+W8+SacOmV1lSIiSVO4EZFklSgB48eb43KmTYP77oOLF2HCBChdGrp1g927ra5SRCQhhRsRuae8ec1emz/+gKVLoWlTuHULvvgCateGhx+GlSs1LkdEcgaFGxFJNXd3c/zNf/5jznD87LPmuJxNm6BNG3OF8pkz4do1qysVEVemcCMi6VK3Lnz5pTlHzr//Df7+cOgQ9OpljssZPhxOn7a6ShFxRQo3IpIhwcHwzjvmxH8ffGCuXXXhAowbByEh0L077N1rdZUi4koUbkQkU+TLB337mr03ERHw4IPmuJzPPoOaNaFlS3MenZs3ra5URJydwo2IZCp3d3jqKdiyBX7+Gbp0Mds2boS2baFoUQgNNScJvHTJ6mpFxBnZDMMwrC4iO0VHR+Pv709UVBR+fn5WlyPiEo4fNxfmnD8fIiPvtHt4QLNm5iDltm3N01giIklJy/e3wo2IZJvYWPMqq2XLzNvvvyd8vWZNM+i0a2c+ttmsqFJEciKFmxQo3IhkP7vdPE11+rS5YGeTJuapqsOH7wSdrVsTzpNTqpTZm9Oundm7kyePdfWLiPUUblKgcCOSvSIioH9/82qqOCVLwtSp0KHDnbbz582JAJctg3XrEs6V4+8Pjz9uBp3HHgP9pyviehRuUqBwI5J9IiKgY0e4+/8ycaebFi9OGHDiXL8OGzaYQWfFCjh79s5refKYK5fHjdMpWTLr6heRnEPhJgUKNyLZw24315+K32MTn81mBpMjR8xTVCnt5+ef75y+Ongw4et16twZp1OtmsbpiDgrhZsUKNyIZI/Nm80elnvZtAmaN0/9fg8evBN0tm9P2CtUuvSdoNOkiXk1log4h7R8f2ueGxHJEqldeiGtSzRUqGAu97B1q/nejz8217Xy9oajR82xPA8/DMWKwfPPm6e+Ll9Oc/kikosp3IhIlggKytztkhIQAC++CMuXmwOSlywxl3soUgQuXjTn1enUyXz++OMwa5bWuxJxBTotJSJZIm7MzcmTiQcUQ+rH3KT32Nu23Tl9dfhwwtfr179z+qpyZY3TEckNNOYmBQo3Itkn7mopSBhw7nW1VGYyDDhw4E7Q+fnnhK+XK3cn6DRqpHE6IjmVwk0KFG5EsldS89wEB8OUKVkfbJJy+rR5efmyZeZ6V/EX8ixcGJ580gw6jz4KefNmf30ikjSFmxQo3Ihkv+RmKLbalSuwdq0ZdFatggsX7rzm7W2uZN6unTlgOSDAujpFROEmRQo3IpKU27fhxx/vnL46cuTOazYbPPDAndNXFStaV6eIq1K4SYHCjYjci2HAb7/dCTr//W/C1++7Dx580Aw8DzwAVaporI5IVlO4SYHCjYik1cmT5uXmy5bB99/DrVsJX8+bF+rVuxN2GjSAwEBrahVxVgo3KVC4EZGMiI6GH34wr7r66SfzPqlJAkNC7oSdBx6AWrXAyyv76xVxFgo3KVC4EZHMZLfDH3+YQSfutn9/4rl9PD3NgBM/8ISEaI4dkdRSuEmBwo2IZLXoaHOcTvzAc+5c4u0CAhKeyqpXD/Lly/56RXIDhZsUKNyISHYzDPPqq/hhZ8+exGN33NygatWEvTsVKpjtIq5O4SYFCjcikhPcuAG7dycMPMePJ97O39/s1YkLO/Xrm5MNirgahZsUKNyISE516tSdgco//QQ7d8L164m3K18+Ye9OtWqQJ0/21yuSnRRuUqBwIyK5xe3bsG/fnbDz889w8GDi7Xx8oG7dhIGnePHsr1ckKyncpEDhRkRyswsXYMeOhIHn0qXE2wUHJzydVbu2GYJEciuFmxQo3IiIM4mNhUOHEo7d2bfPbI/PwwNq1kzYu1O2rC5Fl9xD4SYFCjcikl45dQHQu125Yl6KHjd+Z/t2OHMm8XaFCpnjd8qUgdKlzfu4W6lS5tw8IjmFwk0KFG5EJD0iIqB/f/j77zttJUvC1KnQoYN1daWGYZhXYsXv3dm1C2Jikn+PmxuUKJEw8MS/FS+uS9QleyncpEDhRkTSKiICOnZMPOtw3CmdxYtzfsC5282b8Pvv5vw7d9+OHk36Kq34PD3NGZaTCz+FC+uUl2QuhZsUKNyISFrY7eYpm/g9NvHZbGYPzpEjOfMUVXoYhnkaK6ngc+SI2Qtkt6e8j3z5kg8+ZcpoJmZJO4WbFCjciEhabN4MDz107+02bYLmzbO6mpzh9m0z7CUXfk6fvvc+ihRJPviUKqVFRiWxtHx/e2RTTSIiuVJqvqjTsp0z8PAwe7NKl046+F2/DseOJTzNFT/8XLgA58+bt507E7/fZrv3eB9n6SWTrKFwIyKSgqCgzN3OFfj4QMWK5i0pUVGJA0/827VrZs/Q33+bV6fdLU8es3cnfuAJCoJixcxbQIB5r94f16XTUiIiKYgbc3PyZOIBxeCcY26sZBjmCurJBZ9jx8zTYqnh75848MTd393m768B0DmdTkuJiGQSd3fzcu+OHc0vv/gBJ+7LcMoUBZvMYrPdCR8NGiR+3W43g+bdgefMGTh79s79rVtmD1FUFPz5572PmydP6kJQsWJQtKjW8srp1HMjIpIKSc1zExxsBpvcdhm4szMMc0mKs2cTBp64+7vboqPTfoxChe4dguIe58unXqHMkOuulvroo4+YOHEikZGR1KhRgw8//JD69esnuW1ERATjx4/n8OHD3Lp1i/Lly/P666/z/PPPp+pYCjcikl65ZYZiSZsbN+6EnnuFoXPn7n0Z/N28vVMXgooVM68i099U0nJVuFm4cCHdunVj5syZNGjQgClTprBo0SIOHjxIsWLFEm2/efNmLl68SMWKFfH09GTlypW8/vrrrFq1ilatWt3zeAo3IiKSXrGx5tVeqekROnsWrl5N2/7d3SEw0LwiLCjIvE/qcdGirjdDdK4KNw0aNKBevXpMmzYNgNjYWIKDg+nbty9DhgxJ1T5q167NE088wdixY++5rcKNiIhkl6tXEwefpELQmTPwzz9JD1pPioeHGYLih56kQlCRIs4TgnLNgOKYmBh++eUXhg4d6mhzc3OjZcuWbN++/Z7vNwyD77//noMHD/LOO+9kZakiIiJpljfvncvV7+X2bTPonDpl3k6fvvM4/vOzZ+9MpJjczNlxPDzMkHOvEFS4sPOEILA43Jw/fx673U5AQECC9oCAAP74449k3xcVFUWJEiW4efMm7u7uTJ8+nUceeSTJbW/evMnNmzcdz6PTM3JMREQki3l43AkcKbl1K/kQFP/xuXNmCDpxwrylJE+exCEoqUCUW9YMy5WXgufPn589e/Zw5coVNm7cyMCBAylbtizNk5j7fMKECYwePTr7ixQREckCefKYMziXKJHydrdumae7UgpAp0/fuXT++HHzlhJPz5RDUNzjQoWsDUGWjrmJiYnB19eXxYsX0759e0d7WFgYly5dYtmyZanaz0svvcSJEydYu3ZtoteS6rkJDg7WmBsREREgJiZ1IejcudTvs2ZN2L07c+vMNWNuPD09qVOnDhs3bnSEm9jYWDZu3EifPn1SvZ/Y2NgEASY+Ly8vvDQHt4iISJI8Pc05m4KDU94uJgYiI1MOQKdOmWuGFS2aPbUnx/LTUgMHDiQsLIy6detSv359pkyZwtWrV+nRowcA3bp1o0SJEkyYMAEwTzPVrVuXcuXKcfPmTb777ju++OILZsyYYeXHEBERcWqenuaaXqVKpbxdTAxcuZI9NSXH8nATGhrKuXPnGDlyJJGRkdSsWZM1a9Y4BhkfP34ct3hDuK9evcqrr77K33//jY+PDxUrVmT+/PmEhoZa9RFERHIVTUYoWcnT0xxzYyXL57nJbprnRkRcWVLLSJQsaa6fpWUkJCdLy/e3E13VLiIiKYmIMBcAvXtulJMnzfaICGvqEslsCjciIi7Abjd7bJLqq49rGzAg7esmieRECjciIi5gy5aUZ7M1DHOity1bsq8mkayicCMi4gJOn87c7URyMoUbEREXEBSUuduJ5GQKNyIiLqBJE/OqqOSmxLfZzEncmjTJ3rpEsoLCjYiIC3B3Ny/3hsQBJ+75lCma70acg8KNiIiL6NABFi9OvOBiyZJmu+a5EWdh+QzFIiKSfTp0gHbtNEOxODeFGxERF+PuDs2bW12FSNbRaSkRERFxKgo3IiIi4lQUbkRERMSpKNyIiIiIU9GAYhERybXsdl35JYkp3IiISK4UEWGudB5/QdCSJc3JCjVnj2vTaSkREcl1IiKgY8fEK52fPGm2R0RYU5fkDAo3IiKSq9jtZo+NYSR+La5twABzO3FNCjciIpKrbNmSuMcmPsOAEyfM7cQ1KdyIiEiucvp05m4nzkfhRkREcpWgoMzdTpyPwo2IiOQqTZqYV0XZbEm/brNBcLC5nbgmhRsREclV3N3Ny70hccCJez5liua7cWUKNyIikut06ACLF0OJEgnbS5Y02zXPjWvTJH4iIpIrdegA7dpphmJJTOFGRERyLXd3aN7c6iokp1G4ERERsZjWyMpcCjciIiIW0hpZmU8DikVERCyiNbKyhsKNiIiIBbRGVtZRuBEREbGA1sjKOgo3IiIiFtAaWVlH4UZERMQCWiMr6yjciIiIWEBrZGUdhRsRERELaI2srKNwIyIiYhGtkZU1NImfiIiIhbRGVuZTuBEREbGYs6yRlVOWkVC4ERERkQzLSctIaMyNiIiIZEhOW0ZC4UZERETSLScuI6FwIyIiIumWE5eRULgRERGRdMuJy0go3IiIiEi65cRlJBRuREREJN1y4jISCjciIiKSbjlxGQmFGxEREcmQnLaMhCbxExERkQzLSctIKNyIiIhIpsgpy0jotJSIiIg4FYUbERERcSoKNyIiIuJUFG5ERETEqSjciIiIiFNRuBERERGnonAjIiIiTkXhRkRERJyKwo2IiIg4FZebodgwDACio6MtrkRERERSK+57O+57PCUuF24uX74MQHBwsMWViIiISFpdvnwZf3//FLexGamJQE4kNjaWU6dOkT9/fmx3r80ugJmOg4ODOXHiBH5+flaX4/L0+8hZ9PvIefQ7yVmy6vdhGAaXL1+mePHiuLmlPKrG5Xpu3NzcKFmypNVl5Ap+fn76H0UOot9HzqLfR86j30nOkhW/j3v12MTRgGIRERFxKgo3IiIi4lQUbiQRLy8vRo0ahZeXl9WlCPp95DT6feQ8+p3kLDnh9+FyA4pFRETEuannRkRERJyKwo2IiIg4FYUbERERcSoKNyIiIuJUFG7EYcKECdSrV4/8+fNTrFgx2rdvz8GDB60uS4C3334bm83GgAEDrC7FpZ08eZLnnnuOwoUL4+PjQ7Vq1fjvf/9rdVkuyW63M2LECMqUKYOPjw/lypVj7NixqVp3SDLuhx9+oE2bNhQvXhybzcbSpUsTvG4YBiNHjiQoKAgfHx9atmzJn3/+mW31KdyIw3/+8x969+7NTz/9xPr167l16xaPPvooV69etbo0l7Zz505mzZpF9erVrS7FpV28eJHGjRuTJ08eVq9eze+//86kSZMoWLCg1aW5pHfeeYcZM2Ywbdo0Dhw4wDvvvMO7777Lhx9+aHVpLuHq1avUqFGDjz76KMnX3333XT744ANmzpzJzz//TN68eWnVqhU3btzIlvp0Kbgk69y5cxQrVoz//Oc/NG3a1OpyXNKVK1eoXbs206dP56233qJmzZpMmTLF6rJc0pAhQ9i6dStbtmyxuhQBnnzySQICAvjkk08cbU8//TQ+Pj7Mnz/fwspcj81mY8mSJbRv3x4we22KFy/O66+/zqBBgwCIiooiICCAefPm0aVLlyyvST03kqyoqCgAChUqZHElrqt379488cQTtGzZ0upSXN7y5cupW7cunTp1olixYtSqVYs5c+ZYXZbLatSoERs3buTQoUMA7N27lx9//JHHHnvM4srkyJEjREZGJvj/lr+/Pw0aNGD79u3ZUoPLLZwpqRMbG8uAAQNo3LgxVatWtbocl/T111+za9cudu7caXUpAvz111/MmDGDgQMH8uabb7Jz50769euHp6cnYWFhVpfncoYMGUJ0dDQVK1bE3d0du93OuHHj6Nq1q9WlubzIyEgAAgICErQHBAQ4XstqCjeSpN69e/Pbb7/x448/Wl2KSzpx4gT9+/dn/fr1eHt7W12OYAb+unXrMn78eABq1arFb7/9xsyZMxVuLPDNN9/w5Zdf8tVXX1GlShX27NnDgAEDKF68uH4fotNSklifPn1YuXIlmzZtomTJklaX45J++eUXzp49S+3atfHw8MDDw4P//Oc/fPDBB3h4eGC3260u0eUEBQVRuXLlBG2VKlXi+PHjFlXk2t544w2GDBlCly5dqFatGs8//zyvvfYaEyZMsLo0lxcYGAjAmTNnErSfOXPG8VpWU7gRB8Mw6NOnD0uWLOH777+nTJkyVpfkslq0aMG+ffvYs2eP41a3bl26du3Knj17cHd3t7pEl9O4ceNEUyMcOnSIkJAQiypybdeuXcPNLeFXmLu7O7GxsRZVJHHKlClDYGAgGzdudLRFR0fz888/07Bhw2ypQaelxKF379589dVXLFu2jPz58zvOjfr7++Pj42Nxda4lf/78icY65c2bl8KFC2sMlEVee+01GjVqxPjx4+ncuTM7duxg9uzZzJ492+rSXFKbNm0YN24cpUqVokqVKuzevZvJkyfzwgsvWF2aS7hy5QqHDx92PD9y5Ah79uyhUKFClCpVigEDBvDWW29Rvnx5ypQpw4gRIyhevLjjiqosZ4j8PyDJ29y5c60uTQzDaNasmdG/f3+ry3BpK1asMKpWrWp4eXkZFStWNGbPnm11SS4rOjra6N+/v1GqVCnD29vbKFu2rDFs2DDj5s2bVpfmEjZt2pTk90VYWJhhGIYRGxtrjBgxwggICDC8vLyMFi1aGAcPHsy2+jTPjYiIiDgVjbkRERERp6JwIyIiIk5F4UZEREScisKNiIiIOBWFGxEREXEqCjciIiLiVBRuRERExKko3IiIS7LZbCxdutTqMkQkCyjciEi26969OzabLdGtdevWVpcmIk5Aa0uJiCVat27N3LlzE7R5eXlZVI2IOBP13IiIJby8vAgMDExwK1iwIGCeMpoxYwaPPfYYPj4+lC1blsWLFyd4/759+3j44Yfx8fGhcOHC9OzZkytXriTY5tNPP6VKlSp4eXkRFBREnz59Erx+/vx5nnrqKXx9fSlfvjzLly93vHbx4kW6du1K0aJF8fHxoXz58onCmIjkTAo3IpIjjRgxgqeffpq9e/fStWtXunTpwoEDBwC4evUqrVq1omDBguzcuZNFixaxYcOGBOFlxowZ9O7dm549e7Jv3z6WL1/Offfdl+AYo0ePpnPnzvz66688/vjjdO3alQsXLjiO//vvv7N69WoOHDjAjBkzKFKkSPb9AEQk/bJtiU4Rkf8XFhZmuLu7G3nz5k1wGzdunGEY5gr1r7zySoL3NGjQwOjVq5dhGIYxe/Zso2DBgsaVK1ccr69atcpwc3MzIiMjDcMwjOLFixvDhg1LtgbAGD58uOP5lStXDMBYvXq1YRiG0aZNG6NHjx6Z84FFJFtpzI2IWOKhhx5ixowZCdoKFSrkeNywYcMErzVs2JA9e/YAcODAAWrUqEHevHkdrzdu3JjY2FgOHjyIzWbj1KlTtGjRIsUaqlev7nicN29e/Pz8OHv2LAC9evXi6aefZteuXTz66KO0b9+eRo0apeuzikj2UrgREUvkzZs30WmizOLj45Oq7fLkyZPguc1mIzY2FoDHHnuMY8eO8d1337F+/XpatGhB7969ee+99zK9XhHJXBpzIyI50k8//ZToeaVKlQCoVKkSe/fu5erVq47Xt27dipubGxUqVCB//vyULl2ajRs3ZqiGokWLEhYWxvz585kyZQqzZ8/O0P5EJHuo50ZELHHz5k0iIyMTtHl4eDgG7S5atIi6devy4IMP8uWXX7Jjxw4++eQTALp27cqoUaMICwsjPDycc+fO0bdvX55//nkCAgIACA8P55VXXqFYsWI89thjXL58ma1bt9K3b99U1Tdy5Ejq1KlDlSpVuHnzJitXrnSEKxHJ2RRuRMQSa9asISgoKEFbhQoV+OOPPwDzSqavv/6aV199laCgIBYsWEDlypUB8PX1Ze3atfTv35969erh6+vL008/zeTJkx37CgsL48aNG7z//vsMGjSIIkWK0LFjx1TX5+npydChQzl69Cg+Pj40adKEr7/+OhM+uYhkNZthGIbVRYiIxGez2ViyZAnt27e3uhQRyYU05kZEREScisKNiIiIOBWNuRGRHEdny0UkI9RzIyIiIk5F4UZEREScisKNiIiIOBWFGxEREXEqCjciIiLiVBRuRERExKko3IiIiIhTUbgRERERp6JwIyIiIk7l/wDiUeFgZaekkgAAAABJRU5ErkJggg==", "text/plain": [ "
" ] @@ -797,13 +825,13 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 25, "id": "af51178e-fe0b-40ca-9260-2190fb52d960", "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABZ5klEQVR4nO3deVhUZf8G8HsYZNgRQXYERHNFTFTcSEsLtUjcQjNFs0xzwdBUcs9XKTXDLX3151a5kIZmuRSSlrvmbu6KG4KICwgqyMz5/XFeRkcGnIGBMzD357rmcuaZc858z8zU3JzznOeRCYIggIiIiMiEmEldABEREVF5YwAiIiIik8MARERERCaHAYiIiIhMDgMQERERmRwGICIiIjI5DEBERERkchiAiIiIyOQwABEREZHJYQAiMoD+/fvD19e3ROtOmTIFMpnMsAUZmatXr0Imk2HlypXl+rq7du2CTCbDrl271G26flZlVbOvry/69+9v0G0Skf4YgKhSk8lkOt2e/4EkKq19+/ZhypQpePDggdSlEFERzKUugKgs/fDDDxqPv//+eyQmJhZqr1evXqleZ+nSpVCpVCVad8KECRg3blypXp90V5rPSlf79u3D1KlT0b9/f1StWlXjufPnz8PMjH97EkmNAYgqtQ8++EDj8YEDB5CYmFio/UWPHj2CtbW1zq9TpUqVEtUHAObm5jA353+K5aU0n5UhKBQKSV+/osjJyYGNjY3UZVAlxj9DyOS1a9cODRs2xJEjR/Daa6/B2toaX3zxBQDgl19+wdtvvw0PDw8oFAr4+/tj2rRpUCqVGtt4sV9JQf+R2bNnY8mSJfD394dCoUCzZs1w+PBhjXW19QGSyWQYNmwYNm3ahIYNG0KhUKBBgwbYvn17ofp37dqFpk2bwtLSEv7+/vjvf/+rc7+i3bt3o2fPnqhRowYUCgW8vb3x2Wef4fHjx4X2z9bWFikpKQgPD4etrS2qV6+O0aNHF3ovHjx4gP79+8PBwQFVq1ZFZGSkTqeC/vnnH8hkMqxatarQc7///jtkMhl+++03AMC1a9fw6aefok6dOrCysoKTkxN69uyJq1evvvR1tPUB0rXmkydPon///qhZsyYsLS3h5uaGDz/8EHfv3lUvM2XKFHz++ecAAD8/P/Vp1oLatPUBunLlCnr27Ilq1arB2toaLVq0wJYtWzSWKejP9NNPP2H69Onw8vKCpaUl2rdvj0uXLr10v/V5zx48eIDPPvsMvr6+UCgU8PLyQr9+/ZCRkaFe5smTJ5gyZQpeeeUVWFpawt3dHd26dcPly5c16n3x9LK2vlUF36/Lly+jc+fOsLOzQ58+fQDo/h0FgHPnzuG9995D9erVYWVlhTp16mD8+PEAgJ07d0Imk2Hjxo2F1luzZg1kMhn279//0veRKg/+2UkE4O7du+jUqRN69eqFDz74AK6urgCAlStXwtbWFtHR0bC1tcWff/6JSZMmISsrC7NmzXrpdtesWYOHDx/ik08+gUwmw8yZM9GtWzdcuXLlpUci9uzZg4SEBHz66aews7PDvHnz0L17d1y/fh1OTk4AgGPHjqFjx45wd3fH1KlToVQq8eWXX6J69eo67ff69evx6NEjDBkyBE5OTjh06BDmz5+PmzdvYv369RrLKpVKhIaGIjg4GLNnz8aOHTvwzTffwN/fH0OGDAEACIKALl26YM+ePRg8eDDq1auHjRs3IjIy8qW1NG3aFDVr1sRPP/1UaPn4+Hg4OjoiNDQUAHD48GHs27cPvXr1gpeXF65evYpFixahXbt2OHPmjF5H7/SpOTExEVeuXMGAAQPg5uaGf//9F0uWLMG///6LAwcOQCaToVu3brhw4QLWrl2Lb7/9Fs7OzgBQ5Gdy+/ZttGrVCo8ePcKIESPg5OSEVatW4d1338WGDRvQtWtXjeW/+uormJmZYfTo0cjMzMTMmTPRp08fHDx4sNj91PU9y87ORkhICM6ePYsPP/wQTZo0QUZGBjZv3oybN2/C2dkZSqUS77zzDpKSktCrVy9ERUXh4cOHSExMxOnTp+Hv76/z+18gPz8foaGhaNOmDWbPnq2uR9fv6MmTJxESEoIqVapg0KBB8PX1xeXLl/Hrr79i+vTpaNeuHby9vbF69epC7+nq1avh7++Pli1b6l03VWACkQkZOnSo8OLXvm3btgIAYfHixYWWf/ToUaG2Tz75RLC2thaePHmibouMjBR8fHzUj5OTkwUAgpOTk3Dv3j11+y+//CIAEH799Vd12+TJkwvVBECwsLAQLl26pG47ceKEAECYP3++ui0sLEywtrYWUlJS1G0XL14UzM3NC21TG237FxsbK8hkMuHatWsa+wdA+PLLLzWWffXVV4WgoCD1402bNgkAhJkzZ6rb8vPzhZCQEAGAsGLFimLriYmJEapUqaLxnuXm5gpVq1YVPvzww2Lr3r9/vwBA+P7779VtO3fuFAAIO3fu1NiX5z8rfWrW9rpr164VAAh///23um3WrFkCACE5ObnQ8j4+PkJkZKT68ciRIwUAwu7du9VtDx8+FPz8/ARfX19BqVRq7Eu9evWE3Nxc9bJz584VAAinTp0q9FrP0/U9mzRpkgBASEhIKLS8SqUSBEEQli9fLgAQ5syZU+Qy2t57QXj238bz72vB92vcuHE61a3tO/raa68JdnZ2Gm3P1yMI4vdLoVAIDx48ULelp6cL5ubmwuTJkwu9DlVuPAVGBLFfxoABAwq1W1lZqe8/fPgQGRkZCAkJwaNHj3Du3LmXbjciIgKOjo7qxyEhIQDEUx4v06FDB42/pBs1agR7e3v1ukqlEjt27EB4eDg8PDzUy9WqVQudOnV66fYBzf3LyclBRkYGWrVqBUEQcOzYsULLDx48WONxSEiIxr5s3boV5ubm6iNCACCXyzF8+HCd6omIiMDTp0+RkJCgbvvjjz/w4MEDREREaK376dOnuHv3LmrVqoWqVavi6NGjOr1WSWp+/nWfPHmCjIwMtGjRAgD0ft3nX7958+Zo06aNus3W1haDBg3C1atXcebMGY3lBwwYAAsLC/VjXb9Tur5nP//8MwIDAwsdJQGgPq36888/w9nZWet7VJohHZ7/DLTVXdR39M6dO/j777/x4YcfokaNGkXW069fP+Tm5mLDhg3qtvj4eOTn57+0XyBVPgxARAA8PT01flQK/Pvvv+jatSscHBxgb2+P6tWrq/9HmZmZ+dLtvvg/44IwdP/+fb3XLVi/YN309HQ8fvwYtWrVKrSctjZtrl+/jv79+6NatWrqfj1t27YFUHj/LC0tC53Geb4eQOxn4u7uDltbW43l6tSpo1M9gYGBqFu3LuLj49Vt8fHxcHZ2xhtvvKFue/z4MSZNmgRvb28oFAo4OzujevXqePDggU6fy/P0qfnevXuIioqCq6srrKysUL16dfj5+QHQ7ftQ1Otre62CKxOvXbum0V7S75Su79nly5fRsGHDYrd1+fJl1KlTx6Cd983NzeHl5VWoXZfvaEH4e1nddevWRbNmzbB69Wp12+rVq9GiRQud/5uhyoN9gIig+VdmgQcPHqBt27awt7fHl19+CX9/f1haWuLo0aMYO3asTpdSy+Vyre2CIJTpurpQKpV48803ce/ePYwdOxZ169aFjY0NUlJS0L9//0L7V1Q9hhYREYHp06cjIyMDdnZ22Lx5M3r37q3xYzt8+HCsWLECI0eORMuWLeHg4ACZTIZevXqV6SXu7733Hvbt24fPP/8cjRs3hq2tLVQqFTp27Fjml9YXKOn3orzfs6KOBL3Yab6AQqEoNDyAvt9RXfTr1w9RUVG4efMmcnNzceDAASxYsEDv7VDFxwBEVIRdu3bh7t27SEhIwGuvvaZuT05OlrCqZ1xcXGBpaan1CiBdrgo6deoULly4gFWrVqFfv37q9sTExBLX5OPjg6SkJGRnZ2scUTl//rzO24iIiMDUqVPx888/w9XVFVlZWejVq5fGMhs2bEBkZCS++eYbdduTJ09KNPCgrjXfv38fSUlJmDp1KiZNmqRuv3jxYqFt6nMayMfHR+v7U3CK1cfHR+dtFUfX98zf3x+nT58udlv+/v44ePAgnj59WmRn/oIjUy9u/8UjWsXR9Ttas2ZNAHhp3QDQq1cvREdHY+3atXj8+DGqVKmicXqVTAdPgREVoeAv7ef/ss7Ly8N3330nVUka5HI5OnTogE2bNuHWrVvq9kuXLmHbtm06rQ9o7p8gCJg7d26Ja+rcuTPy8/OxaNEidZtSqcT8+fN13ka9evUQEBCA+Ph4xMfHw93dXSOAFtT+4hGP+fPnF3l0wRA1a3u/ACAuLq7QNgvGr9ElkHXu3BmHDh3SuAQ7JycHS5Ysga+vL+rXr6/rrhRL1/ese/fuOHHihNbLxQvW7969OzIyMrQeOSlYxsfHB3K5HH///bfG8/r896Prd7R69ep47bXXsHz5cly/fl1rPQWcnZ3RqVMn/Pjjj1i9ejU6duyovlKPTAuPABEVoVWrVnB0dERkZCRGjBgBmUyGH374wWCnoAxhypQp+OOPP9C6dWsMGTIESqUSCxYsQMOGDXH8+PFi161bty78/f0xevRopKSkwN7eHj///LNO/ZOKEhYWhtatW2PcuHG4evUq6tevj4SEBL37x0RERGDSpEmwtLTEwIEDC50aeeedd/DDDz/AwcEB9evXx/79+7Fjxw718ABlUbO9vT1ee+01zJw5E0+fPoWnpyf++OMPrUcEg4KCAADjx49Hr169UKVKFYSFhWkd2G/cuHFYu3YtOnXqhBEjRqBatWpYtWoVkpOT8fPPPxts1Ghd37PPP/8cGzZsQM+ePfHhhx8iKCgI9+7dw+bNm7F48WIEBgaiX79++P777xEdHY1Dhw4hJCQEOTk52LFjBz799FN06dIFDg4O6NmzJ+bPnw+ZTAZ/f3/89ttvSE9P17lmfb6j8+bNQ5s2bdCkSRMMGjQIfn5+uHr1KrZs2VLov4V+/fqhR48eAIBp06bp/2ZS5VDu150RSaioy+AbNGigdfm9e/cKLVq0EKysrAQPDw9hzJgxwu+///7SS6sLLvWdNWtWoW0C0LjktqjL4IcOHVpo3RcvoRYEQUhKShJeffVVwcLCQvD39xf+7//+Txg1apRgaWlZxLvwzJkzZ4QOHToItra2grOzs/Dxxx+rL7d/8TJlGxubQutrq/3u3btC3759BXt7e8HBwUHo27evcOzYMZ0ugy9w8eJFAYAAQNizZ0+h5+/fvy8MGDBAcHZ2FmxtbYXQ0FDh3Llzhd4fXS6D16fmmzdvCl27dhWqVq0qODg4CD179hRu3bpV6DMVBEGYNm2a4OnpKZiZmWlcEq/tM7x8+bLQo0cPoWrVqoKlpaXQvHlz4bffftNYpmBf1q9fr9Gu7bJybXR9zwrej2HDhgmenp6ChYWF4OXlJURGRgoZGRnqZR49eiSMHz9e8PPzE6pUqSK4ubkJPXr0EC5fvqxe5s6dO0L37t0Fa2trwdHRUfjkk0+E06dP6/z9EgTdv6OCIAinT59Wfz6WlpZCnTp1hIkTJxbaZm5uruDo6Cg4ODgIjx8/LvZ9o8pLJghG9OcsERlEeHg4/v33X639U4hMXX5+Pjw8PBAWFoZly5ZJXQ5JhH2AiCq4F6cEuHjxIrZu3Yp27dpJUxCRkdu0aRPu3Lmj0bGaTA+PABFVcO7u7ur5qa5du4ZFixYhNzcXx44dQ+3ataUuj8hoHDx4ECdPnsS0adPg7Oxc4sErqXJgJ2iiCq5jx45Yu3Yt0tLSoFAo0LJlS8yYMYPhh+gFixYtwo8//ojGjRtrTMZKpolHgIiIiMjksA8QERERmRwGICIiIjI57AOkhUqlwq1bt2BnZ1eqmY2JiIio/AiCgIcPH8LDw+Olg4gyAGlx69YteHt7S10GERERlcCNGzfg5eVV7DIMQFrY2dkBEN9Ae3t7iashIiIiXWRlZcHb21v9O14cBiAtCk572dvbMwARERFVMLp0X2EnaCIiIjI5DEBERERkchiAiIiIyOQwABEREZHJYQAiIiIik8MARERERCaHAYiIiIhMDgMQERERmRwGICIiIjI5HAmaiIiIyoVSCezeDaSmAu7uQEgIIJdLUwsDEBEREZW5hAQgKgq4efNZm5cXMHcu0K1b+dfDU2BERERUphISgB49NMMPAKSkiO0JCeVfEwMQERERlRmlUjzyIwiFnytoGzlSXK48MQARERFRmdm9u/CRn+cJAnDjhrhceWIfICIiIiNnTJ2H9ZWaatjlDIUBiIiIyIgZW+dhfbm7G3Y5Q+EpMCIiIiNljJ2H9RUSIgY2mUz78zIZ4O0tLleeGICIiIiMkLF2HtaXXC4erQIKh6CCx3Fx5X9KjwGIiIjICBlr5+GS6NYN2LAB8PTUbPfyEtulOJXHPkBERERGyFg7D5dUt25Aly7G05mbAYiIiMgIGWvn4dKQy4F27aSuQsRTYEREREbIWDsPVxaSB6CFCxfC19cXlpaWCA4OxqFDh4pc9unTp/jyyy/h7+8PS0tLBAYGYvv27aXaJhERkTEy1s7DlYWkASg+Ph7R0dGYPHkyjh49isDAQISGhiI9PV3r8hMmTMB///tfzJ8/H2fOnMHgwYPRtWtXHDt2rMTbJCIiMlbG2Hm4spAJgrYL7MpHcHAwmjVrhgULFgAAVCoVvL29MXz4cIwbN67Q8h4eHhg/fjyGDh2qbuvevTusrKzw448/lmib2mRlZcHBwQGZmZmwt7cv7W4SERGVSkUeCbo86fP7LVkn6Ly8PBw5cgQxMTHqNjMzM3To0AH79+/Xuk5ubi4sLS012qysrLBnz54Sb7Ngu7m5uerHWVlZJdonIiIyLpUlOBhT5+HKQrJTYBkZGVAqlXB1ddVod3V1RVpamtZ1QkNDMWfOHFy8eBEqlQqJiYlISEhA6v+uASzJNgEgNjYWDg4O6pu3t3cp946IiKSWkAD4+gKvvw68/774r69vxRg9mcqe5J2g9TF37lzUrl0bdevWhYWFBYYNG4YBAwbAzKx0uxETE4PMzEz17caNGwaqmIiIpFAZppCgsiVZAHJ2doZcLsft27c12m/fvg03Nzet61SvXh2bNm1CTk4Orl27hnPnzsHW1hY1a9Ys8TYBQKFQwN7eXuNGREQVU2WZQoLKlmQByMLCAkFBQUhKSlK3qVQqJCUloWXLlsWua2lpCU9PT+Tn5+Pnn39Gly5dSr1NIiKqHCrTFBJUdiQdCTo6OhqRkZFo2rQpmjdvjri4OOTk5GDAgAEAgH79+sHT0xOxsbEAgIMHDyIlJQWNGzdGSkoKpkyZApVKhTFjxui8TSIiqtwq2xQSVDYkDUARERG4c+cOJk2ahLS0NDRu3Bjbt29Xd2K+fv26Rv+eJ0+eYMKECbhy5QpsbW3RuXNn/PDDD6hatarO2yQiosqtMk4hQYYn6ThAxorjABERVVxKpXi1V0qK9n5AMpk4kGBycsW8JJ6Kps/vd4W6CoyIiMqHUgns2gWsXSv+W5E6DHMKCdIFAxAREWmoDOPncAoJehmeAtOCp8CIyFQVjJ/z4i9DwZGTihYeKstI0KQbfX6/GYC0YAAiIlNU0HemqEvI2XeGjB37ABERkd44fg6ZEgYgIiICwPFzyLQwABEREQCOn0OmhQGIiIgAiB2EvbwKXzpeQCYDvL3F5YgqOgYgIiICwPFzyLQwABERkRrHzyFTIelcYEREZHy6dQO6dOH4OVS5MQAREVEhcjnQrp3UVRCVHQYgIiID4sjDRBUDAxARkYEkJABRUZqDCXp5iR2L2XeGyLiwEzQRkQEUzKH14kjKKSlie0WaSJTIFDAAERGVklIpHvnRNrNiQdvIkeJyRGQcGICIiEqJc2gRVTwMQEREpcQ5tIgqHgYgIqJS4hxaRBUPAxARUSlxDi2iiocBiIiolDiHFlHFwwBERGQAnEOLqGLhQIhERAbCObSIKg4GICIiA+IcWkQVA0+BERERkclhACIiIiKTw1NgRGQUOIs6EZUnBiAikhxnUSei8sZTYEQkKc6iTkRSYAAiIslwFnUikgoDEBFJhrOoE5FUGICISDKcRZ2IpMIARESS4SzqRCQVBiAikgxnUSciqTAAEZFkOIs6EUmFAYiIJMVZ1IlIChwIkYgkx1nUiai8MQARkVHgLOpEldvTp0ByMnDhAnDxItCyJdCihXT1MAARERGRQSiVwLVrYsC5ePFZ2Ll4Ebh6VXNQ0y++YAAiIiKiCkKlEqeqeT7cFNwuXxaP9BTF2hqoXVu8NWxYfjVrwwBEREREGgQBSEvTfiTn0iXgyZOi11UoAH9/MeS88sqzwFO7NuDhUfSwF+WNAYiIiMgECQKQkVH4KM6FC2LIyc4uel1zc6BmTc1wUxB2vLwqxgUMDEBERESV2IMH2o/kXLwoPlcUMzPA11cz5BQEHR8fMQRVZBW8fCIiIsrO1n4k5+JF8ShPcby9tZ+u8vMTT2dVVgxARERUaT15Ih7lKO50TkXy6JHY0fjFIzkvmzDY3V376Sp/f8DKqnxqNzYMQEQVnFLJAQSp8hIEMbzcv//s9uCB7veL66xb2Tg7Fz6KU7s2UKsWYGcndXXGhwGIqAJLSACiooCbN5+1eXmJ82txCgkyFvn5QGZmyQLMgweaY8eUhEwG2Noaz9VHpVGlyrPOxy+GnapVpa6uYmEAIqqgEhKAHj3Ev5Cfl5IitnMeLTKkJ0/0CzDPP374sPSvb2EBODqKt6pVX37/+cd2dmKHXqLnyQThxf99UlZWFhwcHJCZmQl7e3upyyEqRKkUr854/sjP82Qy8UhQcjJPh9HLCQKQni5e+vzi7eZNMcTk5pb+dWxtdQ8wL4YZS8vKcQSHypY+v988AkRUAe3eXXT4AcQftBs3xOU4vxYB4ui9t25pDzmXL+vWSdjMTAwm+gaYgnWqVCnDHSTSEwMQUQX0sis+9F2OKgelUgy+RYWc4joEy2Ti2C61aolXBtWqJd5q1ACqVeOpJKp8GICIKiB3d8MuRxXH06fipJLaQk5ycvHzMMnl4tguBeHm+Zuvb+Ue84XoRZIHoIULF2LWrFlIS0tDYGAg5s+fj+bNmxe5fFxcHBYtWoTr16/D2dkZPXr0QGxsLCwtLQEAU6ZMwdSpUzXWqVOnDs6dO1em+0FUnkJCxD4+KSmFO0EDz/oAhYSUf21Uek+eiGFGW8i5dq34q6IsLDSP4Dx/RKdGDZ6GIiogaQCKj49HdHQ0Fi9ejODgYMTFxSE0NBTnz5+Hi4tLoeXXrFmDcePGYfny5WjVqhUuXLiA/v37QyaTYc6cOerlGjRogB07dqgfm1f08bqJXiCXi5e69+ghhp3nQ1BBR9G4OHaANmY5OeJpqaI6Hhd3eYq1deGQU3Dz9OTnTqQLSZPBnDlz8PHHH2PAgAEAgMWLF2PLli1Yvnw5xo0bV2j5ffv2oXXr1nj//fcBAL6+vujduzcOHjyosZy5uTnc3NzKfgeIJNStm3ipu7ZxgOLieAm8McjMLDrkvKx/lr299oBTqxbg5sYroohKS7IAlJeXhyNHjiAmJkbdZmZmhg4dOmD//v1a12nVqhV+/PFHHDp0CM2bN8eVK1ewdetW9O3bV2O5ixcvwsPDA5aWlmjZsiViY2NRo0aNImvJzc1F7nPXeGZlZZVy74jKR7duQJculWMk6IcPgUOHgNu3xVM8KpX2f0v6XHkvc/cucOdO8fvs5FT4NFXBzdmZIYeoLEkWgDIyMqBUKuHq6qrR7urqWmR/nffffx8ZGRlo06YNBEFAfn4+Bg8ejC+++EK9THBwMFauXIk6deogNTUVU6dORUhICE6fPg27IsYCj42NLdRviKiikMsr5qXuN24Ae/c+u504IQaHysbVVftRHH9/8coqIpJGheocs2vXLsyYMQPfffcdgoODcenSJURFRWHatGmYOHEiAKBTp07q5Rs1aoTg4GD4+Pjgp59+wsCBA7VuNyYmBtHR0erHWVlZ8Pb2LtudITIhSiVw8qRm4Llxo/ByBZdhy+Xi5dbP/6utTZfnDLWMvuvb24shh3MwERknyQKQs7Mz5HI5bt++rdF++/btIvvvTJw4EX379sVHH30EAAgICEBOTg4GDRqE8ePHw0zLABVVq1bFK6+8gkuXLhVZi0KhgILXfxIZzMOHwIEDz8LOgQOFB9qTy4HAQKB1a6BNG6BVK7H/EhFReZAsAFlYWCAoKAhJSUkIDw8HAKhUKiQlJWHYsGFa13n06FGhkCP/X2eHomb0yM7OxuXLlwv1EyIiw9HldJadHdCypRh4WrcGgoPFqRGIiKQg6Smw6OhoREZGomnTpmjevDni4uKQk5OjviqsX79+8PT0RGxsLAAgLCwMc+bMwauvvqo+BTZx4kSEhYWpg9Do0aMRFhYGHx8f3Lp1C5MnT4ZcLkfv3r0l20+iyiQ/Hzh1SrfTWQVhp3VroGHDitk5m4gqJ0kDUEREBO7cuYNJkyYhLS0NjRs3xvbt29Udo69fv65xxGfChAmQyWSYMGECUlJSUL16dYSFhWH69OnqZW7evInevXvj7t27qF69Otq0aYMDBw6gevXq5b5/RJWBrqezGjfWDDyenpKUS0SkE84GrwVngydTdv265tGdkycLn86yt9c8ndW8OU9nEZH0OBs8EekkP7/w1VnaZpn39dU8utOgAU9nEVHFxgBEZEKysjRPZx08qP101quvagYeDw9p6iUiKisMQESVlCAUPp116lTh01kODoVPZ9nYSFMzEVF5YQAiqiTy88XLz58PPCkphZfz9RXH3SkIPPXr83QWEZkeBiCiCiw5Gfj1V2DLFjHw5ORoPs/TWURE2jEAEVUgSqXYb+fXX8Xbv/9qPs/TWUREumEAIjJyDx8Cf/whBp6tWzVnGJfLxaATFgaEhopXZ2mZEYaIiF7AAERkhK5de3aUZ9cuIC/v2XMODkDHjmLo6dQJqFZNsjKJiCosBiAiI6BSAYcOPQs9p05pPl+rlhh4wsLEDsxVqkhTJxFRZcEARCSR7GwgMfFZJ+b09GfPmZk9O7UVFgbUqQPIZNLVSkRU2TAAEZWjGzeeHeXZuRPIzX32nL295qktJyfp6iQiquwYgIjKkEoF/PPPs9Bz4oTm8zVrPjvKExICWFhIUycRkalhACIysJwcYMeOZ6e20tKePWdmJl6mXhB66tXjqS0iIikwAJHJUiqB3buB1FTA3V08AlPSEZFv3gR++00MPX/+CTx58uw5OzvxEvWwMKBzZ8DZ2TD1ExFRyTEAkUlKSACiojRnPvfyAubOBbp1e/n6KhVw9OizU1vHjmk+7+v77ChP27Y8tUVEZGwYgMjkJCQAPXqIk4U+LyVFbN+wQXsIevQISEoSA89vv4lHjgrIZECLFs9CT4MGPLVFRGTMZILw4s8AZWVlwcHBAZmZmbC3t5e6HDIgpVI8OvP8kZ/nyWTikaDkZPF02K1bz05tJSUBjx8/W9bGRvPUlotLuewCEREVQZ/fbx4BIpOye3fR4QcQjwrduAF89JE4GOGRI5rP16jx7ChPu3aAQlGm5RIRURlhACKT8vxpq+KsXCn+K5OJE4oWhJ6AAJ7aIiKqDBiAyKS4u+u2XJs2wIABwNtvA66uZVsTERGVPwYgMilt2oiXoWdkFL2Ml5c4AWlJL4knIiLjZyZ1AUTlQakUr+5q3rzo8COTibe5cxl+iIgqOwYgqtSePhX78zRoAPTsKY7XY20NvPNO4dNhXl5FXwJPRESVC0+BUaX06BGwbBkwezZw/brYVrUqMGIEMHy4eBrMkCNBExFRxcIARJVKZibw3XfAt98Cd+6Iba6uwKhRwCefiDOuF5DLxUvZiYjI9DAAUaWQni723VmwAMjKEtt8fYExY8SruSwtJS2PiIiMDAMQVWg3boinuZYufTZKc716QEwM0KsXUKWKtPUREZFxYgCiCunCBeDrr4EffhA7OgNA06bA+PHAu+8CZuzeT0RExWAAogrl2DEgNla8WqtgFrvXXwe++AJo356jNBMRkW4YgKhC2LMHmDED2LbtWVtYmHiqq2VL6eoiIqKKiQGIjJYgAL//Lgaf3bvFNjMzICICGDcOaNRI2vqIiKjiYgAio6NUAhs3isHn2DGxzcIC6N8f+PxzoFYtScsjIqJKgAGIjMbTp8Dq1cBXXwHnz4tt1tbA4MFAdDTg6SltfUREVHkwAJHkHj8WR22eNavoUZuJiIgMiQGIJFMwanNcnDiQIQC4uYlHewYPBuzsJC2PiIgqMQYgKndFjdo8dqzYz4ejNhMRUVljAKJyo23U5vr1xUvZIyI4ajMREZUfBiAqc9pGbW7WTBy8kKM2ExGRFBiAqMwcPy6O2rx+/bNRm994Qzziw1GbiYhISgxAZHB79ojBZ+vWZ23vvisGnxYtpKuLiIioAAMQGURRozb36iWO2hwQIG19REREz2MAolIpbtTmMWMAf39JyyMiItKKAYhKhKM2ExFRRcYARHpLThY7MScni48dHcURm0eMAJycpK2NiIhIFwxApLdJk8Tw4+oKjB4NfPIJR20mIqKKhQGI9JKcDKxdK97fsgUICpK2HiIiopLgEHSkl9mzxY7Pb77J8ENERBUXAxDp7PZtYPly8X5MjLS1EBERlQYDEOksLg548gQIDgbatZO6GiIiopJjACKdZGYC330n3o+J4TQWRERUsTEAkU6++w7IyhJnbw8Lk7oaIiKi0pE8AC1cuBC+vr6wtLREcHAwDh06VOzycXFxqFOnDqysrODt7Y3PPvsMT548KdU2qXiPH4unvwBxWgvO3k5ERBWdpD9l8fHxiI6OxuTJk3H06FEEBgYiNDQU6enpWpdfs2YNxo0bh8mTJ+Ps2bNYtmwZ4uPj8cUXX5R4m/Ryy5cD6emAj484txcREVFFJxMEQZDqxYODg9GsWTMsWLAAAKBSqeDt7Y3hw4dj3LhxhZYfNmwYzp49i6SkJHXbqFGjcPDgQezZs6dE29QmKysLDg4OyMzMhL29fWl3s0J7+hSoXRu4dg1YsAAYOlTqioiIiLTT5/dbsiNAeXl5OHLkCDp06PCsGDMzdOjQAfv379e6TqtWrXDkyBH1Ka0rV65g69at6Ny5c4m3ScVbt04MPy4uwIcfSl0NERGRYUg2EnRGRgaUSiVcXV012l1dXXHu3Dmt67z//vvIyMhAmzZtIAgC8vPzMXjwYPUpsJJsEwByc3ORm5urfpyVlVXS3apUVCpxslMAGDkSsLKStBwiIiKDqVDdWXft2oUZM2bgu+++w9GjR5GQkIAtW7Zg2rRppdpubGwsHBwc1Ddvb28DVVyx/forcOYMYG8PfPqp1NUQEREZjt4ByNfXF19++SWuX79eqhd2dnaGXC7H7du3Ndpv374NNzc3retMnDgRffv2xUcffYSAgAB07doVM2bMQGxsLFQqVYm2CQAxMTHIzMxU327cuFGqfasMBAGIjRXvf/op4OAgbT1ERESGpHcAGjlyJBISElCzZk28+eabWLduncbpI11ZWFggKChIo0OzSqVCUlISWrZsqXWdR48eweyFa7DlcjkAQBCEEm0TABQKBezt7TVupm7XLuDgQcDSUjz9RUREVJmUKAAdP34chw4dQr169TB8+HC4u7tj2LBhOHr0qF7bio6OxtKlS7Fq1SqcPXsWQ4YMQU5ODgYMGAAA6NevH2Kem3QqLCwMixYtwrp165CcnIzExERMnDgRYWFh6iD0sm2SbgqO/gwcCLzQpYqIiKjiE0opLy9PiIuLExQKhWBmZiYEBgYKy5YtE1QqlU7rz58/X6hRo4ZgYWEhNG/eXDhw4ID6ubZt2wqRkZHqx0+fPhWmTJki+Pv7C5aWloK3t7fw6aefCvfv39d5m7rIzMwUAAiZmZl6rVdZ/POPIACCIJcLQnKy1NUQERHpRp/f7xKPA/T06VNs3LgRK1asQGJiIlq0aIGBAwfi5s2bWLhwId544w2sWbPGsGmtnJj6OEA9egA//wz07Qt8/73U1RAREelGn99vvS+DP3r0KFasWIG1a9fCzMwM/fr1w7fffou6deuql+natSuaNWumf+UkuXPngIQE8f7YsdLWQkREVFb0DkDNmjXDm2++iUWLFiE8PBxVqlQptIyfnx96cc6ECmnmTPEKsC5dgAYNpK6GiIiobOh9CuzatWvw8fEpq3qMgqmeArtxA6hZE8jPBw4cAIKDpa6IiIhId2U6FUZ6ejoOHjxYqP3gwYP4559/9N0cGZFvvhHDz+uvM/wQEVHlpncAGjp0qNaBAlNSUjCUM2VWWBkZwNKl4v3nRh4gIiKqlPQOQGfOnEGTJk0Ktb/66qs4c+aMQYqi8jdvHvDoERAUBDw3lywREVGlpHcAUigUhaaaAIDU1FSYm0s2tyqVwsOHwPz54v2YGEAmk7YeIiKisqZ3AHrrrbfUc2cVePDgAb744gu8+eabBi2Oysd//ws8eADUqQN07Sp1NURERGVP70M2s2fPxmuvvQYfHx+8+uqrAIDjx4/D1dUVP/zwg8ELpLKVmwvMmSPeHzsWMNM7EhMREVU8egcgT09PnDx5EqtXr8aJEydgZWWFAQMGoHfv3lrHBCLjtmoVkJoKeHkBffpIXQ0REVH5KFGnHRsbGwwaNMjQtVA5y88XBz4EgNGjAQsLaeshIiIqLyXutXzmzBlcv34deXl5Gu3vvvtuqYui8rFhA3D5MuDkBHz0kdTVEBERlR+9A9CVK1fQtWtXnDp1CjKZDAUDScv+d+mQUqk0bIVUJgQB+Oor8X5UFGBjI209RERE5UnvLq9RUVHw8/NDeno6rK2t8e+//+Lvv/9G06ZNsWvXrjIokcrCtm3AiROArS0wbJjU1RAREZUvvY8A7d+/H3/++SecnZ1hZmYGMzMztGnTBrGxsRgxYgSOHTtWFnWSgcXGiv8OHgw4OkpbCxERUXnT+wiQUqmEnZ0dAMDZ2Rm3bt0CAPj4+OD8+fOGrY7KxO7dwJ49Yqfnzz6TuhoiIqLyp/cRoIYNG+LEiRPw8/NDcHAwZs6cCQsLCyxZsgQ1a9YsixrJwAqO/vTvD3h46L++UimGqNRUwN0dCAkB5HKDlkhERFSm9A5AEyZMQE5ODgDgyy+/xDvvvIOQkBA4OTkhPj7e4AWSYR0/Lvb/MTMDxozRf/2EBLHT9M2bz9q8vIC5c4Fu3QxWJhERUZmSCQWXcZXCvXv34OjoqL4SrKLLysqCg4MDMjMzYW9vL3U5BtWrFxAfL/67dq1+6yYkAD16iFeQPa/gY9+wgSGIiIiko8/vt159gJ4+fQpzc3OcPn1ao71atWqVJvxUZpcuAevXi/fHjdNvXaVSPPKjLS4XtI0cKS5HRERk7PQKQFWqVEGNGjU41k8FNXMmoFIBnTsDgYH6rbt7t+ZprxcJAnDjhrgcERGRsdP7KrDx48fjiy++wL1798qiHiojt26J834BQEyM/uunphp2OSIiIinp3Ql6wYIFuHTpEjw8PODj4wObF4YQPnr0qMGKI8OZMwfIywPatBFv+nJ3N+xyREREUtI7AIWHh5dBGVSW7t0DFi8W75fk6A8gXuru5QWkpGjvBySTic+HhJS8TiIiovKidwCaPHlyWdRBZWjBAiAnR+z306lTybYhl4uXuvfoIYad50NQQf/3uDiOB0RERBWD3n2AqGLJyQHmzRPvjxv3LKyURLdu4qXunp6a7V5evASeiIgqFr2PAJmZmRV7yTuvEDMuS5cCd+8C/v7i0ZvS6tYN6NKFI0ETEVHFpncA2rhxo8bjp0+f4tixY1i1ahWmTp1qsMKo9PLygG++Ee+PGQOY6/1payeXA+3aGWZbREREUjDISNAAsGbNGsTHx+OXX34xxOYkVVlGgl6+HBg4UDxKk5wMKBRSV0RERFR2ymwk6OK0aNECSUlJhtoclZJSCXz9tXg/Oprhh4iI6HkGCUCPHz/GvHnz4Pli71iSzMaNwIULgKMj8MknUldDRERkXPTuFfLipKeCIODhw4ewtrbGjz/+aNDiqGQEAYiNFe8PGwbY2UlbDxERkbHROwB9++23GgHIzMwM1atXR3BwMBwdHQ1aHJVMYiJw9ChgbQ2MGCF1NURERMZH7wDUv3//MiiDDKng6M/HHwPOztLWQkREZIz07gO0YsUKrF+/vlD7+vXrsapgtk2SzIEDwK5dQJUqwKhRUldDRERknPQOQLGxsXDWcljBxcUFM2bMMEhRVHIFR38++ADw9pa2FiIiImOldwC6fv06/Pz8CrX7+Pjg+vXrBimKSub0aWDzZnG6i7Fjpa6GiIjIeOkdgFxcXHDy5MlC7SdOnICTk5NBiqKSKRj3p3t3oE4daWshIiIyZnoHoN69e2PEiBHYuXMnlEollEol/vzzT0RFRaFXr15lUSPpIDkZWLtWvB8TI20tRERExk7vq8CmTZuGq1evon379jD/3+RSKpUK/fr1Yx8gCc2eLY7+/NZbQJMmUldDRERk3Eo8F9jFixdx/PhxWFlZISAgAD4+PoauTTIVbS6w27cBX1/gyRNg505OVEpERKZJn9/vEs8PXrt2bdSuXbukq5MBxcWJ4adFC6BtW6mrISIiMn569wHq3r07vi7obfucmTNnomfPngYpinSXmQl89514PyZGvAKMiIiIiqd3APr777/RuXPnQu2dOnXC33//bZCiSHfffQdkZQENGgDvvCN1NURERBWD3gEoOzsbFhYWhdqrVKmCrKwsgxRFunn8WDz9BQDjxgFmen+aREREpknvn8yAgADEx8cXal+3bh3q169vkKJIN8uXA+npYgdojkBARESkO707QU+cOBHdunXD5cuX8cYbbwAAkpKSsGbNGmzYsMHgBZJ2T58Cs2aJ9z//HDAvcXd2IiIi06P3z2ZYWBg2bdqEGTNmYMOGDbCyskJgYCD+/PNPVKtWrSxqJC3WrQOuXQNcXIABA6SuhoiIqGIp0XGDt99+G2+//TYA8Zr7tWvXYvTo0Thy5AiUSqVBC6TCVCrgq6/E+599BlhZSVsPERFRRVPibrN///03IiMj4eHhgW+++QZvvPEGDhw4YMjaqAi//gqcOQPY2wNDhkhdDRERUcWj1xGgtLQ0rFy5EsuWLUNWVhbee+895ObmYtOmTewAXU4EAYiNFe8PHQo4OEhbDxERUUWk8xGgsLAw1KlTBydPnkRcXBxu3bqF+fPnl2VtpMWuXcDBg4ClJTBypNTVEBERVUw6B6Bt27Zh4MCBmDp1Kt5++23I5XKDFbFw4UL4+vrC0tISwcHBOHToUJHLtmvXDjKZrNCtoE8SAPTv37/Q8x07djRYvVIqmG924ECxAzQRERHpT+cAtGfPHjx8+BBBQUEIDg7GggULkJGRUeoC4uPjER0djcmTJ+Po0aMIDAxEaGgo0tPTtS6fkJCA1NRU9e306dOQy+WFpuHo2LGjxnJr164tda1S++cfYMcOQC4HRo+WuhoiIqKKS+cA1KJFCyxduhSpqan45JNPsG7dOnh4eEClUiExMREPHz4sUQFz5szBxx9/jAEDBqB+/fpYvHgxrK2tsXz5cq3LV6tWDW5ubupbYmIirK2tCwUghUKhsZyjo2OJ6jMmBX1/3n9fHPyQiIiISkbvq8BsbGzw4YcfYs+ePTh16hRGjRqFr776Ci4uLnj33Xf12lZeXh6OHDmCDh06PCvIzAwdOnTA/v37ddrGsmXL0KtXL9jY2Gi079q1Cy4uLqhTpw6GDBmCu3fv6lWbsTl3Dti4Ubw/dqy0tRAREVV0pZo9qk6dOpg5cyZu3rxZolNMGRkZUCqVcHV11Wh3dXVFWlraS9c/dOgQTp8+jY8++kijvWPHjvj++++RlJSEr7/+Gn/99Rc6depU5BhFubm5yMrK0rgZm6+/Fq8A69JFnPiUiIiISs4gEyjI5XKEh4cjPDzcEJvT2bJlyxAQEIDmzZtrtPd6bmKsgIAANGrUCP7+/ti1axfat29faDuxsbGYOnVqmddbUtevAz/+KN6PiZG2FiIiospA0vnDnZ2dIZfLcfv2bY3227dvw83Nrdh1c3JysG7dOgwcOPClr1OzZk04Ozvj0qVLWp+PiYlBZmam+nbjxg3dd6IcfPMNkJ8PvP46EBwsdTVEREQVn6QByMLCAkFBQUhKSlK3qVQqJCUloWXLlsWuu379euTm5uKDDz546evcvHkTd+/ehbu7u9bnFQoF7O3tNW7G4s4dYOlS8T6P/hARERmGpAEIAKKjo7F06VKsWrUKZ8+exZAhQ5CTk4MB/5vhs1+/fojR8su/bNkyhIeHw8nJSaM9Ozsbn3/+OQ4cOICrV68iKSkJXbp0Qa1atRAaGlou+2RI8+YBjx8DQUHAc33FiYiIqBQM0geoNCIiInDnzh1MmjQJaWlpaNy4MbZv367uGH39+nWYmWnmtPPnz2PPnj34448/Cm1PLpfj5MmTWLVqFR48eAAPDw+89dZbmDZtGhQKRbnsk6FkZQELFoj3Y2IAmUzaeoiIiCoLmSAIgtRFGJusrCw4ODggMzNT0tNhs2YBY8YAdeqIk5+aSX68joiIyHjp8/vNn1Qj9eQJMGeOeH/sWIYfIiIiQ+LPqpFatQpISwO8vIA+faSuhoiIqHJhADJC+fnAzJni/dGjAQsLaeshIiKqbBiAjND69cCVK4CTE/DCINdERERkAAxARkYQgK++Eu9HRQEvTHFGREREBsAAZGS2bgVOngRsbYFhw6SuhoiIqHJiADIysbHiv4MHA46O0tZCRERUWTEAGZHdu4G9e8VOz599JnU1RERElRcDkBEpOPrTvz/g4SFpKURERJUaA5CROH4c2LZNHPBwzBipqyEiIqrcGICMRMGVX++9B/j7S1sLERFRZccAZAQuXRLH/gGAceOkrYWIiMgUMAAZgZkzAZUK6NwZCAyUuhoiIqLKjwFIYrduifN+AUBMjLS1EBERmQoGIInNmQPk5QFt2og3IiIiKnsMQBK6dw9YvFi8z6M/RERE5YcBSEILFgA5OWK/n06dpK6GiIjIdDAASSQnB5g3T7wfEwPIZNLWQ0REZEoYgCSydClw9y5QqxbQo4fU1RAREZkWBiAJ5OUB33wj3h8zBpDLpa2HiIjI1DAASeDHH4GbN8X5vvr1k7oaIiIi08MAVM6USuDrr8X70dGAQiFtPURERKaIAaicbdwIXLgAODoCgwZJXQ0REZFpYgAqR4IAxMaK94cPB+zspK2HiIjIVDEAlaPERODoUcDaGhgxQupqiIiITBcDUDn65Rfx30GDACcnaWshIiIyZeZSF2BKFiwAunYF6tWTuhIiIiLTxgBUjmQyoEMHqasgIiIingIjIiIik8MARERERCaHAYiIiIhMDgMQERERmRwGICIiIjI5DEBERERkchiAiIiIyOQwABEREZHJYQAiIiIik8MARERERCaHAYiIiIhMDgMQERERmRwGICIiIjI5DEBERERkchiAiIiIyOQwABEREZHJYQAiIiIik8MARERERCaHAYiIiIhMDgMQERERmRwGICIiIjI5DEBERERkchiAiIiIyOQwABEREZHJMYoAtHDhQvj6+sLS0hLBwcE4dOhQkcu2a9cOMpms0O3tt99WLyMIAiZNmgR3d3dYWVmhQ4cOuHjxYnnsChEREVUAkgeg+Ph4REdHY/LkyTh69CgCAwMRGhqK9PR0rcsnJCQgNTVVfTt9+jTkcjl69uypXmbmzJmYN28eFi9ejIMHD8LGxgahoaF48uRJee0WERERGTGZIAiClAUEBwejWbNmWLBgAQBApVLB29sbw4cPx7hx4166flxcHCZNmoTU1FTY2NhAEAR4eHhg1KhRGD16NAAgMzMTrq6uWLlyJXr16vXSbWZlZcHBwQGZmZmwt7cv3Q4SERFRudDn91vSI0B5eXk4cuQIOnTooG4zMzNDhw4dsH//fp22sWzZMvTq1Qs2NjYAgOTkZKSlpWls08HBAcHBwUVuMzc3F1lZWRo3IiIiqrwkDUAZGRlQKpVwdXXVaHd1dUVaWtpL1z906BBOnz6Njz76SN1WsJ4+24yNjYWDg4P65u3tre+uEBERUQUieR+g0li2bBkCAgLQvHnzUm0nJiYGmZmZ6tuNGzcMVCEREREZI0kDkLOzM+RyOW7fvq3Rfvv2bbi5uRW7bk5ODtatW4eBAwdqtBesp882FQoF7O3tNW5ERERUeUkagCwsLBAUFISkpCR1m0qlQlJSElq2bFnsuuvXr0dubi4++OADjXY/Pz+4ublpbDMrKwsHDx586TaJiIjINJhLXUB0dDQiIyPRtGlTNG/eHHFxccjJycGAAQMAAP369YOnpydiY2M11lu2bBnCw8Ph5OSk0S6TyTBy5Ej85z//Qe3ateHn54eJEyfCw8MD4eHh5bVbREREZMQkD0ARERG4c+cOJk2ahLS0NDRu3Bjbt29Xd2K+fv06zMw0D1SdP38ee/bswR9//KF1m2PGjEFOTg4GDRqEBw8eoE2bNti+fTssLS3LfH+IiIjI+Ek+DpAx4jhAREREFU+FGQeIiIiISAoMQERERGRyGICIiIjI5DAAERERkclhACIiIiKTwwBEREREJocBiIiIiEwOAxARERGZHAYgIiIiMjkMQERERGRyGICIiIjI5DAAERERkclhACIiIiKTwwBEREREJocBiIiIiEwOAxARERGZHAYgIiIiMjkMQERERGRyGICIiIjI5DAAERERkclhACIiIiKTwwBEREREJocBiIiIiEwOAxARERGZHAYgIiIiMjkMQERERGRyGICIiIjI5DAAERERkclhACIiIiKTYy51AUREZFqUSiWePn0qdRlUAVWpUgVyudwg22IAIiKiciEIAtLS0vDgwQOpS6EKrGrVqnBzc4NMJivVdhiAiIioXBSEHxcXF1hbW5f6B4xMiyAIePToEdLT0wEA7u7updoeAxAREZU5pVKpDj9OTk5Sl0MVlJWVFQAgPT0dLi4upTodxk7QRERU5gr6/FhbW0tcCVV0Bd+h0vYjYwAiIqJyw9NeVFqG+g4xABEREZUjX19fxMXF6bz8rl27IJPJ2HncwNgHiIiIKgylEti9G0hNBdzdgZAQwEBXRRfysiMNkydPxpQpU/Te7uHDh2FjY6Pz8q1atUJqaiocHBz0fi0qGgMQERFVCAkJQFQUcPPmszYvL2DuXKBbN8O/Xmpqqvp+fHw8Jk2ahPPnz6vbbG1t1fcFQYBSqYS5+ct/VqtXr65XHRYWFnBzc9NrHXo5ngIjIiKjl5AA9OihGX4AICVFbE9IMPxrurm5qW8ODg6QyWTqx+fOnYOdnR22bduGoKAgKBQK7NmzB5cvX0aXLl3g6uoKW1tbNGvWDDt27NDY7ounwGQyGf7v//4PXbt2hbW1NWrXro3Nmzern3/xFNjKlStRtWpV/P7776hXrx5sbW3RsWNHjcCWn5+PESNGoGrVqnBycsLYsWMRGRmJ8PDwIvf37t276N27Nzw9PWFtbY2AgACsXbtWYxmVSoWZM2eiVq1aUCgUqFGjBqZPn65+/ubNm+jduzeqVasGGxsbNG3aFAcPHizBu1/2GICIiMioKZXikR9BKPxcQdvIkeJy5W3cuHH46quvcPbsWTRq1AjZ2dno3LkzkpKScOzYMXTs2BFhYWG4fv16sduZOnUq3nvvPZw8eRKdO3dGnz59cO/evSKXf/ToEWbPno0ffvgBf//9N65fv47Ro0ern//666+xevVqrFixAnv37kVWVhY2bdpUbA1PnjxBUFAQtmzZgtOnT2PQoEHo27cvDh06pF4mJiYGX331FSZOnIgzZ85gzZo1cHV1BQBkZ2ejbdu2SElJwebNm3HixAmMGTMGKpVKh3dSAgIVkpmZKQAQMjMzpS6FiKhSePz4sXDmzBnh8ePHeq+7c6cgiFGn+NvOnQYvW23FihWCg4PDczXtFAAImzZteum6DRo0EObPn69+7OPjI3z77bfqxwCECRMmqB9nZ2cLAIRt27ZpvNb9+/fVtQAQLl26pF5n4cKFgqurq/qxq6urMGvWLPXj/Px8oUaNGkKXLl103WVBEATh7bffFkaNGiUIgiBkZWUJCoVCWLp0qdZl//vf/wp2dnbC3bt39XoNfRX3XdLn95t9gIiIyKg9d2bHIMsZUtOmTTUeZ2dnY8qUKdiyZQtSU1ORn5+Px48fv/QIUKNGjdT3bWxsYG9vrx7xWBtra2v4+/urH7u7u6uXz8zMxO3bt9G8eXP183K5HEFBQcUejVEqlZgxYwZ++uknpKSkIC8vD7m5uepxd86ePYvc3Fy0b99e6/rHjx/Hq6++imrVqhW7r8aCAYiIiIyarjMelHJmhBJ58Wqu0aNHIzExEbNnz0atWrVgZWWFHj16IC8vr9jtVKlSReOxTCYrNqxoW17Qdo5QD7NmzcLcuXMRFxeHgIAA2NjYYOTIkeraC0ZhLsrLnjc27ANERERGLSREvNqrqKvSZTLA21tcTmp79+5F//790bVrVwQEBMDNzQ1Xr14t1xocHBzg6uqKw4cPq9uUSiWOHj1a7Hp79+5Fly5d8MEHHyAwMBA1a9bEhQsX1M/Xrl0bVlZWSEpK0rp+o0aNcPz48WL7LhkTBiAiIjJqcrl4qTtQOAQVPI6LK7vxgPRRu3ZtJCQk4Pjx4zhx4gTef/99SToBDx8+HLGxsfjll19w/vx5REVF4f79+8WObVS7dm0kJiZi3759OHv2LD755BPcvn1b/bylpSXGjh2LMWPG4Pvvv8fly5dx4MABLFu2DADQu3dvuLm5ITw8HHv37sWVK1fw888/Y//+/WW+vyXBAEREREavWzdgwwbA01Oz3ctLbC+LcYBKYs6cOXB0dESrVq0QFhaG0NBQNGnSpNzrGDt2LHr37o1+/fqhZcuWsLW1RWhoKCwtLYtcZ8KECWjSpAlCQ0PRrl07dZh53sSJEzFq1ChMmjQJ9erVQ0REhLrvkYWFBf744w+4uLigc+fOCAgIwFdffVWqCUvLkkwo7UnDSigrKwsODg7IzMyEvb291OUQEVV4T548QXJyMvz8/Ir9EX6Z8hwJujJRqVSoV68e3nvvPUybNk3qckqluO+SPr/f7ARNREQVhlwOtGsndRXG79q1a/jjjz/Qtm1b5ObmYsGCBUhOTsb7778vdWlGg6fAiIiIKhkzMzOsXLkSzZo1Q+vWrXHq1Cns2LED9erVk7o0o8EjQERERJWMt7c39u7dK3UZRo1HgIiIiMjkMAARERGRyZE8AC1cuBC+vr6wtLREcHCwxqRr2jx48ABDhw6Fu7s7FAoFXnnlFWzdulX9/JQpUyCTyTRudevWLevdICIiogpE0j5A8fHxiI6OxuLFixEcHIy4uDiEhobi/PnzcHFxKbR8Xl4e3nzzTbi4uGDDhg3w9PTEtWvXULVqVY3lGjRogB07dqgfm5uzqxMRERE9I2kymDNnDj7++GMMGDAAALB48WJs2bIFy5cvx7hx4wotv3z5cty7dw/79u1Tz4Pi6+tbaDlzc3O4ubmVae1ERERUcUl2CiwvLw9HjhxBhw4dnhVjZoYOHToUOWz25s2b0bJlSwwdOhSurq5o2LAhZsyYAaVSqbHcxYsX4eHhgZo1a6JPnz4vnYU3NzcXWVlZGjciIiKqvCQLQBkZGVAqlXB1ddVod3V1RVpamtZ1rly5gg0bNkCpVGLr1q2YOHEivvnmG/znP/9RLxMcHIyVK1di+/btWLRoEZKTkxESEoKHDx8WWUtsbCwcHBzUN29vb8PsJBERmbx27dph5MiR6se+vr6Ii4srdh2ZTIZNmzaV+rUNtZ3KSPJO0PpQqVRwcXHBkiVLEBQUhIiICIwfPx6LFy9WL9OpUyf07NkTjRo1QmhoKLZu3YoHDx7gp59+KnK7MTExyMzMVN9u3LhRHrtDRERGLCwsDB07dtT63O7duyGTyXDy5Em9t3v48GEMGjSotOVpmDJlCho3blyoPTU1FZ06dTLoa1UWkvUBcnZ2hlwu15hpFgBu375dZP8dd3d3VKlSRWNitXr16iEtLQ15eXmwsLAotE7VqlXxyiuv4NKlS0XWolAooFAoSrgnRERUGQ0cOBDdu3fHzZs34eXlpfHcihUr0LRpUzRq1Ejv7VavXt1QJb4U+8MWTbIjQBYWFggKCkJSUpK6TaVSISkpCS1bttS6TuvWrXHp0iWoVCp124ULF+Du7q41/ABAdnY2Ll++DHd3d8PuABERVWrvvPMOqlevjpUrV2q0Z2dnY/369Rg4cCDu3r2L3r17w9PTE9bW1ggICMDatWuL3e6Lp8AuXryI1157DZaWlqhfvz4SExMLrTN27Fi88sorsLa2Rs2aNTFx4kQ8ffoUALBy5UpMnToVJ06cUA//UlDzi6fATp06hTfeeANWVlZwcnLCoEGDkJ2drX6+f//+CA8Px+zZs+Hu7g4nJycMHTpU/VraXL58GV26dIGrqytsbW3RrFkzjSuxAbGv7dixY+Ht7Q2FQoFatWph2bJl6uf//fdfvPPOO7C3t4ednR1CQkJw+fLlYt/H0pL0KrDo6GhERkaiadOmaN68OeLi4pCTk6O+Kqxfv37w9PREbGwsAGDIkCFYsGABoqKiMHz4cFy8eBEzZszAiBEj1NscPXo0wsLC4OPjg1u3bmHy5MmQy+Xo3bu3JPtIRETaCQLw6FH5v661NSCTvXw5c3Nz9OvXDytXrsT48eMh+99K69evh1KpRO/evZGdnY2goCCMHTsW9vb22LJlC/r27Qt/f380b978pa+hUqnQrVs3uLq64uDBg8jMzNToL1TAzs4OK1euhIeHB06dOoWPP/4YdnZ2GDNmDCIiInD69Gls375dHTwcHBwKbSMnJwehoaFo2bIlDh8+jPT0dHz00UcYNmyYRsjbuXMn3N3dsXPnTly6dAkRERFo3LgxPv74Y637kJ2djc6dO2P69OlQKBT4/vvvERYWhvPnz6NGjRoAxN/z/fv3Y968eQgMDERycjIyMjIAACkpKXjttdfQrl07/Pnnn7C3t8fevXuRn5//0vevVASJzZ8/X6hRo4ZgYWEhNG/eXDhw4ID6ubZt2wqRkZEay+/bt08IDg4WFAqFULNmTWH69OlCfn6++vmIiAjB3d1dsLCwEDw9PYWIiAjh0qVLetWUmZkpABAyMzNLtW8vys8XhJ07BWHNGvHf58omIqrUHj9+LJw5c0Z4/Pixui07WxDEGFS+t+xs3es+e/asAEDYuXOnui0kJET44IMPilzn7bffFkaNGqV+3LZtWyEqKkr92MfHR/j2228FQRCE33//XTA3NxdSUlLUz2/btk0AIGzcuLHI15g1a5YQFBSkfjx58mQhMDCw0HLPb2fJkiWCo6OjkP3cG7BlyxbBzMxMSEtLEwRBECIjIwUfHx+N39WePXsKERERRdaiTYMGDYT58+cLgiAI58+fFwAIiYmJWpeNiYkR/Pz8hLy8PJ22re27VECf32/JRwgcNmwYhg0bpvW5Xbt2FWpr2bIlDhw4UOT21q1bZ6jSDCohAYiKAm7efNbm5QXMnQt06yZdXUREVLS6deuiVatWWL58Odq1a4dLly5h9+7d+PLLLwEASqUSM2bMwE8//YSUlBTk5eUhNzcX1tbWOm3/7Nmz8Pb2hoeHh7pNWzeQ+Ph4zJs3D5cvX0Z2djby8/Nhb2+v176cPXsWgYGBsLGxUbe1bt0aKpUK58+fV1+V3aBBA42+tu7u7jh16lSR283OzsaUKVOwZcsWpKamIj8/H48fP1YPQXP8+HHI5XK0bdtW6/rHjx9HSEiIeny/8iJ5ADIFCQlAjx7i3x7PS0kR2zdsYAgiItNjbQ081/2kXF9XHwMHDsTw4cOxcOFCrFixAv7+/uof81mzZmHu3LmIi4tDQEAAbGxsMHLkSOTl5Rms3v3796NPnz6YOnUqQkND4eDggHXr1uGbb74x2Gs878UgIpPJNPrevmj06NFITEzE7NmzUatWLVhZWaFHjx7q98DKyqrY13vZ82WFAaiMKZXikZ8Xww8gtslkwMiRQJcuwHOBm4io0pPJgOcORhit9957D1FRUVizZg2+//57DBkyRN0faO/evejSpQs++OADAGKfngsXLqB+/fo6bbtevXq4ceMGUlNT1RfrvHiWY9++ffDx8cH48ePVbdeuXdNYxsLCotCgwNpea+XKlcjJyVEfBdq7dy/MzMxQp04dnerVZu/evejfvz+6du0KQDwidPXqVfXzAQEBUKlU+OuvvzQGPy7QqFEjrFq1Ck+fPi3Xo0AVahygimj3bs3TXi8SBODGDXE5IiIyPra2toiIiEBMTAxSU1PRv39/9XO1a9dGYmIi9u3bh7Nnz+KTTz4pNLxLcTp06IBXXnkFkZGROHHiBHbv3q0RdApe4/r161i3bh0uX76MefPmYePGjRrL+Pr6Ijk5GcePH0dGRgZyc3MLvVafPn1gaWmJyMhInD59Gjt37sTw4cPRt2/fQoMS66N27dpISEjA8ePHceLECbz//vsaR4x8fX0RGRmJDz/8EJs2bUJycjJ27dqlHp9v2LBhyMrKQq9evfDPP//g4sWL+OGHH3D+/PkS16QLBqAylppq2OWIiKj8DRw4EPfv30doaKhGf50JEyagSZMmCA0NRbt27eDm5obw8HCdt2tmZoaNGzfi8ePHaN68OT766CNMnz5dY5l3330Xn332GYYNG4bGjRtj3759mDhxosYy3bt3R8eOHfH666+jevXqWi/Ft7a2xu+//4579+6hWbNm6NGjB9q3b48FCxbo92a8YM6cOXB0dESrVq0QFhaG0NBQNGnSRGOZRYsWoUePHvj0009Rt25dfPzxx8jJyQEAODk54c8//0R2djbatm2LoKAgLF26tMyPBskEQdvJGdOWlZUFBwcHZGZm6t3J7EW7dgGvv/7y5XbuBNq1K9VLEREZrSdPniA5ORl+fn6wtLSUuhyqwIr7Lunz+80jQGUsJES82quoMSdkMsDbW1yOiIiIygcDUBmTy8VL3YHCIajgcVwcO0ATERGVJwagctCtm3ipu6enZruXFy+BJyIikgIvgy8n3bqJl7rv3i12eHZ3F0978cgPERFR+WMAKkdyOTs6ExERGQOeAiMionLDC4+ptAz1HWIAIiKiMlcwpssjKaZ/p0ql4DtU2nGCeAqMiIjKnFwuR9WqVZGeng5AHJRPVtT4IERaCIKAR48eIT09HVWrVtWYsLUkGICIiKhcuLm5AYA6BBGVRNWqVdXfpdJgACIionIhk8ng7u4OFxcXPH36VOpyqAKqUqVKqY/8FGAAIiKiciWXyw32I0ZUUuwETURERCaHAYiIiIhMDgMQERERmRz2AdKiYJClrKwsiSshIiIiXRX8busyWCIDkBYPHz4EAHh7e0tcCREREenr4cOHcHBwKHYZmcBxyQtRqVS4desW7OzsOFBXEbKysuDt7Y0bN27A3t5e6nJMHj8P48LPw7jw8zAuZfl5CIKAhw8fwsPDA2Zmxffy4REgLczMzODl5SV1GRWCvb09/4diRPh5GBd+HsaFn4dxKavP42VHfgqwEzQRERGZHAYgIiIiMjkMQFQiCoUCkydPhkKhkLoUAj8PY8PPw7jw8zAuxvJ5sBM0ERERmRweASIiIiKTwwBEREREJocBiIiIiEwOAxARERGZHAYg0llsbCyaNWsGOzs7uLi4IDw8HOfPn5e6LPqfr776CjKZDCNHjpS6FJOWkpKCDz74AE5OTrCyskJAQAD++ecfqcsySUqlEhMnToSfnx+srKzg7++PadOm6TRPFJXe33//jbCwMHh4eEAmk2HTpk0azwuCgEmTJsHd3R1WVlbo0KEDLl68WG71MQCRzv766y8MHToUBw4cQGJiIp4+fYq33noLOTk5Updm8g4fPoz//ve/aNSokdSlmLT79++jdevWqFKlCrZt24YzZ87gm2++gaOjo9SlmaSvv/4aixYtwoIFC3D27Fl8/fXXmDlzJubPny91aSYhJycHgYGBWLhwodbnZ86ciXnz5mHx4sU4ePAgbGxsEBoaiidPnpRLfbwMnkrszp07cHFxwV9//YXXXntN6nJMVnZ2Npo0aYLvvvsO//nPf9C4cWPExcVJXZZJGjduHPbu3Yvdu3dLXQoBeOedd+Dq6oply5ap27p37w4rKyv8+OOPElZmemQyGTZu3Ijw8HAA4tEfDw8PjBo1CqNHjwYAZGZmwtXVFStXrkSvXr3KvCYeAaISy8zMBABUq1ZN4kpM29ChQ/H222+jQ4cOUpdi8jZv3oymTZuiZ8+ecHFxwauvvoqlS5dKXZbJatWqFZKSknDhwgUAwIkTJ7Bnzx506tRJ4sooOTkZaWlpGv/fcnBwQHBwMPbv318uNXAyVCoRlUqFkSNHonXr1mjYsKHU5ZisdevW4ejRozh8+LDUpRCAK1euYNGiRYiOjsYXX3yBw4cPY8SIEbCwsEBkZKTU5ZmccePGISsrC3Xr1oVcLodSqcT06dPRp08fqUszeWlpaQAAV1dXjXZXV1f1c2WNAYhKZOjQoTh9+jT27NkjdSkm68aNG4iKikJiYiIsLS2lLocg/mHQtGlTzJgxAwDw6quv4vTp01i8eDEDkAR++uknrF69GmvWrEGDBg1w/PhxjBw5Eh4eHvw8iKfASH/Dhg3Db7/9hp07d8LLy0vqckzWkSNHkJ6ejiZNmsDc3Bzm5ub466+/MG/ePJibm0OpVEpdoslxd3dH/fr1Ndrq1auH69evS1SRafv8888xbtw49OrVCwEBAejbty8+++wzxMbGSl2ayXNzcwMA3L59W6P99u3b6ufKGgMQ6UwQBAwbNgwbN27En3/+CT8/P6lLMmnt27fHqVOncPz4cfWtadOm6NOnD44fPw65XC51iSandevWhYaGuHDhAnx8fCSqyLQ9evQIZmaaP3NyuRwqlUqiiqiAn58f3NzckJSUpG7LysrCwYMH0bJly3KpgafASGdDhw7FmjVr8Msvv8DOzk59ntbBwQFWVlYSV2d67OzsCvW/srGxgZOTE/tlSeSzzz5Dq1atMGPGDLz33ns4dOgQlixZgiVLlkhdmkkKCwvD9OnTUaNGDTRo0ADHjh3DnDlz8OGHH0pdmknIzs7GpUuX1I+Tk5Nx/PhxVKtWDTVq1MDIkSPxn//8B7Vr14afnx8mTpwIDw8P9ZViZU4g0hEArbcVK1ZIXRr9T9u2bYWoqCipyzBpv/76q9CwYUNBoVAIdevWFZYsWSJ1SSYrKytLiIqKEmrUqCFYWloKNWvWFMaPHy/k5uZKXZpJ2Llzp9bfjMjISEEQBEGlUgkTJ04UXF1dBYVCIbRv3144f/58udXHcYCIiIjI5LAPEBEREZkcBiAiIiIyOQxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiKgIMpkMmzZtkroMIioDDEBEZJT69+8PmUxW6NaxY0epSyOiSoBzgRGR0erYsSNWrFih0aZQKCSqhogqEx4BIiKjpVAo4ObmpnFzdHQEIJ6eWrRoETp16gQrKyvUrFkTGzZs0Fj/1KlTeOONN2BlZQUnJycMGjQI2dnZGsssX74cDRo0gEKhgLu7O4YNG6bxfEZGBrp27Qpra2vUrl0bmzdvVj93//599OnTB9WrV4eVlRVq165dKLARkXFiACKiCmvixIno3r07Tpw4gT59+qBXr144e/YsACAnJwehoaFwdHTE4cOHsX79euzYsUMj4CxatAhDhw7FoEGDcOrUKWzevBm1atXSeI2pU6fivffew8mTJ9G5c2f06dMH9+7dU7/+mTNnsG3bNpw9exaLFi2Cs7Nz+b0BRFRy5TbtKhGRHiIjIwW5XC7Y2Nho3KZPny4IgiAAEAYPHqyxTnBwsDBkyBBBEARhyZIlgqOjo5Cdna1+fsuWLYKZmZmQlpYmCIIgeHh4COPHjy+yBgDChAkT1I+zs7MFAMK2bdsEQRCEsLAwYcCAAYbZYSIqV+wDRERG6/XXX8eiRYs02qpVq6a+37JlS43nWrZsiePHjwMAzp49i8DAQNjY2Kifb926NVQqFc6fPw+ZTIZbt26hffv2xdbQqFEj9X0bGxvY29sjPT0dADBkyBB0794dR48exVtvvYXw8HC0atWqRPtKROWLAYiIjJaNjU2hU1KGYmVlpdNyVapU0Xgsk8mgUqkAAJ06dcK1a9ewdetWJCYmon379hg6dChmz55t8HqJyLDYB4iIKqwDBw4UelyvXj0AQL169XDixAnk5OSon9+7dy/MzMxQp04d2NnZwdfXF0lJSaWqoXr16oiMjMSPP/6IuLg4LFmypFTbI6LywSNARGS0cnNzkZaWptFmbm6u7mi8fv16NG3aFG3atMHq1atx6NAhLFu2DADQp08fTJ48GZGRkZgyZQru3LmD4cOHo2/fvnB1dQUATJkyBYMHD4aLiws6deqEhw8fYu/evRg+fLhO9U2aNAlBQUFo0KABcnNz8dtvv6kDGBEZNwYgIjJa27dvh7u7u0ZbnTp1cO7cOQDiFVrr1q3Dp59+Cnd3d6xduxb169cHAFhbW+P3339HVFQUmjVrBmtra3Tv3h1z5sxRbysyMhJPnjzBt99+i9GjR8PZ2Rk9evTQuT4LCwvExMTg6tWrsLKyQkhICNatW2eAPSeisiYTBEGQuggiIn3JZDJs3LgR4eHhUpdCRBUQ+wARERGRyWEAIiIiIpPDPkBEVCHx7D0RlQaPABEREZHJYQAiIiIik8MARERERCaHAYiIiIhMDgMQERERmRwGICIiIjI5DEBERERkchiAiIiIyOQwABEREZHJ+X9ZQiGFKsjNOgAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABbR0lEQVR4nO3deXhMZ/sH8O9km+wRSWQTSYRaI/bUErRN36BNxR6UWFqllpBqUXsVfVGNrZRXUbWkNFQpGim1L0WQij2ESEIsiQQRk/P74/wyjEyWSSY5M5nv57rmmplnnnPOfWaGufOcZ5EJgiCAiIiIyIAYSR0AERERUUVjAkREREQGhwkQERERGRwmQERERGRwmAARERGRwWECRERERAaHCRAREREZHCZAREREZHCYABEREZHBYQJEpAUDBw6El5dXqbadPn06ZDKZdgPSMTdu3IBMJsOaNWsq9Lj79++HTCbD/v37lWUl/azKK2YvLy8MHDhQq/skIs0xAaJKTSaTlej26g8kUVkdOXIE06dPx6NHj6QOhYgKYSJ1AETlad26dSrPf/rpJ8TExBQor1evXpmOs3LlSuTl5ZVq28mTJ2PChAllOj6VXFk+q5I6cuQIZsyYgYEDB6JKlSoqr126dAlGRvzbk0hqTICoUvvwww9Vnh87dgwxMTEFyl/35MkTWFpalvg4pqampYoPAExMTGBiwn+KFaUsn5U2yOVySY+vL7Kzs2FlZSV1GFSJ8c8QMngdOnRAw4YNcerUKbRr1w6Wlpb48ssvAQC//fYb3nvvPbi5uUEul8PHxwczZ86EQqFQ2cfr/Ury+4/Mnz8fK1asgI+PD+RyOVq0aIGTJ0+qbKuuD5BMJsPIkSOxbds2NGzYEHK5HA0aNMDu3bsLxL9//340b94c5ubm8PHxwQ8//FDifkUHDx5Ez549UaNGDcjlcnh4eGDs2LF4+vRpgfOztrZGcnIyQkJCYG1tDScnJ4wbN67Ae/Ho0SMMHDgQdnZ2qFKlCsLCwkp0Keiff/6BTCbD2rVrC7y2Z88eyGQy7NixAwBw8+ZNfPrpp6hTpw4sLCzg4OCAnj174saNG8UeR10foJLGfO7cOQwcOBA1a9aEubk5XFxcMHjwYNy/f19ZZ/r06fj8888BAN7e3srLrPmxqesDdP36dfTs2RNVq1aFpaUl3nzzTezcuVOlTn5/pl9++QWzZs1C9erVYW5ujnfeeQdXr14t9rw1ec8ePXqEsWPHwsvLC3K5HNWrV8eAAQOQnp6urPPs2TNMnz4db7zxBszNzeHq6opu3brh2rVrKvG+fnlZXd+q/O/XtWvX0LlzZ9jY2KBfv34ASv4dBYCLFy+iV69ecHJygoWFBerUqYNJkyYBAPbt2weZTIatW7cW2G7Dhg2QyWQ4evRose8jVR78s5MIwP3799GpUyeEhobiww8/hLOzMwBgzZo1sLa2RkREBKytrfHXX39h6tSpyMzMxLx584rd74YNG/D48WN88sknkMlkmDt3Lrp164br168X2xJx6NAhREdH49NPP4WNjQ0WLVqE7t27IykpCQ4ODgCAM2fOoGPHjnB1dcWMGTOgUCjw1VdfwcnJqUTnvXnzZjx58gTDhw+Hg4MDTpw4gcWLF+P27dvYvHmzSl2FQoGgoCD4+/tj/vz52Lt3L7799lv4+Phg+PDhAABBENClSxccOnQIw4YNQ7169bB161aEhYUVG0vz5s1Rs2ZN/PLLLwXqR0VFwd7eHkFBQQCAkydP4siRIwgNDUX16tVx48YNLFu2DB06dMCFCxc0ar3TJOaYmBhcv34dgwYNgouLC/7991+sWLEC//77L44dOwaZTIZu3brh8uXL2LhxI7777js4OjoCQKGfSVpaGlq3bo0nT55g9OjRcHBwwNq1a/HBBx9gy5Yt6Nq1q0r9b775BkZGRhg3bhwyMjIwd+5c9OvXD8ePHy/yPEv6nmVlZSEgIAAJCQkYPHgwmjZtivT0dGzfvh23b9+Go6MjFAoF3n//fcTGxiI0NBTh4eF4/PgxYmJiEB8fDx8fnxK///levHiBoKAgtG3bFvPnz1fGU9Lv6Llz5xAQEABTU1MMHToUXl5euHbtGn7//XfMmjULHTp0gIeHB9avX1/gPV2/fj18fHzQqlUrjeMmPSYQGZARI0YIr3/t27dvLwAQli9fXqD+kydPCpR98skngqWlpfDs2TNlWVhYmODp6al8npiYKAAQHBwchAcPHijLf/vtNwGA8PvvvyvLpk2bViAmAIKZmZlw9epVZdnZs2cFAMLixYuVZcHBwYKlpaWQnJysLLty5YpgYmJSYJ/qqDu/OXPmCDKZTLh586bK+QEQvvrqK5W6TZo0EZo1a6Z8vm3bNgGAMHfuXGXZixcvhICAAAGAsHr16iLjmThxomBqaqrynuXk5AhVqlQRBg8eXGTcR48eFQAIP/30k7Js3759AgBh3759Kufy6melSczqjrtx40YBgHDgwAFl2bx58wQAQmJiYoH6np6eQlhYmPL5mDFjBADCwYMHlWWPHz8WvL29BS8vL0GhUKicS7169YScnBxl3YULFwoAhPPnzxc41qtK+p5NnTpVACBER0cXqJ+XlycIgiD8+OOPAgBhwYIFhdZR994Lwst/G6++r/nfrwkTJpQobnXf0Xbt2gk2NjYqZa/GIwji90sulwuPHj1Slt29e1cwMTERpk2bVuA4VLnxEhgRxH4ZgwYNKlBuYWGhfPz48WOkp6cjICAAT548wcWLF4vdb+/evWFvb698HhAQAEC85FGcwMBAlb+kGzVqBFtbW+W2CoUCe/fuRUhICNzc3JT1atWqhU6dOhW7f0D1/LKzs5Geno7WrVtDEAScOXOmQP1hw4apPA8ICFA5lz/++AMmJibKFiEAMDY2xqhRo0oUT+/evZGbm4vo6Ghl2Z9//olHjx6hd+/eauPOzc3F/fv3UatWLVSpUgWnT58u0bFKE/Orx3327BnS09Px5ptvAoDGx331+C1btkTbtm2VZdbW1hg6dChu3LiBCxcuqNQfNGgQzMzMlM9L+p0q6Xv266+/ws/Pr0ArCQDlZdVff/0Vjo6Oat+jskzp8OpnoC7uwr6j9+7dw4EDBzB48GDUqFGj0HgGDBiAnJwcbNmyRVkWFRWFFy9eFNsvkCofJkBEANzd3VV+VPL9+++/6Nq1K+zs7GBrawsnJyflf5QZGRnF7vf1/4zzk6GHDx9qvG3+9vnb3r17F0+fPkWtWrUK1FNXpk5SUhIGDhyIqlWrKvv1tG/fHkDB8zM3Ny9wGefVeACxn4mrqyusra1V6tWpU6dE8fj5+aFu3bqIiopSlkVFRcHR0RFvv/22suzp06eYOnUqPDw8IJfL4ejoCCcnJzx69KhEn8urNIn5wYMHCA8Ph7OzMywsLODk5ARvb28AJfs+FHZ8dcfKH5l48+ZNlfLSfqdK+p5du3YNDRs2LHJf165dQ506dbTaed/ExATVq1cvUF6S72h+8ldc3HXr1kWLFi2wfv16Zdn69evx5ptvlvjfDFUe7ANEBNW/MvM9evQI7du3h62tLb766iv4+PjA3Nwcp0+fxvjx40s0lNrY2FhtuSAI5bptSSgUCrz77rt48OABxo8fj7p168LKygrJyckYOHBggfMrLB5t6927N2bNmoX09HTY2Nhg+/bt6NOnj8qP7ahRo7B69WqMGTMGrVq1gp2dHWQyGUJDQ8t1iHuvXr1w5MgRfP7552jcuDGsra2Rl5eHjh07lvvQ+nyl/V5U9HtWWEvQ653m88nl8gLTA2j6HS2JAQMGIDw8HLdv30ZOTg6OHTuGJUuWaLwf0n9MgIgKsX//fty/fx/R0dFo166dsjwxMVHCqF6qVq0azM3N1Y4AKsmooPPnz+Py5ctYu3YtBgwYoCyPiYkpdUyenp6IjY1FVlaWSovKpUuXSryP3r17Y8aMGfj111/h7OyMzMxMhIaGqtTZsmULwsLC8O233yrLnj17VqqJB0sa88OHDxEbG4sZM2Zg6tSpyvIrV64U2Kcml4E8PT3Vvj/5l1g9PT1LvK+ilPQ98/HxQXx8fJH78vHxwfHjx5Gbm1toZ/78lqnX9/96i1ZRSvodrVmzJgAUGzcAhIaGIiIiAhs3bsTTp09hamqqcnmVDAcvgREVIv8v7Vf/sn7+/Dm+//57qUJSYWxsjMDAQGzbtg137txRll+9ehW7du0q0faA6vkJgoCFCxeWOqbOnTvjxYsXWLZsmbJMoVBg8eLFJd5HvXr14Ovri6ioKERFRcHV1VUlAc2P/fUWj8WLFxfauqCNmNW9XwAQGRlZYJ/589eUJCHr3LkzTpw4oTIEOzs7GytWrICXlxfq169f0lMpUknfs+7du+Ps2bNqh4vnb9+9e3ekp6erbTnJr+Pp6QljY2McOHBA5XVN/v2U9Dvq5OSEdu3a4ccff0RSUpLaePI5OjqiU6dO+Pnnn7F+/Xp07NhROVKPDAtbgIgK0bp1a9jb2yMsLAyjR4+GTCbDunXrtHYJShumT5+OP//8E23atMHw4cOhUCiwZMkSNGzYEHFxcUVuW7duXfj4+GDcuHFITk6Gra0tfv311xL1TypMcHAw2rRpgwkTJuDGjRuoX78+oqOjNe4f07t3b0ydOhXm5uYYMmRIgUsj77//PtatWwc7OzvUr18fR48exd69e5XTA5RHzLa2tmjXrh3mzp2L3NxcuLu7488//1TbItisWTMAwKRJkxAaGgpTU1MEBwerndhvwoQJ2LhxIzp16oTRo0ejatWqWLt2LRITE/Hrr79qbdbokr5nn3/+ObZs2YKePXti8ODBaNasGR48eIDt27dj+fLl8PPzw4ABA/DTTz8hIiICJ06cQEBAALKzs7F37158+umn6NKlC+zs7NCzZ08sXrwYMpkMPj4+2LFjB+7evVvimDX5ji5atAht27ZF06ZNMXToUHh7e+PGjRvYuXNngX8LAwYMQI8ePQAAM2fO1PzNpMqhwsedEUmosGHwDRo0UFv/8OHDwptvvilYWFgIbm5uwhdffCHs2bOn2KHV+UN9582bV2CfAFSG3BY2DH7EiBEFtn19CLUgCEJsbKzQpEkTwczMTPDx8RH+97//CZ999plgbm5eyLvw0oULF4TAwEDB2tpacHR0FD7++GPlcPvXhylbWVkV2F5d7Pfv3xf69+8v2NraCnZ2dkL//v2FM2fOlGgYfL4rV64IAAQAwqFDhwq8/vDhQ2HQoEGCo6OjYG1tLQQFBQkXL14s8P6UZBi8JjHfvn1b6Nq1q1ClShXBzs5O6Nmzp3Dnzp0Cn6kgCMLMmTMFd3d3wcjISGVIvLrP8Nq1a0KPHj2EKlWqCObm5kLLli2FHTt2qNTJP5fNmzerlKsbVq5OSd+z/Pdj5MiRgru7u2BmZiZUr15dCAsLE9LT05V1njx5IkyaNEnw9vYWTE1NBRcXF6FHjx7CtWvXlHXu3bsndO/eXbC0tBTs7e2FTz75RIiPjy/x90sQSv4dFQRBiI+PV34+5ubmQp06dYQpU6YU2GdOTo5gb28v2NnZCU+fPi3yfaPKSyYIOvTnLBFpRUhICP7991+1/VOIDN2LFy/g5uaG4OBgrFq1SupwSCLsA0Sk515fEuDKlSv4448/0KFDB2kCItJx27Ztw71791Q6VpPhYQsQkZ5zdXVVrk918+ZNLFu2DDk5OThz5gxq164tdXhEOuP48eM4d+4cZs6cCUdHx1JPXkmVAztBE+m5jh07YuPGjUhNTYVcLkerVq0we/ZsJj9Er1m2bBl+/vlnNG7cWGUxVjJMbAEiIiIig8M+QERERGRwmAARERGRwWEfIDXy8vJw584d2NjYlGllYyIiIqo4giDg8ePHcHNzK3YSUSZAaty5cwceHh5Sh0FERESlcOvWLVSvXr3IOkyA1LCxsQEgvoG2trYSR0NEREQlkZmZCQ8PD+XveFGYAKmRf9nL1taWCRAREZGeKUn3FXaCJiIiIoPDBIiIiIgMDhMgIiIiMjhMgIiIiMjgMAEiIiIig8MEiIiIiAwOEyAiIiIyOEyAiIiIyOAwASIiIiKDw5mgiYiIqEIoFMDBg0BKCuDqCgQEAMbG0sTCBIiIiIjKXXQ0EB4O3L79sqx6dWDhQqBbt4qPh5fAiIiIqFxFRwM9eqgmPwCQnCyWR0dXfExMgIiIiKjcKBRiy48gFHwtv2zMGLFeRWICREREROXm4MGCLT+vEgTg1i2xXkViAkRERETlJiVFu/W0hQkQERERlRtXV+3W0xaOAiMiItJxujR8XFMBAeJor+Rk9f2AZDLx9YCAio2LLUBEREQ6LDoa8PIC3noL6NtXvPfykmbkVGkYG4tD3QEx2XlV/vPIyIpP6JgAERER6ShdHD5eGt26AVu2AO7uquXVq4vlUswDJBMEdQ1Shi0zMxN2dnbIyMiAra2t1OEQEZEBUijElp7CRlDlXzpKTNSfy2HlfSlPk99v9gEiIiLSQZoMH+/QocLCKhNjY92JlZfAiIiIdJCuDh+vLJgAERER6SBdHT5eWfASGBERVVocPk6FkbwFaOnSpfDy8oK5uTn8/f1x4sSJQuvm5ubiq6++go+PD8zNzeHn54fdu3eXaZ9ERFQ5cfg4FUXSBCgqKgoRERGYNm0aTp8+DT8/PwQFBeHu3btq60+ePBk//PADFi9ejAsXLmDYsGHo2rUrzpw5U+p9EhFR5cPh41QcSYfB+/v7o0WLFliyZAkAIC8vDx4eHhg1ahQmTJhQoL6bmxsmTZqEESNGKMu6d+8OCwsL/Pzzz6XapzocBk9EpL84fNxwafL7LVkL0PPnz3Hq1CkEBga+DMbICIGBgTh69KjabXJycmBubq5SZmFhgUOHDpV6n/n7zczMVLkREZF+0tXVx8sif/h4nz7iPZOfspMsAUpPT4dCoYCzs7NKubOzM1JTU9VuExQUhAULFuDKlSvIy8tDTEwMoqOjkfL/YwBLs08AmDNnDuzs7JQ3Dw+PMp4dERFJhcPHqSQk7wStiYULF6J27dqoW7cuzMzMMHLkSAwaNAhGRmU7jYkTJyIjI0N5u3XrlpYiJiKiisbh41QSkiVAjo6OMDY2Rlpamkp5WloaXFxc1G7j5OSEbdu2ITs7Gzdv3sTFixdhbW2NmjVrlnqfACCXy2Fra6tyIyIi/ZQ/fPz1kVP5ZDLAw4PDxw2dZAmQmZkZmjVrhtjYWGVZXl4eYmNj0apVqyK3NTc3h7u7O168eIFff/0VXbp0KfM+iYiocuDwcSoJSS+BRUREYOXKlVi7di0SEhIwfPhwZGdnY9CgQQCAAQMGYOLEicr6x48fR3R0NK5fv46DBw+iY8eOyMvLwxdffFHifRIRUfEUCmD/fmDjRvFeoZA6Is1w+DgVR9KZoHv37o179+5h6tSpSE1NRePGjbF7925lJ+akpCSV/j3Pnj3D5MmTcf36dVhbW6Nz585Yt24dqlSpUuJ9EhFR0aKjgfBw1ZFU1auLrSr6lDh06wZ06cLh46SepPMA6SrOA0REhip/AsHXfxnyLx2x9YR0mV7MA0RERLpFoRBbftT9WZxfNmaM/l0OI1KHCRAREQGonBMIEhWGCRAREQHgBIJkWJgAERERAE4gSIaFCRAREQHgBIJkWJgAERERAE4gSIaFCRARESlxAkEyFJJOhEhERLqHEwiSIWACREREBRgbAx06SB0FUflhAkREpEUKBVtOiPQBEyAiIi2pLGtoERkCdoImItKC/DW0Xp9JOTlZLI+OliYuIlKPCRARURlxDS0i/cMEiIiojLiGFpH+YQJERFRGXEOLSP8wASIiKiOuoUWkf5gAERGVEdfQItI/TICIiMqIa2gR6R8mQEREWsA1tIj0CydCJCLSEq6hRaQ/mAAREWkR19AiKpwgAGlpQGIi4OICeHtLFwsTICLSCVxDi6hyyMoSE5zr19XfP30q1vvqK2DKFOniZAJERJLjGlpE+iM3V5zYs7AkJz296O3zR0WamVVMvIVhAkREkspfQ+v1ZSTy19BiB2KiiiUIwL17hSc4t24Vv6xL1ari5a2aNQve16ghffIDADJBULd6jWHLzMyEnZ0dMjIyYGtrK3U4RJWWQgF4eRW+jIRMJrYEJSbychiRNmVnAzduFH6ZKju76O3lcvHfrroEx9sbsLOriLMoSJPfb7YAEZFkNFlDix2LiUpOoRD/bRWW4KSlFb29TAa4uRWe4Li6AkZ6PpEOEyAikgzX0CIqHUEAHjwoPMG5eRN48aLofdjZFZ7geHoC5uYVcy5SYQJERJLhGlr0qtxccYTQkyea34rbLidHbNV49WZkpNnz0myj7X3m5IjJzfXrwOPHRb+fpqbiZarC+uLY21fIx6qzmAARkWTy19BKTi7YCRp42QeIa2hJS6HQLDEpbRJTXIsFFeTqWniC4+bGvnNFYQJERJLJX0OrRw8x2Xk1CeIaWtr35Alw/7546eTV+6LKMjLEVoeKJJMBlpaF3ywsin5d3S1/1FFenvg9y78V97wkdSpynyYm4iiqmjXF1h0Li4r9bCoTJkBEJKn8NbTUzQMUGckh8Oo8fy4mJ+oSmKKSmWfPyn5sdclHaRKSoraTywsuKkukbUyAiEhyhrqGlkIBPHpUeAJTWDKTlVX6Y5qYAA4O4jwtr96rK6taFahSBbCyEhMTc3P9H/lDlI8JEBHphMqyhlZWFnDunNhRtbhk5tEj9X2fSkImEzuxFpfEvF5mY8PWFSKACRARUamlpgJxccCZM+J9XBxw5YrmSY2tbclbZV5tmWFrDFHpMQEiIipGXh5w9WrBZCc1VX19NzfgjTcAR8fiExt7e3G4MhFVLCZARESvePYMiI9XTXbOnlW/NIBMBtSpAzRuDDRpIt43bgxUq1ahIRNRKTABIiKD9eDBy9ac/GQnIUH9Qo/m5kCjRqrJjq+v2EGYiPQPEyAiqvQEQeyU/Hqyk5Skvr6Dw8skJ//+jTfEEVREVDnwnzMRVSq5uWIrzuvJzqNH6uvXrPny0lV+suPuzpFSRJUdEyAi0luPH4v9c15NduLjxYkCX2dqCjRooJro+PmJC0ISkeFhAkREOk8QxBFXr47AOnNGHJmljq1twVad+vVfLodARMQEiIh0ikIhJjb5yU7+/d276uu7uxfsr+PlxTlyiKhoTICI9JxCoZ9LSAiCGHN8PPDvvy/vz58XF+18nZERULeuasuOnx/g5FTRkRNRZcAEiEiPRUerX0R04ULdWkT07l3VJCf/vrCOyRYWYnLzarLTsKG4HhURkTYwASLSU9HRQI8eBZddSE4Wy7dsqfgk6MEDMbF5PdlJT1df39gYqFVLTG4aNBDvGzYUh5zrQysWEekvmSCUdim+yiszMxN2dnbIyMiAra2t1OEQFaBQiP1cXm35eZVMJrYEJSaWTyKRmQlcuFCwRSclpfB4atZ8meTk39epA8jl2o+PiAyTJr/fbAEi0kMHDxae/ABiq9CtW2K9sqywnp0tzqnzapLz77+FTyAIADVqqCY5DRoA9erx8hUR6RYmQER6qLCWltLWe/YMuHSpYItOYmLhK5u7uakmOQ0aiEPN2WhKRPqACRCRHnJ1LV293Fzg8uWCLTpXrogrnqvj5KSa5OQ/trcv2zkQEUmJCRCRHgoIEPv4JCcX3kLj4iJ2Pp4582XCc/mymASpY29fsEWnQQOubE5ElZPkU4UtXboUXl5eMDc3h7+/P06cOFFk/cjISNSpUwcWFhbw8PDA2LFj8ezZM+Xr06dPh0wmU7nVrVu3vE+DqEIZG4tD3YuSmgr07AlMnQpERYlJUG4uYG0NvPkmMGQIsGAB8OefYiJ1/77YZ2jZMmDkSOCtt5j8EFHlJWkLUFRUFCIiIrB8+XL4+/sjMjISQUFBuHTpEqqp+Z93w4YNmDBhAn788Ue0bt0aly9fxsCBAyGTybBgwQJlvQYNGmDv3r3K5yZcwpkqobZtgX79gI0bxVFhr7OwEPvkvD7yysODC30SEUmaGSxYsAAff/wxBg0aBABYvnw5du7ciR9//BETJkwoUP/IkSNo06YN+vbtCwDw8vJCnz59cPz4cZV6JiYmcHFxKf8TIKpggvCylebXX19ezrKyEicOrFcPeP99wNdXHCbPuXSIiNST7BLY8+fPcerUKQQGBr4MxsgIgYGBOHr0qNptWrdujVOnTikvk12/fh1//PEHOnfurFLvypUrcHNzQ82aNdGvXz8kFTVmF0BOTg4yMzNVbkS65NEjYNEisRWnfXtg0yYx+WnZEvjxR3Gm5cOHgf/9DwgJAXx8mPwQERVFshag9PR0KBQKODs7q5Q7Ozvj4sWLarfp27cv0tPT0bZtWwiCgBcvXmDYsGH48ssvlXX8/f2xZs0a1KlTBykpKZgxYwYCAgIQHx8PGxsbtfudM2cOZsyYob2TI9ICQQBOngSWLxcTnqdPxXIrK/HS1yefAE2bShsjEZG+krwTtCb279+P2bNn4/vvv8fp06cRHR2NnTt3YubMmco6nTp1Qs+ePdGoUSMEBQXhjz/+wKNHj/DLL78Uut+JEyciIyNDebt161ZFnA6RWllZwMqVQPPmgL8/sHq1mPw0bAgsXSp2WP7hByY/RERlIVkLkKOjI4yNjZGWlqZSnpaWVmj/nSlTpqB///746KOPAAC+vr7Izs7G0KFDMWnSJBgZFcznqlSpgjfeeANXr14tNBa5XA455+MniZ0/L7b2rFsHPH4slsnlQK9ewLBhQKtW7LxMRKQtkrUAmZmZoVmzZoiNjVWW5eXlITY2Fq1atVK7zZMnTwokOcb/39GhsCXNsrKycO3aNbiWdOY4ogr07Bnw88/iiK5GjYDvvxeTn1q1gPnzxeUufvoJaN2ayQ8RkTZJOgosIiICYWFhaN68OVq2bInIyEhkZ2crR4UNGDAA7u7umDNnDgAgODgYCxYsQJMmTeDv74+rV69iypQpCA4OViZC48aNQ3BwMDw9PXHnzh1MmzYNxsbG6NOnj2TnSfS6K1fEy1irV4srqAOAiYnYgXnYMHEOHjUNmkREpCWSJkC9e/fGvXv3MHXqVKSmpqJx48bYvXu3smN0UlKSSovP5MmTIZPJMHnyZCQnJ8PJyQnBwcGYNWuWss7t27fRp08f3L9/H05OTmjbti2OHTsGJyenCj8/olfl5gLbt4uXuV6ZpgoeHsDQoeLEhGyoJCKqGDKhsGtHBiwzMxN2dnbIyMiALVd2pDJKShI7Nf/vf+LszIB4OatzZ7G1p1MnDlknItIGTX6/OUUyUTlQKIA9e8TWnp07Xy406uwMfPSRePPykjREIiKDxgSISItSU8WJCVesAG7efFn+9ttia0+XLoCZmXTxERGRiAkQURkJArB/v9jaEx0NvHghltvbA4MGif176tSRNEQiInoNEyCiUnrwAFi7Vkx8Ll9+Wd6qldja07OnuCApERHpHiZARBoQBOD4cXEx0l9+EefxAQBra6B/f3F5Cj8/aWMkIqLiMQEiKoHHj4H168XWnrNnX5b7+QHDhwN9+wKFLDVHREQ6iAkQURHOnhVbe9avF9foAgBzcyA0VLzM1bIlZ2gmItJHTIDIYCkUwMGDQEqKOAFhQIA4H8/Tp+LlreXLgWPHXtavU0dMesLCxA7ORESkv5gAkUGKjgbCw8W1tvI5OwMtWgCHDwMPH4plpqZAt25i4tO+PVt7iIgqCyZAZHCio4EePcQOza9KSwN27BAfe3mJHZoHDRITIyIiqlyYAJFBUSjElp+iFoBxdAQuXeKEhURElRnXmyaDcvCg6mUvddLTgSNHKiYeIiKSBhMgMih37pSsXkpK+cZBRETSYgJEBuPpU3FF9pJwdS3fWIiISFpMgMggpKWJC5Lu21d0PZkM8PAQh8QTEVHlxQSIKr34eMDfX5zTx94emDFDTHReH9Ke/zwyUpwPiIiIKi8mQFSp7d4NtG4N3LwJ1K4tJkFTpwJbtgDu7qp1q1cXy7t1kyZWIiKqOBwGT5XW0qXA6NFAXp44ieGvvwIODuJr3boBXbqonwmaiIgqPyZAVOm8eAFERACLF4vPBw4Efvih4Lw+xsZAhw4VHR0REekCJkBUqWRmiguV7tolPp8zBxg/nktYEBGRKiZAVGncvAm8/77Y6dnCAli3DujeXeqoiIhIFzEBokrh+HGxT09aGuDiAvz+O9C8udRRERGRruIoMNJ7v/wi9uVJSwP8/IATJ5j8EBFR0ZgAkd4SBODrr4HevYFnz8TLX4cOiRMZEhERFYUJEOmlnBwgLAyYMkV8PnYssG0bYG0taVhERKQn2AeI9E56OtC1q9jaY2wMLFkCDBsmdVRERKRPmACRXrl4UbzUde0aYGcHbN4MvPuu1FEREZG+YQJEeiM2FujRA3j0CPD2BnbsAOrXlzoqIiLSR+wDRHph5UqgY0cx+WnTRhz2zuSHiIhKiwkQ6TSFAhg3Dhg6VFziol8/YO9ewMlJ6siIiEif8RIY6aysLDHh2b5dfP7VV8DkyVzWgoiIyo4JEOmk27eB4GAgLg6Qy4E1a8Q1voiIiLSBCRDpnFOngA8+AO7cES91/fYb0KqV1FEREVFlwj5ApFO2bgXatROTnwYNxGUtmPwQEZG2MQEinSAIwNy54urtT54AQUHA4cOAl5fUkRERUWXEBIgk9/w58PHHwPjxYiI0YoQ4x4+dndSRERFRZcU+QCSphw/FVp99+wAjIyAyEhg1SuqoiIiosmMCRJK5ehV47z3g8mVxEdOoKKBzZ6mjIiIiQ8AEiCRx4IC4oOmDB0CNGuIlL19fqaMiIiJDwT5AVOHWrgUCA8Xkp2VLcVkLJj9ERFSRmABRhcnLA778Ehg4EMjNBXr2BPbvB1xcpI6MiIgMDRMgqhBPngC9ewNz5ojPJ00CNm0CLCykjYuIiAwT+wBRuUtJAbp0AU6eBExNgf/9DxgwQOqoiIjIkDEBonJ19qy4ptetW4CDgzjTc0CA1FEREZGh4yUwKjc7dgBt24rJT506wLFjTH6IiEg3MAEirRMEYOFC8bJXVhbw9tvA0aNArVpSR0ZERCRiAkRa9eKFuJTFmDHiqK+PPwZ27wbs7aWOjIiI6CX2ASKtycgAevUC/vwTkMmAefOAiAjxMRERkS5hAkRakZgIvP8+cOECYGkJbNggXgIjIiLSRUyAqMyOHAFCQoB79wA3N+D334GmTaWOioiIqHCS9wFaunQpvLy8YG5uDn9/f5w4caLI+pGRkahTpw4sLCzg4eGBsWPH4tmzZ2XaJ5Xehg1iJ+d794AmTYATJ5j8EBGR7pM0AYqKikJERASmTZuG06dPw8/PD0FBQbh7967a+hs2bMCECRMwbdo0JCQkYNWqVYiKisKXX35Z6n1S6QgCMH060K8fkJMjtgAdPAi4u0sdGRERUfFkgiAIUh3c398fLVq0wJIlSwAAeXl58PDwwKhRozBhwoQC9UeOHImEhATExsYqyz777DMcP34chw4dKtU+1cnMzISdnR0yMjJga2tb1tOsdHJzgbAwYONG8fnnnwPffAMYSd6eSEREhkyT32/JfrKeP3+OU6dOITAw8GUwRkYIDAzE0aNH1W7TunVrnDp1SnlJ6/r16/jjjz/QuXPnUu8TAHJycpCZmalyo8J9/72Y/JiYACtXAnPnMvkhIiL9Ilkn6PT0dCgUCjg7O6uUOzs74+LFi2q36du3L9LT09G2bVsIgoAXL15g2LBhyktgpdknAMyZMwczZswo4xkZBoUCiIwUH3/3HfDRR5KGQ0REVCp69Xf7/v37MXv2bHz//fc4ffo0oqOjsXPnTsycObNM+504cSIyMjKUt1u3bmkp4spn2zbgxg1xXa/Bg6WOhoiIqHQkawFydHSEsbEx0tLSVMrT0tLg4uKidpspU6agf//++Oj/mx18fX2RnZ2NoUOHYtKkSaXaJwDI5XLI5fIynpFh+O478X7YMHG+HyIiIn2kcQuQl5cXvvrqKyQlJZXpwGZmZmjWrJlKh+a8vDzExsaiVatWard58uQJjF7rbGJsbAwAEAShVPukkjt+HDh8GDA1FZe7ICIi0lcaJ0BjxoxBdHQ0atasiXfffRebNm1CTk5OqQ4eERGBlStXYu3atUhISMDw4cORnZ2NQYMGAQAGDBiAiRMnKusHBwdj2bJl2LRpExITExETE4MpU6YgODhYmQgVt08qvfzWn759AVdXaWMhIiIqE6GUTp06JYwaNUpwdHQU7O3thREjRginTp3SeD+LFy8WatSoIZiZmQktW7YUjh07pnytffv2QlhYmPJ5bm6uMH36dMHHx0cwNzcXPDw8hE8//VR4+PBhifdZEhkZGQIAISMjQ+Pzqaxu3hQEY2NBAAQhLk7qaIiIiArS5Pe7zPMA5ebm4vvvv8f48eORm5sLX19fjB49GoMGDYJMT1fB5DxABX3+OTB/vjjr8ytXGImIiHSGJr/fpe4EnZubi61bt2L16tWIiYnBm2++iSFDhuD27dv48ssvsXfvXmzYsKG0uycd8vgxsGKF+HjsWGljISIi0gaNE6DTp09j9erV2LhxI4yMjDBgwAB89913qFu3rrJO165d0aJFC60GStJZvRrIzATeeAP4/zkniYiI9JrGCVCLFi3w7rvvYtmyZQgJCYGpqWmBOt7e3ggNDdVKgCStVyc+HDuWMz4TEVHloHECdP36dXh6ehZZx8rKCqtXry51UKQ7tm8HEhOBqlWBAQOkjoaIiEg7NP57/u7duzh+/HiB8uPHj+Off/7RSlCkOxYsEO858SEREVUmGidAI0aMULtURHJyMkZwdrxK5eRJ4NAhTnxIRESVj8YJ0IULF9C0adMC5U2aNMGFCxe0EhTphvyJD0NDATc3aWMhIiLSJo0TILlcXmCtLQBISUmBiYlkS4uRlt26Bfzyi/iYQ9+JiKiy0TgB+s9//qNcPT3fo0eP8OWXX+Ldd9/VanAknSVLxBFgHToATZpIHQ0REZF2adxkM3/+fLRr1w6enp5o8v+/jHFxcXB2dsa6deu0HiBVvKws4IcfxMcREdLGQkREVB40ToDc3d1x7tw5rF+/HmfPnoWFhQUGDRqEPn36qJ0TiPTP6tVARgZQuzbw3ntSR0NERKR9peq0Y2VlhaFDh2o7FtIBCgWwcKH4eMwYTnxIRESVU6l7LV+4cAFJSUl4/vy5SvkHH3xQ5qBIOr//Dly7BtjbA2FhUkdDRERUPko1E3TXrl1x/vx5yGQy5C8mn7/yu0Kh0G6EVKHyh74PGwZYWUkbCxERUXnR+AJHeHg4vL29cffuXVhaWuLff//FgQMH0Lx5c+zfv78cQqSK8s8/wIEDgIkJJz4kIqLKTeMWoKNHj+Kvv/6Co6MjjIyMYGRkhLZt22LOnDkYPXo0zpw5Ux5xUgV4deJDd3dpYyEiIipPGrcAKRQK2NjYAAAcHR1x584dAICnpycuXbqk3eiowty+zYkPiYjIcGjcAtSwYUOcPXsW3t7e8Pf3x9y5c2FmZoYVK1agZs2a5REjVYAlS4AXL4D27QE1K50QERFVKhonQJMnT0Z2djYA4KuvvsL777+PgIAAODg4ICoqSusBUvnjxIdERGRoNE6AgoKClI9r1aqFixcv4sGDB7C3t1eOBCP9snYt8OgRUKsW8P77xddXKICDB4GUFMDVFQgIAIyNyz1MIiIirdGoD1Bubi5MTEwQHx+vUl61alUmP3oqLw+IjBQfl2Tiw+howMsLeOstoG9f8d7LSywnIiLSFxolQKampqhRowbn+qlEduwArl4FqlQpfuLD6GigRw+xw/SrkpPFciZBRESkLzQeBTZp0iR8+eWXePDgQXnEQxVswQLx/pNPAGvrwuspFEB4OPD/816qyC8bM0asR0REpOs07gO0ZMkSXL16FW5ubvD09ITVa9MFnz59WmvBUfk6fRr4+29x4sORI4uue/BgwZafVwkCcOuWWK9DB62GSUREpHUaJ0AhISHlEAZJIX/iw169gOrVi66bklKyfZa0HhERkZQ0ToCmTZtWHnFQBUtOBjZtEh+XZOJDV9eS7bek9YiIiKSkcR8gqhyWLhUnPmzXDmjevPj6AQFiK1Fhg/1kMsDDQ6xHRESk6zROgIyMjGBsbFzojXRfdjawfLn4uKTLXhgbAwsXio9fT4Lyn0dGcj4gIiLSDxpfAtu6davK89zcXJw5cwZr167FjBkztBYYlZ+1a4GHDwEfHyA4uOTbdesGbNkijgZ7tUN09epi8tOtm9ZDJSIiKhcyQVA3sFlzGzZsQFRUFH777Tdt7E5SmZmZsLOzQ0ZGBmxtbaUOR6vy8oC6dYErV4BFi4BRozTfB2eCJiIiXaTJ77fWEqDr16+jUaNGyMrK0sbuJFWZE6Dffwc++ACwsxNbcYqa+4eIiEifaPL7rZVO0E+fPsWiRYvg7u6ujd1ROcof+l7cxIdERESVmcZ9gF5f9FQQBDx+/BiWlpb4+eeftRocadeZM8C+feLlquImPiQiIqrMNE6AvvvuO5UEyMjICE5OTvD394e9vb1WgyPtenXiQw8PaWMhIiKSktb6AFUmlbEP0J074qrtubnAyZMlm/uHiIhIn5RrH6DVq1dj8+bNBco3b96MtWvXaro7qiBLl4rJT9u2TH6IiIg0ToDmzJkDR0fHAuXVqlXD7NmztRIUadeTJy8nPoyIkDYWIiIiXaBxApSUlARvb+8C5Z6enkhKStJKUKRdP/0EPHgA1KwpDoEnIiIydBonQNWqVcO5c+cKlJ89exYODg5aCYq0Jy/vZefn8HBOWEhERASUIgHq06cPRo8ejX379kGhUEChUOCvv/5CeHg4QkNDyyNGKoNdu4DLl8WJDwcNkjoaIiIi3aDxMPiZM2fixo0beOedd2BiIm6el5eHAQMGsA+QDlqwQLz/+GPAxkbaWIiIiHRFqYfBX7lyBXFxcbCwsICvry88PT21HZtkKssw+Lg4oEkT8bLX9etAjRpSR0RERFR+NPn91rgFKF/t2rVRu3bt0m5OFSAyUrzv2ZPJDxER0as07gPUvXt3/Pe//y1QPnfuXPTs2VMrQVHZpaQAGzaIj8eOlTYWIiIiXaNxAnTgwAF07ty5QHmnTp1w4MABrQRFZff99+LEh23aAC1bSh0NERGRbtE4AcrKyoKZmVmBclNTU2RmZmolKCqbJ0+AZcvEx2z9ISIiKkjjBMjX1xdRUVEFyjdt2oT69etrJSgqm3XrgPv3AW9vICRE6miIiIh0j8adoKdMmYJu3brh2rVrePvttwEAsbGx2LBhA7Zs2aL1AEkzeXkvOz9z4kMiIiL1NE6AgoODsW3bNsyePRtbtmyBhYUF/Pz88Ndff6Fq1arlESNpYPdu4OJFwNYWGDxY6miIiIh0k8aXwADgvffew+HDh5GdnY3r16+jV69eGDduHPz8/EoVxNKlS+Hl5QVzc3P4+/vjxIkThdbt0KEDZDJZgdt7772nrDNw4MACr3fs2LFUsekbTnxIRERUvFIlQIA4GiwsLAxubm749ttv8fbbb+PYsWMa7ycqKgoRERGYNm0aTp8+DT8/PwQFBeHu3btq60dHRyMlJUV5i4+Ph7GxcYEh+B07dlSpt3HjxlKdpz45dw6IjQWMjIBRo6SOhoiISHdpdAksNTUVa9aswapVq5CZmYlevXohJycH27ZtK3UH6AULFuDjjz/GoP9fqGr58uXYuXMnfvzxR0yYMKFA/dcvs23atAmWlpYFEiC5XA4XF5dSxaSv8hc97dEDqEQTcxMREWldiVuAgoODUadOHZw7dw6RkZG4c+cOFi9eXKaDP3/+HKdOnUJgYODLgIyMEBgYiKNHj5ZoH6tWrUJoaCisrKxUyvfv349q1aqhTp06GD58OO7fv1+mWHVdaurLiQ8jIqSNhYiISNeVuAVo165dGD16NIYPH661JTDS09OhUCjg7OysUu7s7IyLFy8Wu/2JEycQHx+PVatWqZR37NgR3bp1g7e3N65du4Yvv/wSnTp1wtGjR2GsZlhUTk4OcnJylM/1cT6j778Hnj8HWrUC/P2ljoaIiEi3lbgF6NChQ3j8+DGaNWsGf39/LFmyBOnp6eUZW7FWrVoFX19ftHxtquPQ0FB88MEH8PX1RUhICHbs2IGTJ09i//79avczZ84c2NnZKW8eHh4VEL32PH36cuJDtv4QEREVr8QJ0JtvvomVK1ciJSUFn3zyCTZt2gQ3Nzfk5eUhJiYGjx8/1vjgjo6OMDY2Rlpamkp5Wlpasf13srOzsWnTJgwZMqTY49SsWROOjo64evWq2tcnTpyIjIwM5e3WrVslPwkd8PPPQHo64OXFiQ+JiIhKQuNRYFZWVhg8eDAOHTqE8+fP47PPPsM333yDatWq4YMPPtBoX2ZmZmjWrBliY2OVZXl5eYiNjUWrVq2K3Hbz5s3IycnBhx9+WOxxbt++jfv378PV1VXt63K5HLa2tio3fSEILzs/jx4NmGg8sxMREZHhKfUweACoU6cO5s6di9u3b5d6mHlERARWrlyJtWvXIiEhAcOHD0d2drZyVNiAAQMwceLEAtutWrUKISEhcHBwUCnPysrC559/jmPHjuHGjRuIjY1Fly5dUKtWLQQFBZUqRl22Zw+QkCDO+VOCxjAiIiJCKWaCVsfY2BghISEIKcX1l969e+PevXuYOnUqUlNT0bhxY+zevVvZMTopKQlGRqp52qVLl3Do0CH8+eefamM5d+4c1q5di0ePHsHNzQ3/+c9/MHPmTMjl8lKdny7Ln/jwo4/E2Z+JiIioeDJBEASpg9A1mZmZsLOzQ0ZGhk5fDjt/HmjUSJz48No1sQ8QERGRodLk97tMl8BIWvmLnnbvzuSHiIhIE0yA9FRamjj6CwDGjpU2FiIiIn3DBEhPLVsmTnz45pvi5IdERERUckyA9NDTp+LMzwAnPiQiIioNJkB6aP164N49ccHTrl2ljoaIiEj/MAHSM5z4kIiIqOyYAOmZP/8ELlwArK058SEREVFpMQHSM69OfGhnJ20sRERE+ooJkB6JjxdbgIyMxMtfREREVDpMgPRI/sSHXbsC3t6ShkJERKTXmADpibt3X058yKHvREREZcMESE8sWwbk5AAtW3LiQyIiorJiAqQHnj0Dli4VH0dEADKZtPEQERHpOyZAemDDBnHiwxo1xIVPiYiIqGyYAOk4QXg59H3UKE58SEREpA1MgHTc3r3Av/+KEx9+9JHU0RAREVUOTIB0XH7rz5AhQJUqkoZCRERUaTAB0mEXLgC7d4udnjnxIRERkfYwAdJhr058WLOmpKEQERFVKkyAdNS9e8BPP4mPx46VNhYiIqLKhgmQjlq+XJz4sEULoE0bqaMhIiKqXJgA6aBnz4AlS8THnPiQiIhI+5gA6aCNG8W1v6pX58SHRERE5YEJkI4RBOC778THo0cDpqbSxkNERFQZMQHSMbGxwPnzgJUV8PHHUkdDRERUOTEB0jH5Ex8OHsyJD4mIiMoLEyAdkpAA7NoldnoOD5c6GiIiosqLCZAOyZ/4sEsXwMdH0lCIiIgqNSZAOiI9/eXEhxER0sZCRERU2TEB0hHLl4vz/zRvDrRtK3U0RERElRsTIB2Qk/Ny4sOxYznxIRERUXljAqQDNm0C0tIAd3egZ0+poyEiIqr8mABJTBBeDn0fNYoTHxIREVUEJkAS27cPOHcOsLQEhg6VOhoiIiLDwARIYq9OfGhvL20sREREhoIJkIQuXgR27uTEh0RERBWNCZCEFi4U7z/4AKhVS9pYiIiIDAkTIIncvw+sXSs+5sSHREREFYsJkER++AF4+hRo2hQICJA6GiIiIsPCBEgCOTnA4sXi44gITnxIRERU0ZgASSAqCkhNBdzcOPEhERGRFJgAVTBBAL77Tnw8ahRgZiZtPERERIaICVAF278fiIvjxIdERERSYgJUwfInPhw4EKhaVdJQiIiIDBYToAp0+TKwYwcnPiQiIpIaE6AKFBkp3gcHA2+8IWkoREREBs1E6gAMyWefASYmHPlFREQkNSZAFcjHB1i0SOooiIiIiJfAiIiIyOAwASIiIiKDoxMJ0NKlS+Hl5QVzc3P4+/vjxIkThdbt0KEDZDJZgdt7772nrCMIAqZOnQpXV1dYWFggMDAQV65cqYhTISIiIj0geQIUFRWFiIgITJs2DadPn4afnx+CgoJw9+5dtfWjo6ORkpKivMXHx8PY2Bg9X+lZPHfuXCxatAjLly/H8ePHYWVlhaCgIDx79qyiTouIiIh0mEwQBEHKAPz9/dGiRQssWbIEAJCXlwcPDw+MGjUKEyZMKHb7yMhITJ06FSkpKbCysoIgCHBzc8Nnn32GcePGAQAyMjLg7OyMNWvWIDQ0tNh9ZmZmws7ODhkZGbC1tS3bCRIREVGF0OT3W9IWoOfPn+PUqVMIDAxUlhkZGSEwMBBHjx4t0T5WrVqF0NBQWFlZAQASExORmpqqsk87Ozv4+/sXus+cnBxkZmaq3IiIiKjykjQBSk9Ph0KhgLOzs0q5s7MzUlNTi93+xIkTiI+Px0cffaQsy99Ok33OmTMHdnZ2ypuHh4emp0JERER6RPI+QGWxatUq+Pr6omXLlmXaz8SJE5GRkaG83bp1S0sREhERkS6SNAFydHSEsbEx0tLSVMrT0tLg4uJS5LbZ2dnYtGkThgwZolKev50m+5TL5bC1tVW5ERERUeUlaQJkZmaGZs2aITY2VlmWl5eH2NhYtGrVqshtN2/ejJycHHz44Ycq5d7e3nBxcVHZZ2ZmJo4fP17sPomIiMgwSL4URkREBMLCwtC8eXO0bNkSkZGRyM7OxqBBgwAAAwYMgLu7O+bMmaOy3apVqxASEgIHBweVcplMhjFjxuDrr79G7dq14e3tjSlTpsDNzQ0hISEVdVpERESkwyRPgHr37o179+5h6tSpSE1NRePGjbF7925lJ+akpCQYGak2VF26dAmHDh3Cn3/+qXafX3zxBbKzszF06FA8evQIbdu2xe7du2Fubl7u50NERES6T/J5gHQR5wEiIiLSP3ozDxARERGRFJgAERERkcFhAkREREQGhwkQERERGRwmQERERGRwmAARERGRwWECRERERAaHCRAREREZHCZAREREZHCYABEREZHBYQJEREREBocJEBERERkcJkBERERkcJgAERERkcFhAkREREQGhwkQERERGRwmQERERGRwmAARERGRwWECRERERAaHCRAREREZHCZAREREZHCYABEREZHBYQJEREREBocJEBERERkcJkBERERkcJgAERERkcFhAkREREQGhwkQERERGRwmQERERGRwmAARERGRwWECRERERAaHCRAREREZHCZAREREZHCYABEREZHBYQJEREREBsdE6gCIiMiwKBQK5ObmSh0G6SFTU1MYGxtrZV9MgIiIqEIIgoDU1FQ8evRI6lBIj1WpUgUuLi6QyWRl2g8TICIiqhD5yU+1atVgaWlZ5h8wMiyCIODJkye4e/cuAMDV1bVM+2MCRERE5U6hUCiTHwcHB6nDIT1lYWEBALh79y6qVatWpsth7ARNRETlLr/Pj6WlpcSRkL7L/w6VtR8ZEyAiIqowvOxFZaWt7xATICIiogrk5eWFyMjIEtffv38/ZDIZO49rGfsAERGR3lAogIMHgZQUwNUVCAgAtDQquoDiWhqmTZuG6dOna7zfkydPwsrKqsT1W7dujZSUFNjZ2Wl8LCocEyAiItIL0dFAeDhw+/bLsurVgYULgW7dtH+8lJQU5eOoqChMnToVly5dUpZZW1srHwuCAIVCAROT4n9WnZycNIrDzMwMLi4uGm1DxeMlMCIi0nnR0UCPHqrJDwAkJ4vl0dHaP6aLi4vyZmdnB5lMpnx+8eJF2NjYYNeuXWjWrBnkcjkOHTqEa9euoUuXLnB2doa1tTVatGiBvXv3quz39UtgMpkM//vf/9C1a1dYWlqidu3a2L59u/L11y+BrVmzBlWqVMGePXtQr149WFtbo2PHjioJ24sXLzB69GhUqVIFDg4OGD9+PMLCwhASElLo+d6/fx99+vSBu7s7LC0t4evri40bN6rUycvLw9y5c1GrVi3I5XLUqFEDs2bNUr5++/Zt9OnTB1WrVoWVlRWaN2+O48ePl+LdL39MgIiISKcpFGLLjyAUfC2/bMwYsV5FmzBhAr755hskJCSgUaNGyMrKQufOnREbG4szZ86gY8eOCA4ORlJSUpH7mTFjBnr16oVz586hc+fO6NevHx48eFBo/SdPnmD+/PlYt24dDhw4gKSkJIwbN075+n//+1+sX78eq1evxuHDh5GZmYlt27YVGcOzZ8/QrFkz7Ny5E/Hx8Rg6dCj69++PEydOKOtMnDgR33zzDaZMmYILFy5gw4YNcHZ2BgBkZWWhffv2SE5Oxvbt23H27Fl88cUXyMvLK8E7KQGBCsjIyBAACBkZGVKHQkRUKTx9+lS4cOGC8PTpU4233bdPEMRUp+jbvn1aD1tp9erVgp2d3Ssx7RMACNu2bSt22wYNGgiLFy9WPvf09BS+++475XMAwuTJk5XPs7KyBADCrl27VI718OFDZSwAhKtXryq3Wbp0qeDs7Kx87uzsLMybN0/5/MWLF0KNGjWELl26lPSUBUEQhPfee0/47LPPBEEQhMzMTEEulwsrV65UW/eHH34QbGxshPv372t0DE0V9V3S5PebfYCIiEinvXJlRyv1tKl58+Yqz7OysjB9+nTs3LkTKSkpePHiBZ4+fVpsC1CjRo2Uj62srGBra6uc8VgdS0tL+Pj4KJ+7uroq62dkZCAtLQ0tW7ZUvm5sbIxmzZoV2RqjUCgwe/Zs/PLLL0hOTsbz58+Rk5OjnHcnISEBOTk5eOedd9RuHxcXhyZNmqBq1apFnquuYAJEREQ6raQrHpRxZYRSeX0017hx4xATE4P58+ejVq1asLCwQI8ePfD8+fMi92NqaqryXCaTFZmsqKsvqLtGqIF58+Zh4cKFiIyMhK+vL6ysrDBmzBhl7PmzMBemuNd1DfsAERGRTgsIEEd7FTYqXSYDPDzEelI7fPgwBg4ciK5du8LX1xcuLi64ceNGhcZgZ2cHZ2dnnDx5UlmmUChw+vTpIrc7fPgwunTpgg8//BB+fn6oWbMmLl++rHy9du3asLCwQGxsrNrtGzVqhLi4uCL7LukSyROgpUuXwsvLC+bm5vD391fpbKXOo0ePMGLECLi6ukIul+ONN97AH3/8oXx9+vTpkMlkKre6deuW92kQEVE5MTYWh7oDBZOg/OeRkeU3H5AmateujejoaMTFxeHs2bPo27evJJ2AR40ahTlz5uC3337DpUuXEB4ejocPHxY5t1Ht2rURExODI0eOICEhAZ988gnS0tKUr5ubm2P8+PH44osv8NNPP+HatWs4duwYVq1aBQDo06cPXFxcEBISgsOHD+P69ev49ddfcfTo0XI/39KQNAGKiopCREQEpk2bhtOnT8PPzw9BQUGFXvd8/vw53n33Xdy4cQNbtmzBpUuXsHLlSri7u6vUa9CgAVJSUpS3Q4cOVcTpEBFROenWDdiyBXjtv3tUry6Wl8c8QKWxYMEC2Nvbo3Xr1ggODkZQUBCaNm1a4XGMHz8effr0wYABA9CqVStYW1sjKCgI5ubmhW4zefJkNG3aFEFBQejQoYMymXnVlClT8Nlnn2Hq1KmoV68eevfurfzNNjMzw59//olq1aqhc+fO8PX1xTfffFOmBUvLk0wo60XDMvD390eLFi2wZMkSAOL8Ah4eHhg1ahQmTJhQoP7y5csxb948XLx4scD1z3zTp0/Htm3bEBcXV+q4MjMzYWdnh4yMDNja2pZ6P0REJHr27BkSExPh7e1d5I9wcSpyJujKJC8vD/Xq1UOvXr0wc+ZMqcMpk6K+S5r8fkvWAvT8+XOcOnUKgYGBL4MxMkJgYGChzWXbt29Hq1atMGLECDg7O6Nhw4aYPXs2FK9N/nDlyhW4ubmhZs2a6NevX7G973NycpCZmalyIyIi3WNsDHToAPTpI94z+VHv5s2bWLlyJS5fvozz589j+PDhSExMRN++faUOTWdIlgClp6dDoVAoJ1DK5+zsjNTUVLXbXL9+HVu2bIFCocAff/yBKVOm4Ntvv8XXX3+trOPv7481a9Zg9+7dWLZsGRITExEQEIDHjx8XGsucOXNgZ2envHl4eGjnJImIiCRgZGSENWvWoEWLFmjTpg3Onz+PvXv3ol69elKHpjP0ahh8Xl4eqlWrhhUrVijnNEhOTsa8efMwbdo0AECnTp2U9Rs1agR/f394enril19+wZAhQ9Tud+LEiYiIiFA+z8zMZBJERER6y8PDA4cPH5Y6DJ0mWQLk6OgIY2NjlR7mAJCWllboom+urq4wNTVV6VBVr149pKam4vnz5zAzMyuwTZUqVfDGG2/g6tWrhcYil8shl8tLeSZERESkbyS7BGZmZoZmzZqpzCeQl5eH2NhYtGrVSu02bdq0wdWrV1WGFF6+fBmurq5qkx9AnJXz2rVrcJVihiwiIiLSSZIOg4+IiMDKlSuxdu1aJCQkYPjw4cjOzsagQYMAAAMGDMDEiROV9YcPH44HDx4gPDwcly9fxs6dOzF79myMGDFCWWfcuHH4+++/cePGDRw5cgRdu3aFsbEx+vTpU+HnR0RERLpJ0j5AvXv3xr179zB16lSkpqaicePG2L17t7JjdFJSEoyMXuZoHh4e2LNnD8aOHYtGjRrB3d0d4eHhGD9+vLLO7du30adPH9y/fx9OTk5o27Ytjh07Bicnpwo/PyIiItJNks4DpKs4DxARkXZpax4gIr2fB4iIiIhIKkyAiIiIylGHDh0wZswY5XMvLy9ERkYWuY1MJsO2bdvKfGxt7acyYgJERESkRnBwMDp27Kj2tYMHD0Imk+HcuXMa7/fkyZMYOnRoWcNTMX36dDRu3LhAeUpKisr8ePQSEyAiIiI1hgwZgpiYGNy+fbvAa6tXr0bz5s3RqFEjjffr5OQES0tLbYRYLBcXF85zVwgmQERERGq8//77cHJywpo1a1TKs7KysHnzZgwZMgT3799Hnz594O7uDktLS/j6+mLjxo1F7vf1S2BXrlxBu3btYG5ujvr16yMmJqbANuPHj8cbb7wBS0tL1KxZE1OmTEFubi4AYM2aNZgxYwbOnj0LmUwGmUymjPn1S2Dnz5/H22+/DQsLCzg4OGDo0KHIyspSvj5w4ECEhIRg/vz5cHV1hYODA0aMGKE8ljrXrl1Dly5d4OzsDGtra7Ro0QJ79+5VqZOTk4Px48fDw8MDcrkctWrVwqpVq5Sv//vvv3j//fdha2sLGxsbBAQE4Nq1a0W+j2WlV0thEBFR5SEIwJMnFX9cS0tAJiu+nomJCQYMGIA1a9Zg0qRJkP3/Rps3b4ZCoUCfPn2QlZWFZs2aYfz48bC1tcXOnTvRv39/+Pj4oGXLlsUeIy8vD926dYOzszOOHz+OjIwMlf5C+WxsbLBmzRq4ubnh/Pnz+Pjjj2FjY4MvvvgCvXv3Rnx8PHbv3q1MPOzs7ArsIzs7G0FBQWjVqhVOnjyJu3fv4qOPPsLIkSNVkrx9+/bB1dUV+/btw9WrV9G7d280btwYH3/8sdpzyMrKQufOnTFr1izI5XL89NNPCA4OxqVLl1CjRg0A4rx+R48exaJFi+Dn54fExESkp6cDAJKTk9GuXTt06NABf/31F2xtbXH48GG8ePGi2PevTAQqICMjQwAgZGRkaHW/L14Iwr59grBhg3j/4oVWd09EpLOePn0qXLhwQXj69KmyLCtLEMQ0qGJvWVkljzshIUEAIOzbt09ZFhAQIHz44YeFbvPee+8Jn332mfJ5+/bthfDwcOVzT09P4bvvvhMEQRD27NkjmJiYCMnJycrXd+3aJQAQtm7dWugx5s2bJzRr1kz5fNq0aYKfn1+Beq/uZ8WKFYK9vb2Q9cobsHPnTsHIyEhITU0VBEEQwsLCBE9PT+HFKz9QPXv2FHr37l1oLOo0aNBAWLx4sSAIgnDp0iUBgBATE6O27sSJEwVvb2/h+fPnJdq3uu9SPk1+v3kJrIJERwNeXsBbbwF9+4r3Xl5iORER6aa6deuidevW+PHHHwEAV69excGDB5WLaysUCsycORO+vr6oWrUqrK2tsWfPHiQlJZVo/wkJCfDw8ICbm5uyTN1yUFFRUWjTpg1cXFxgbW2NyZMnl/gYrx7Lz88PVlZWyrI2bdogLy8Ply5dUpY1aNBAZc1NV1dX3L17t9D9ZmVlYdy4cahXrx6qVKkCa2trJCQkKOOLi4uDsbEx2rdvr3b7uLg4BAQEwNTUVKPzKSteAqsA0dFAjx7i3x6vSk4Wy7dsAbp1kyY2IiKpWFoCr3Q/qdDjamLIkCEYNWoUli5ditWrV8PHx0f5Yz5v3jwsXLgQkZGR8PX1hZWVFcaMGYPnz59rLd6jR4+iX79+mDFjBoKCgmBnZ4dNmzbh22+/1doxXvV6IiKTyVTW4HzduHHjEBMTg/nz56NWrVqwsLBAjx49lO+BhYVFkccr7vXywgSonCkUQHh4weQHEMtkMmDMGKBLF+CVhJuIqNKTyYBXGiN0Vq9evRAeHo4NGzbgp59+wvDhw5X9gQ4fPowuXbrgww8/BCD26bl8+TLq169fon3Xq1cPt27dQkpKinLR7mPHjqnUOXLkCDw9PTFp0iRl2c2bN1XqmJmZQaFQFHusNWvWIDs7W9kKdPjwYRgZGaFOnToliledw4cPY+DAgejatSsAsUXoxo0bytd9fX2Rl5eHv//+G4GBgQW2b9SoEdauXYvc3NwKbQXiJbBydvAgoGYEpZIgALduifWIiEj3WFtbo3fv3pg4cSJSUlIwcOBA5Wu1a9dGTEwMjhw5goSEBHzyySdIS0sr8b4DAwPxxhtvICwsDGfPnsXBgwdVEp38YyQlJWHTpk24du0aFi1ahK1bt6rU8fLyQmJiIuLi4pCeno6cnJwCx+rXrx/Mzc0RFhaG+Ph47Nu3D6NGjUL//v2Va3CWRu3atREdHY24uDicPXsWffv2VWkx8vLyQlhYGAYPHoxt27YhMTER+/fvxy+//AIAGDlyJDIzMxEaGop//vkHV65cwbp161Quy5UHJkDlLCVFu/WIiKjiDRkyBA8fPkRQUJBKf53JkyejadOmCAoKQocOHeDi4oKQkJAS79fIyAhbt27F06dP0bJlS3z00UeYNWuWSp0PPvgAY8eOxciRI9G4cWMcOXIEU6ZMUanTvXt3dOzYEW+99RacnJzUDsW3tLTEnj178ODBA7Ro0QI9evTAO++8gyVLlmj2ZrxmwYIFsLe3R+vWrREcHIygoCA0bdpUpc6yZcvQo0cPfPrpp6hbty4+/vhjZGdnAwAcHBzw119/ISsrC+3bt0ezZs2wcuXKcm8N4mKoamhzMdT9+8UOz8XZtw/o0KFMhyIi0llcDJW0hYuh6omAAKB69cLnnJDJAA8PsR4RERFVDCZA5czYGFi4UHz8ehKU/zwykh2giYiIKhIToArQrZs41N3dXbW8enUOgSciIpICh8FXkG7dxKHuBw+KHZ5dXcXLXmz5ISIiqnhMgCqQsTE7OhMREekCXgIjIqIKw4HHVFba+g4xASIionKXP6fLEymWf6dKJf87VNZ5gngJjIiIyp2xsTGqVKmiXFTT0tJSuZwEUUkIgoAnT57g7t27qFKlisqCraXBBIiIiCqEi4sLABS5sjhRcapUqaL8LpUFEyAiIqoQMpkMrq6uqFatGnJzc6UOh/SQqalpmVt+8jEBIiKiCmVsbKy1HzGi0mInaCIiIjI4TICIiIjI4DABIiIiIoPDPkBq5E+ylJmZKXEkREREVFL5v9slmSyRCZAajx8/BgB4eHhIHAkRERFp6vHjx7CzsyuyjkzgvOQF5OXl4c6dO7CxseFEXYXIzMyEh4cHbt26BVtbW6nDMXj8PHQLPw/dws9Dt5Tn5yEIAh4/fgw3NzcYGRXdy4ctQGoYGRmhevXqUoehF2xtbfkfig7h56Fb+HnoFn4euqW8Po/iWn7ysRM0ERERGRwmQERERGRwmABRqcjlckybNg1yuVzqUAj8PHQNPw/dws9Dt+jK58FO0ERERGRw2AJEREREBocJEBERERkcJkBERERkcJgAERERkcFhAkQlNmfOHLRo0QI2NjaoVq0aQkJCcOnSJanDov/3zTffQCaTYcyYMVKHYtCSk5Px4YcfwsHBARYWFvD19cU///wjdVgGSaFQYMqUKfD29oaFhQV8fHwwc+bMEq0TRWV34MABBAcHw83NDTKZDNu2bVN5XRAETJ06Fa6urrCwsEBgYCCuXLlSYfExAaIS+/vvvzFixAgcO3YMMTExyM3NxX/+8x9kZ2dLHZrBO3nyJH744Qc0atRI6lAM2sOHD9GmTRuYmppi165duHDhAr799lvY29tLHZpB+u9//4tly5ZhyZIlSEhIwH//+1/MnTsXixcvljo0g5CdnQ0/Pz8sXbpU7etz587FokWLsHz5chw/fhxWVlYICgrCs2fPKiQ+DoOnUrt37x6qVauGv//+G+3atZM6HIOVlZWFpk2b4vvvv8fXX3+Nxo0bIzIyUuqwDNKECRNw+PBhHDx4UOpQCMD7778PZ2dnrFq1SlnWvXt3WFhY4Oeff5YwMsMjk8mwdetWhISEABBbf9zc3PDZZ59h3LhxAICMjAw4OztjzZo1CA0NLfeY2AJEpZaRkQEAqFq1qsSRGLYRI0bgvffeQ2BgoNShGLzt27ejefPm6NmzJ6pVq4YmTZpg5cqVUodlsFq3bo3Y2FhcvnwZAHD27FkcOnQInTp1kjgySkxMRGpqqsr/W3Z2dvD398fRo0crJAYuhkqlkpeXhzFjxqBNmzZo2LCh1OEYrE2bNuH06dM4efKk1KEQgOvXr2PZsmWIiIjAl19+iZMnT2L06NEwMzNDWFiY1OEZnAkTJiAzMxN169aFsbExFAoFZs2ahX79+kkdmsFLTU0FADg7O6uUOzs7K18rb0yAqFRGjBiB+Ph4HDp0SOpQDNatW7cQHh6OmJgYmJubSx0OQfzDoHnz5pg9ezYAoEmTJoiPj8fy5cuZAEngl19+wfr167FhwwY0aNAAcXFxGDNmDNzc3Ph5EC+BkeZGjhyJHTt2YN++fahevbrU4RisU6dO4e7du2jatClMTExgYmKCv//+G4sWLYKJiQkUCoXUIRocV1dX1K9fX6WsXr16SEpKkigiw/b5559jwoQJCA0Nha+vL/r374+xY8dizpw5Uodm8FxcXAAAaWlpKuVpaWnK18obEyAqMUEQMHLkSGzduhV//fUXvL29pQ7JoL3zzjs4f/484uLilLfmzZujX79+iIuLg7GxsdQhGpw2bdoUmBri8uXL8PT0lCgiw/bkyRMYGan+zBkbGyMvL0+iiCift7c3XFxcEBsbqyzLzMzE8ePH0apVqwqJgZfAqMRGjBiBDRs24LfffoONjY3yOq2dnR0sLCwkjs7w2NjYFOh/ZWVlBQcHB/bLksjYsWPRunVrzJ49G7169cKJEyewYsUKrFixQurQDFJwcDBmzZqFGjVqoEGDBjhz5gwWLFiAwYMHSx2aQcjKysLVq1eVzxMTExEXF4eqVauiRo0aGDNmDL7++mvUrl0b3t7emDJlCtzc3JQjxcqdQFRCANTeVq9eLXVo9P/at28vhIeHSx2GQfv999+Fhg0bCnK5XKhbt66wYsUKqUMyWJmZmUJ4eLhQo0YNwdzcXKhZs6YwadIkIScnR+rQDMK+ffvU/maEhYUJgiAIeXl5wpQpUwRnZ2dBLpcL77zzjnDp0qUKi4/zABEREZHBYR8gIiIiMjhMgIiIiMjgMAEiIiIig8MEiIiIiAwOEyAiIiIyOEyAiIiIyOAwASIiIiKDwwSIiKgQMpkM27ZtkzoMIioHTICISCcNHDgQMpmswK1jx45Sh0ZElQDXAiMindWxY0esXr1apUwul0sUDRFVJmwBIiKdJZfL4eLionKzt7cHIF6eWrZsGTp16gQLCwvUrFkTW7ZsUdn+/PnzePvtt2FhYQEHBwcMHToUWVlZKnV+/PFHNGjQAHK5HK6urhg5cqTK6+np6ejatSssLS1Ru3ZtbN++Xfnaw4cP0a9fPzg5OcHCwgK1a9cukLARkW5iAkREemvKlCno3r07zp49i379+iE0NBQJCQkAgOzsbAQFBcHe3h4nT57E5s2bsXfvXpUEZ9myZRgxYgSGDh2K8+fPY/v27ahVq5bKMWbMmIFevXrh3Llz6Ny5M/r164cHDx4oj3/hwgXs2rULCQkJWLZsGRwdHSvuDSCi0quwZVeJiDQQFhYmGBsbC1ZWViq3WbNmCYIgCACEYcOGqWzj7+8vDB8+XBAEQVixYoVgb28vZGVlKV/fuXOnYGRkJKSmpgqCIAhubm7CpEmTCo0BgDB58mTl86ysLAGAsGvXLkEQBCE4OFgYNGiQdk6YiCoU+wARkc566623sGzZMpWyqlWrKh+3atVK5bVWrVohLi4OAJCQkAA/Pz9YWVkpX2/Tpg3y8vJw6dIlyGQy3LlzB++8806RMTRq1Ej52MrKCra2trh79y4AYPjw4ejevTtOnz6N//znPwgJCUHr1q1Lda5EVLGYABGRzrKysipwSUpbLCwsSlTP1NRU5blMJkNeXh4AoFOnTrh58yb++OMPxMTE4J133sGIESMwf/58rcdLRNrFPkBEpLeOHTtW4Hm9evUAAPXq1cPZs2eRnZ2tfP3w4cMwMjJCnTp1YGNjAy8vL8TGxpYpBicnJ4SFheHnn39GZGQkVqxYUab9EVHFYAsQEemsnJwcpKamqpSZmJgoOxpv3rwZzZs3R9u2bbF+/XqcOHECq1atAgD069cP06ZNQ1hYGKZPn4579+5h1KhR6N+/P5ydnQEA06dPx7Bhw1CtWjV06tQJjx8/xuHDhzFq1KgSxTd16lQ0a9YMDRo0QE5ODnbs2KFMwIhItzEBIiKdtXv3bri6uqqU1alTBxcvXgQgjtDatGkTPv30U7i6umLjxo2oX78+AMDS0hJ79uxBeHg4WrRoAUtLS3Tv3h0LFixQ7issLAzPnj3Dd999h3HjxsHR0RE9evQocXxmZmaYOHEibty4AQsLCwQEBGDTpk1aOHMiKm8yQRAEqYMgItKUTCbD1q1bERISInUoRKSH2AeIiIiIDA4TICIiIjI47ANERHqJV++JqCzYAkREREQGhwkQERERGRwmQERERGRwmAARERGRwWECRERERAaHCRAREREZHCZAREREZHCYABEREZHBYQJEREREBuf/AI3YOGCNAw66AAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -823,9 +851,19 @@ "plt.show()" ] }, + { + "cell_type": "markdown", + "id": "7865d6f2", + "metadata": {}, + "source": [ + "### Export the model\n", + "\n", + "We can export the model including the TextVectorization layer inside the model to conduct inference on raw text." + ] + }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 26, "id": "93b0a42c-437e-41bb-99e7-d58cb8036a3a", "metadata": {}, "outputs": [ @@ -833,8 +871,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 5ms/step - accuracy: 0.4993 - binary_accuracy: 0.0000e+00 - loss: 0.0000e+00\n", - "{'accuracy': 0.5000399947166443, 'binary_accuracy': 0.0, 'loss': 0.0}\n" + "\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 6ms/step - accuracy: 0.4957 - binary_accuracy: 0.0000e+00 - loss: 0.0000e+00\n", + "{'accuracy': 0.5, 'binary_accuracy': 0.0, 'loss': 0.0}\n" ] } ], @@ -855,27 +893,35 @@ ] }, { - "cell_type": "code", - "execution_count": 32, - "id": "8939539b-a600-48b1-a55e-3f1087f4a855", + "cell_type": "markdown", + "id": "d0795584", "metadata": {}, - "outputs": [ - { - "name": "stdout", + "source": [ + "Conduct inference on new data:" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "8939539b-a600-48b1-a55e-3f1087f4a855", + "metadata": {}, + "outputs": [ + { + "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 52ms/step\n" + "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 54ms/step\n" ] }, { "data": { "text/plain": [ - "array([[0.5858644 ],\n", - " [0.5499682 ],\n", - " [0.53613865]], dtype=float32)" + "array([[0.6689762 ],\n", + " [0.62865674],\n", + " [0.6049684 ]], dtype=float32)" ] }, - "execution_count": 32, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -900,22 +946,22 @@ }, { "cell_type": "code", - "execution_count": 33, - "id": "e2aa1770-2bf9-436b-af32-16b64fc97490", + "execution_count": 28, + "id": "3e520822", "metadata": {}, "outputs": [], "source": [ - "!rm -rf text_model" + "os.mkdir('models') if not os.path.exists('models') else None" ] }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 29, "id": "7f22cc32-2708-4808-8e76-99024da87a21", "metadata": {}, "outputs": [], "source": [ - "export_model.save('text_model.keras')" + "export_model.save('models/text_model.keras')" ] }, { @@ -928,7 +974,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 30, "id": "c9cf2c7f-5e86-4ff8-984e-dd0ed7a3ece9", "metadata": {}, "outputs": [ @@ -1020,7 +1066,7 @@ "# register callables as custom objects before loading\n", "custom_objects = {\"vectorize_layer\": vectorize_layer, \"custom_standardization\": custom_standardization}\n", "with tf.keras.utils.custom_object_scope(custom_objects):\n", - " new_model = tf.keras.models.load_model('text_model.keras', compile=False)\n", + " new_model = tf.keras.models.load_model('models/text_model.keras', compile=False)\n", "\n", "new_model.summary()" ] @@ -1035,7 +1081,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 31, "id": "531680b2-42ef-4205-9a38-6995aee9f340", "metadata": {}, "outputs": [ @@ -1049,12 +1095,12 @@ { "data": { "text/plain": [ - "array([[0.5858644 ],\n", - " [0.5499682 ],\n", - " [0.53613865]], dtype=float32)" + "array([[0.6689762 ],\n", + " [0.62865674],\n", + " [0.6049684 ]], dtype=float32)" ] }, - "execution_count": 36, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -1071,91 +1117,296 @@ "## PySpark" ] }, + { + "cell_type": "code", + "execution_count": 32, + "id": "d6d515c2-ce53-4af5-a936-ae91fdecea99", + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.ml.functions import predict_batch_udf\n", + "from pyspark.sql.functions import struct, col, array, pandas_udf\n", + "from pyspark.sql.types import ArrayType, FloatType, DoubleType\n", + "from pyspark.sql import SparkSession\n", + "from pyspark import SparkConf\n", + "import pandas as pd\n", + "import json" + ] + }, { "cell_type": "markdown", - "id": "0b5ac416-a37f-4f0b-8c77-628f0fcede69", + "id": "39c35256", "metadata": {}, "source": [ - "## Inference using Spark DL API\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment." + "Check the cluster environment to handle any platform-specific Spark configurations." ] }, { "cell_type": "code", - "execution_count": 37, - "id": "d6d515c2-ce53-4af5-a936-ae91fdecea99", + "execution_count": 36, + "id": "31de0c5f", "metadata": {}, "outputs": [], "source": [ - "import os\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import struct, col\n", - "from pyspark.sql.types import ArrayType, FloatType, DoubleType\n", - "from pyspark.sql import SparkSession\n", - "from pyspark import SparkConf" + "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n", + "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n", + "on_standalone = not (on_databricks or on_dataproc)" + ] + }, + { + "cell_type": "markdown", + "id": "55ad7f00", + "metadata": {}, + "source": [ + "#### Create Spark Session\n", + "\n", + "For local standalone clusters, we'll connect to the cluster and create the Spark Session. \n", + "For CSP environments, Spark will either be preconfigured (Databricks) or we'll need to create the Spark Session (Dataproc)." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "id": "6b653c43", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/06 22:02:26 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/01/06 22:02:26 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", + "25/01/06 22:02:26 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" + ] + } + ], "source": [ - "import os\n", - "conda_env = os.environ.get(\"CONDA_PREFIX\")\n", - "\n", "conf = SparkConf()\n", + "\n", "if 'spark' not in globals():\n", - " # If Spark is not already started with Jupyter, attach to Spark Standalone\n", - " import socket\n", - " hostname = socket.gethostname()\n", - " conf.setMaster(f\"spark://{hostname}:7077\") # assuming Master is on default port 7077\n", - "conf.set(\"spark.task.maxFailures\", \"1\")\n", - "conf.set(\"spark.driver.memory\", \"8g\")\n", - "conf.set(\"spark.executor.memory\", \"8g\")\n", - "conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", - "conf.set(\"spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled\", \"false\")\n", - "conf.set(\"spark.sql.pyspark.jvmStacktrace.enabled\", \"true\")\n", - "conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", - "conf.set(\"spark.python.worker.reuse\", \"true\")\n", - "# Create Spark Session\n", + " if on_standalone:\n", + " import socket\n", + " \n", + " conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + " hostname = socket.gethostname()\n", + " conf.setMaster(f\"spark://{hostname}:7077\")\n", + " conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", + " conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_env}/lib:{conda_env}/lib/python3.11/site-packages/nvidia_pytriton.libs:$LD_LIBRARY_PATH\")\n", + " source = \"/usr/lib/x86_64-linux-gnu/libstdc++.so.6\"\n", + " target = f\"{conda_env}/lib/libstdc++.so.6\"\n", + " try:\n", + " if os.path.islink(target) or os.path.exists(target):\n", + " os.remove(target)\n", + " os.symlink(source, target)\n", + " except OSError as e:\n", + " print(f\"Error creating symlink: {e}\")\n", + " elif on_dataproc:\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conda_lib_path=\"/opt/conda/miniconda3/lib\"\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_lib_path}:$LD_LIBRARY_PATH\")\n", + " conf.set(\"spark.executorEnv.TF_GPU_ALLOCATOR\", \"cuda_malloc_async\")\n", + " conf.set(\"spark.executor.instances\", \"4\") # dataproc defaults to 2\n", + "\n", + " conf.set(\"spark.executor.cores\", \"8\")\n", + " conf.set(\"spark.task.resource.gpu.amount\", \"0.125\")\n", + " conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n", + " conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", + " conf.set(\"spark.python.worker.reuse\", \"true\")\n", + "\n", + "conf.set(\"spark.sql.execution.arrow.maxRecordsPerBatch\", \"1000\")\n", "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", "sc = spark.sparkContext" ] }, + { + "cell_type": "markdown", + "id": "53b39d27", + "metadata": {}, + "source": [ + "Load the IMDB dataset. We'll perform inference on the first sentence of each sample." + ] + }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 38, "id": "ef3309eb", "metadata": {}, "outputs": [], "source": [ "from datasets import load_dataset\n", "\n", - "# load IMDB reviews (test) dataset\n", - "data = load_dataset(\"imdb\", split=\"test\")\n", - "lines = []\n", - "for example in data:\n", - " lines.append([example[\"text\"].split(\".\")[0]])\n", - " \n", - "df = spark.createDataFrame(lines, ['lines']).repartition(8)" + "dataset = load_dataset(\"imdb\", split=\"test\")\n", + "dataset = dataset.to_pandas().drop(columns=\"label\")" + ] + }, + { + "cell_type": "markdown", + "id": "3a7672d1", + "metadata": {}, + "source": [ + "#### Create PySpark DataFrame" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "bb05466f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "StructType([StructField('text', StringType(), True)])" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = spark.createDataFrame(dataset).repartition(8)\n", + "df.schema" ] }, { "cell_type": "code", "execution_count": 40, + "id": "3f0a594b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/06 22:02:35 WARN TaskSetManager: Stage 0 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n", + " \r" + ] + }, + { + "data": { + "text/plain": [ + "[Row(text=\"Anyone remember the first CKY, CKY2K etc..? Back when it was about making crazy cool stuff, rather than watching Bam Margera act like a douchebag, spoiled 5 year old, super/rock-star wannabe.

The show used to be awesome, however, Bam's fame and wealth has led him to believe, that we now enjoy him acting childish and idiotic, more than actual cool stuff, that used to be in ex. CKY2K.

The acts are so repetitive, there's like nothing new, except annoying stupidity and rehearsed comments... The only things we see is Bam Margera, so busy showing us how much he doesn't care, how much money he got or whatsoever.

I really got nothing much left to say except, give us back CKY2K, cause Bam suck..

I enjoy watching Steve-o, Knoxville etc. a thousand times more.\")]" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.take(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "9d9db063", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/06 22:02:36 WARN TaskSetManager: Stage 3 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + ] + } + ], + "source": [ + "data_path = \"spark-dl-datasets/imdb_test\"\n", + "if on_databricks:\n", + " data_path = \"dbfs:/FileStore/\" + data_path\n", + "\n", + "df.write.mode(\"overwrite\").parquet(data_path)" + ] + }, + { + "cell_type": "markdown", + "id": "2f78a16a", + "metadata": {}, + "source": [ + "#### Load and Preprocess PySpark DataFrame\n", + "\n", + "Define our preprocess function. We'll take the first sentence of each sample as our input for sentiment analysis." + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "1c081557", + "metadata": {}, + "outputs": [], + "source": [ + "@pandas_udf(\"string\")\n", + "def preprocess(text: pd.Series) -> pd.Series:\n", + " return pd.Series([s.split(\".\")[0] for s in text])" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "id": "60af570a", + "metadata": {}, + "outputs": [], + "source": [ + "# Limit to N rows, since this can be slow\n", + "df = spark.read.parquet(data_path).limit(512).repartition(8)" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "a690f6df", + "metadata": {}, + "outputs": [], + "source": [ + "input_df = df.select(preprocess(col(\"text\")).alias(\"lines\")).cache()" + ] + }, + { + "cell_type": "markdown", + "id": "01166d97", + "metadata": {}, + "source": [ + "## Inference using Spark DL API\n", + "\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "\n", + "- predict_batch_fn uses Tensorflow APIs to load the model and return a predict function which operates on numpy arrays \n", + "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" + ] + }, + { + "cell_type": "code", + "execution_count": 45, "id": "7b7a8395-e2ae-4c3c-bf57-763dfde600ad", "metadata": {}, "outputs": [], "source": [ - "text_model_path = \"{}/text_model.keras\".format(os.getcwd())" + "text_model_path = \"{}/models/text_model.keras\".format(os.getcwd())\n", + "\n", + "# For cloud environments, copy the model to the distributed file system.\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-models\")\n", + " dbfs_model_path = \"/dbfs/FileStore/spark-dl-models/text_model.keras\"\n", + " shutil.copy(text_model_path, dbfs_model_path)\n", + " text_model_path = dbfs_model_path\n", + "elif on_dataproc:\n", + " # GCS is mounted at /mnt/gcs by the init script\n", + " models_dir = \"/mnt/gcs/spark-dl/models\"\n", + " os.mkdir(models_dir) if not os.path.exists(models_dir) else None\n", + " gcs_model_path = models_dir + \"/text_model.keras\"\n", + " shutil.copy(text_model_path, gcs_model_path)\n", + " text_model_path = gcs_model_path" ] }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 46, "id": "8c0524cf-3a75-4fb8-8025-f0654acce13e", "metadata": {}, "outputs": [], @@ -1206,7 +1457,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 47, "id": "0d603644-d938-4c87-aa8a-2512251638d5", "metadata": {}, "outputs": [], @@ -1218,7 +1469,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 48, "id": "0b480622-8dc1-4879-933e-c43112768630", "metadata": {}, "outputs": [ @@ -1226,97 +1477,76 @@ "name": "stderr", "output_type": "stream", "text": [ - " \r" + "[Stage 9:> (0 + 8) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 22.3 ms, sys: 6.39 ms, total: 28.7 ms\n", - "Wall time: 5.95 s\n" + "CPU times: user 5.29 ms, sys: 4.43 ms, total: 9.73 ms\n", + "Wall time: 4.3 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" ] } ], "source": [ "%%time\n", - "predictions = df.withColumn(\"preds\", classify(struct(\"lines\")))\n", + "predictions = input_df.withColumn(\"preds\", classify(struct(\"lines\")))\n", "results = predictions.collect()" ] }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 49, "id": "31b0a262-387e-4a5e-a60e-b9b8ee456199", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 5:==============================================> (8 + 2) / 10]\r" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 98.3 ms, sys: 8.08 ms, total: 106 ms\n", - "Wall time: 1.24 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" + "CPU times: user 4.94 ms, sys: 0 ns, total: 4.94 ms\n", + "Wall time: 150 ms\n" ] } ], "source": [ "%%time\n", - "predictions = df.withColumn(\"preds\", classify(\"lines\"))\n", + "predictions = input_df.withColumn(\"preds\", classify(\"lines\"))\n", "results = predictions.collect()" ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 50, "id": "7ef9e431-59f5-4b29-9f79-ae16a9cfb0b9", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[Stage 8:==============================================> (8 + 2) / 10]\r" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 16.1 ms, sys: 4.41 ms, total: 20.6 ms\n", - "Wall time: 1.18 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" + "CPU times: user 2.38 ms, sys: 2.54 ms, total: 4.92 ms\n", + "Wall time: 206 ms\n" ] } ], "source": [ "%%time\n", - "predictions = df.withColumn(\"preds\", classify(col(\"lines\")))\n", + "predictions = input_df.withColumn(\"preds\", classify(col(\"lines\")))\n", "results = predictions.collect()" ] }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 51, "id": "9a325ee2-3268-414a-bb75-a5fcf794f512", "metadata": { "scrolled": true @@ -1329,26 +1559,26 @@ "+--------------------------------------------------------------------------------+----------+\n", "| lines| preds|\n", "+--------------------------------------------------------------------------------+----------+\n", - "|i do not understand at all why this movie received such good grades from crit...| 0.5006337|\n", - "| I am a big fan of The ABC Movies of the Week genre|0.57577586|\n", - "| Strangeland is a terrible horror/technological thriller| 0.5441176|\n", - "|Sex,Drugs,Rock & Roll is without a doubt the worst product of Western Civiliz...|0.53261155|\n", - "| Not to be mistaken as the highly touted Samuel L| 0.5785005|\n", - "|Following the pleasingly atmospheric original and the amusingly silly second ...|0.51597977|\n", - "| No idea how this is rated as high as it is (5|0.55052567|\n", - "|When I saw this in the cinema, I remember wincing at the bad acting about a m...|0.52347463|\n", - "| I was shocked at how bad it was and unable to turn away from the disaster| 0.5262873|\n", - "|I don't know if this exceptionally dull movie was intended as an unofficial s...| 0.5175539|\n", - "|Greetings All,

Isn't it amazing the power that films have on you a...|0.64540815|\n", - "| I'm sorry but this guy is not funny| 0.5385401|\n", - "|This movie is so dull I spent half of it on IMDb while it was open in another...| 0.5182078|\n", - "| OK, lets start with the best| 0.5611213|\n", - "|This show had a promising start as sort of the opposite of 'Oceans 11' but ha...| 0.5557351|\n", - "|The 3rd in the series finds Paul Kersey (Bronson) turning vigilante to get re...|0.56089103|\n", - "| I'm not sure I've ever seen a film as bad as this| 0.54292|\n", - "| Steven Seagal has made a really dull, bad and boring movie| 0.5089991|\n", - "| You have to acknowledge Cimino's contribution to cinema| 0.5760211|\n", - "|***SPOILER ALERT*** Disjointed and confusing arson drama that has to do with ...| 0.5469447|\n", + "|The only reason I'm even giving this movie a 4 is because it was made in to a...|0.52321863|\n", + "|Awkward disaster mishmash has a team of scavengers coming across the overturn...|0.55067354|\n", + "|Here is a fantastic concept for a film - a series of meteors crash into a sma...| 0.6197893|\n", + "| I walked out of the cinema having suffered this film after 30 mins| 0.5503541|\n", + "|A wildly uneven film where the major problem is the uneasy mix of comedy and ...| 0.5540192|\n", + "|Leonard Rossiter and Frances de la Tour carry this film, not without a strugg...| 0.5467422|\n", + "| A good cast| 0.5688838|\n", + "|Yet again, I appear to be the only person on planet Earth who is capable of c...|0.55650306|\n", + "|As a serious horror fan, I get that certain marketing ploys are used to sell ...| 0.5629433|\n", + "|Upon writing this review I have difficulty trying to think of what to write a...| 0.5383269|\n", + "| Simply awful| 0.5275883|\n", + "|I am a fan of Ed Harris' work and I really had high expectations about this film|0.55910736|\n", + "| Well|0.56994545|\n", + "| This is a new approach to comedy| 0.5674365|\n", + "| It's been mentioned by others the inane dialogue in this series and I agree|0.55741817|\n", + "|One of the most boring movies I've ever had to sit through, it's completely f...| 0.5303776|\n", + "|This movie was playing on Lifetime Movie Network last month and I decided to ...| 0.5663204|\n", + "| 1983's \"Frightmare\" is an odd little film| 0.560836|\n", + "| 'Felony' is a B-movie| 0.5602156|\n", + "| This movie defines the word \"confused\"| 0.5535761|\n", "+--------------------------------------------------------------------------------+----------+\n", "only showing top 20 rows\n", "\n" @@ -1361,79 +1591,35 @@ }, { "cell_type": "markdown", - "id": "579b53bf-5a8a-4f85-a5b5-fb82a4be7f06", + "id": "ad9b07e6", "metadata": {}, "source": [ - "### Using Triton Inference Server\n", + "## Using Triton Inference Server\n", + "In this section, we demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. \n", + "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server. \n", + "\n", + "The process looks like this:\n", + "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n", + "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n", + "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n", + "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n", "\n", - "Note: you can restart the kernel and run from this point to simulate running in a different node or environment." + "\"drawing\"" ] }, { "cell_type": "markdown", - "id": "8598edb1-acb7-4704-8f0d-20b0f431a323", + "id": "889a1623", "metadata": {}, "source": [ - "This notebook uses the [Python backend with a custom execution environment](https://github.com/triton-inference-server/python_backend#creating-custom-execution-environments) for Triton 24.08, using a conda-pack environment created as follows:\n", - "```\n", - "conda create -n tf-gpu -c conda-forge python=3.10.0\n", - "conda activate tf-gpu\n", - "\n", - "export PYTHONNOUSERSITE=True\n", - "pip install numpy==1.26.4 tensorflow[and-cuda] conda-pack\n", - "\n", - "conda pack # tf-gpu.tar.gz\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "id": "772e337e-1098-4c7b-ba81-8cb221a518e2", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "import os\n", - "from pyspark.ml.functions import predict_batch_udf\n", - "from pyspark.sql.functions import col, struct\n", - "from pyspark.sql.types import ArrayType, FloatType" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "id": "69d0c93a-bb0b-46c5-9d28-7b08a2e70964", - "metadata": { - "tags": [ - "TRITON" - ] - }, - "outputs": [], - "source": [ - "%%bash\n", - "# copy custom model to expected layout for Triton\n", - "rm -rf models\n", - "mkdir -p models\n", - "cp -r models_config/text_classification models\n", - "\n", - "# add custom execution environment\n", - "cp tf-gpu.tar.gz models" + "First we'll cleanup the vocabulary layer of the model to remove non-ASCII characters. This ensures the inputs can be properly serialized and sent to Triton." ] }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 52, "id": "f4f14c8f", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ "import unicodedata\n", @@ -1454,299 +1640,401 @@ "normalized_vocab = normalize_vocabulary(vocab)\n", "\n", "# Reassign the cleaned vocabulary to the TextVectorization layer\n", - "vectorize_layer.set_vocabulary(normalized_vocab)\n", - "\n", - "# Save the model with the cleaned vocabulary\n", - "export_model.save('text_model_cleaned.keras')" + "vectorize_layer.set_vocabulary(normalized_vocab)" ] }, { - "cell_type": "markdown", - "id": "0d8c9ab3-57c4-45bb-9bcf-6433337ef9b5", + "cell_type": "code", + "execution_count": 53, + "id": "9614a192", "metadata": {}, + "outputs": [], "source": [ - "#### Start Triton Server on each executor" + "# Save the model with the cleaned vocabulary\n", + "triton_model_path = '{}/models/text_model_cleaned.keras'.format(os.getcwd())\n", + "export_model.save(triton_model_path)\n", + "\n", + "# For cloud environments, copy the model to the distributed file system.\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-models\")\n", + " dbfs_model_path = \"/dbfs/FileStore/spark-dl-models/text_model_cleaned.keras\"\n", + " shutil.copy(triton_model_path, dbfs_model_path)\n", + " triton_model_path = dbfs_model_path\n", + "elif on_dataproc:\n", + " # GCS is mounted at /mnt/gcs by the init script\n", + " models_dir = \"/mnt/gcs/spark-dl/models\"\n", + " os.mkdir(models_dir) if not os.path.exists(models_dir) else None\n", + " gcs_model_path = models_dir + \"/text_model_cleaned.keras\"\n", + " shutil.copy(triton_model_path, gcs_model_path)\n", + " triton_model_path = gcs_model_path" ] }, { "cell_type": "code", - "execution_count": 50, - "id": "a7fb146c-5319-4831-85f7-f2f3c084b042", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 54, + "id": "32d0142a", + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial" + ] + }, + { + "cell_type": "markdown", + "id": "edddffb9", + "metadata": {}, + "source": [ + "Import the utility functions from pytriton_utils.py:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "444bad3f", + "metadata": {}, + "outputs": [], + "source": [ + "sc.addPyFile(\"pytriton_utils.py\")\n", + "\n", + "from pytriton_utils import (\n", + " use_stage_level_scheduling,\n", + " find_ports,\n", + " start_triton,\n", + " stop_triton\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "f0923a56", + "metadata": {}, + "source": [ + "Define the Triton Server function:" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "id": "a4d37d33", + "metadata": {}, + "outputs": [], + "source": [ + "def triton_server(ports, model_path):\n", + " import time\n", + " import signal\n", + " import numpy as np\n", + " import tensorflow as tf\n", + " from pytriton.decorators import batch\n", + " from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n", + " from pytriton.triton import Triton, TritonConfig\n", + " from pyspark import TaskContext\n", + " from tensorflow.keras import layers \n", + "\n", + " \n", + " print(f\"SERVER: Initializing model on worker {TaskContext.get().partitionId()}.\")\n", + " # Enable GPU memory growth\n", + " gpus = tf.config.experimental.list_physical_devices('GPU')\n", + " if gpus:\n", + " try:\n", + " for gpu in gpus:\n", + " tf.config.experimental.set_memory_growth(gpu, True)\n", + " except RuntimeError as e:\n", + " print(e)\n", + "\n", + " def custom_standardization(input_data):\n", + " lowercase = tf.strings.lower(input_data)\n", + " stripped_html = tf.strings.regex_replace(lowercase, \"
\", \" \")\n", + " return tf.strings.regex_replace(\n", + " stripped_html, \"[%s]\" % re.escape(string.punctuation), \"\"\n", + " )\n", + "\n", + " max_features = 10000\n", + " sequence_length = 250\n", + "\n", + " vectorize_layer = layers.TextVectorization(\n", + " standardize=custom_standardization,\n", + " max_tokens=max_features,\n", + " output_mode=\"int\",\n", + " output_sequence_length=sequence_length,\n", + " )\n", + "\n", + " custom_objects = {\"vectorize_layer\": vectorize_layer,\n", + " \"custom_standardization\": custom_standardization}\n", + "\n", + " with tf.keras.utils.custom_object_scope(custom_objects):\n", + " model = tf.keras.models.load_model(model_path)\n", + "\n", + " @batch\n", + " def _infer_fn(**inputs):\n", + " sentences = inputs[\"text\"]\n", + " print(f\"SERVER: Received batch of size {len(sentences)}.\")\n", + " decoded_sentences = tf.convert_to_tensor(np.vectorize(lambda x: x.decode('utf-8'))(sentences))\n", + " return {\n", + " \"preds\": model.predict(decoded_sentences)\n", + " }\n", + " \n", + " workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n", + " triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n", + " with Triton(config=triton_conf, workspace=workspace_path) as triton:\n", + " triton.bind(\n", + " model_name=\"TextModel\",\n", + " infer_func=_infer_fn,\n", + " inputs=[\n", + " Tensor(name=\"text\", dtype=np.bytes_, shape=(-1,)),\n", + " ],\n", + " outputs=[\n", + " Tensor(name=\"preds\", dtype=np.float32, shape=(-1,)),\n", + " ],\n", + " config=ModelConfig(\n", + " max_batch_size=128,\n", + " batcher=DynamicBatcher(max_queue_delay_microseconds=5000), # 5ms\n", + " ),\n", + " strict=True,\n", + " )\n", + "\n", + " def _stop_triton(signum, frame):\n", + " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", + " triton.stop()\n", + "\n", + " signal.signal(signal.SIGTERM, _stop_triton)\n", + "\n", + " print(\"SERVER: Serving inference\")\n", + " triton.serve()" + ] + }, + { + "cell_type": "markdown", + "id": "d340e231", + "metadata": {}, + "source": [ + "#### Start Triton servers" + ] + }, + { + "cell_type": "markdown", + "id": "bad219c9", + "metadata": {}, + "source": [ + "**Specify the number of nodes in the cluster.** \n", + "Following the README, the example standalone cluster uses 1 node. The example Databricks/Dataproc cluster scripts use 4 nodes by default. " + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "id": "a01c6198", + "metadata": {}, + "outputs": [], + "source": [ + "# Change based on cluster setup\n", + "num_nodes = 1 if on_standalone else 4" + ] + }, + { + "cell_type": "markdown", + "id": "e4eaf6da", + "metadata": {}, + "source": [ + "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "id": "4d5dc419", + "metadata": {}, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - " \r" + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" ] - }, - { - "data": { - "text/plain": [ - "[True]" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ - "num_executors = 1\n", - "triton_models_dir = \"{}/models\".format(os.getcwd())\n", - "text_model_dir = \"{}/text_model_cleaned.keras\".format(os.getcwd())\n", - "nodeRDD = sc.parallelize(list(range(num_executors)), num_executors)\n", - "\n", - "def start_triton(it):\n", - " import docker\n", - " import time\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " if containers:\n", - " print(\">>>> containers: {}\".format([c.short_id for c in containers]))\n", - " else:\n", - " container=client.containers.run(\n", - " \"nvcr.io/nvidia/tritonserver:24.08-py3\", \"tritonserver --model-repository=/models\",\n", - " detach=True,\n", - " device_requests=[docker.types.DeviceRequest(device_ids=[\"0\"], capabilities=[['gpu']])],\n", - " name=\"spark-triton\",\n", - " network_mode=\"host\",\n", - " remove=True,\n", - " shm_size=\"128M\",\n", - " volumes={\n", - " triton_models_dir: {\"bind\": \"/models\", \"mode\": \"ro\"},\n", - " text_model_dir: {\"bind\": \"/text_model_cleaned.keras\", \"mode\": \"ro\"}\n", - " }\n", - " )\n", - " print(\">>>> starting triton: {}\".format(container.short_id))\n", - "\n", - " # wait for triton to be running\n", - " time.sleep(15)\n", - " client = grpcclient.InferenceServerClient(\"localhost:8001\")\n", - " ready = False\n", - " while not ready:\n", - " try:\n", - " ready = client.is_server_ready()\n", - " except Exception as e:\n", - " time.sleep(5)\n", - " \n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(start_triton).collect()" + "sc = spark.sparkContext\n", + "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" ] }, { "cell_type": "markdown", - "id": "287873da-6202-4b55-97fb-cda8644b1fee", + "id": "0bdba73f", "metadata": {}, "source": [ - "#### Run inference" + "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." ] }, { "cell_type": "code", - "execution_count": 51, - "id": "41106a02-236e-4cb3-ac51-76aa64b663c2", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 60, + "id": "7fa58218", + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "+----------------------------------------------------------------------------------------------------+\n", - "| lines|\n", - "+----------------------------------------------------------------------------------------------------+\n", - "|i do not understand at all why this movie received such good grades from critics - - i've seen te...|\n", - "| I am a big fan of The ABC Movies of the Week genre|\n", - "| Strangeland is a terrible horror/technological thriller|\n", - "| Sex,Drugs,Rock & Roll is without a doubt the worst product of Western Civilization|\n", - "| Not to be mistaken as the highly touted Samuel L|\n", - "|Following the pleasingly atmospheric original and the amusingly silly second one, this incredibly...|\n", - "| No idea how this is rated as high as it is (5|\n", - "|When I saw this in the cinema, I remember wincing at the bad acting about a minute or two into th...|\n", - "| I was shocked at how bad it was and unable to turn away from the disaster|\n", - "|I don't know if this exceptionally dull movie was intended as an unofficial sequel to 'The French...|\n", - "|Greetings All,

Isn't it amazing the power that films have on you after the 1st viewing...|\n", - "| I'm sorry but this guy is not funny|\n", - "|This movie is so dull I spent half of it on IMDb while it was open in another tab on Netflix tryi...|\n", - "| OK, lets start with the best|\n", - "|This show had a promising start as sort of the opposite of 'Oceans 11' but has developed into a s...|\n", - "|The 3rd in the series finds Paul Kersey (Bronson) turning vigilante to get revenge on the thugs t...|\n", - "| I'm not sure I've ever seen a film as bad as this|\n", - "| Steven Seagal has made a really dull, bad and boring movie|\n", - "| You have to acknowledge Cimino's contribution to cinema|\n", - "|***SPOILER ALERT*** Disjointed and confusing arson drama that has to do with a sinister plan to b...|\n", - "+----------------------------------------------------------------------------------------------------+\n", - "only showing top 20 rows\n", - "\n" + "Using ports [7000, 7001, 7002]\n" ] } ], "source": [ - "from datasets import load_dataset\n", - "\n", - "# load IMDB reviews (test) dataset\n", - "data = load_dataset(\"imdb\", split=\"test\")\n", - "lines = []\n", - "for example in data:\n", - " lines.append([example[\"text\"].split(\".\")[0]])\n", - "\n", - "df = spark.createDataFrame(lines, ['lines']).repartition(10)\n", - "df.show(truncate=100)" + "model_name = \"TextModel\"\n", + "ports = find_ports()\n", + "assert len(ports) == 3\n", + "print(f\"Using ports {ports}\")" ] }, { "cell_type": "code", - "execution_count": 52, - "id": "8b763167-7f50-4278-9bc9-6c3433b62294", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 61, + "id": "bdcf9187", + "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "['lines']" - ] - }, - "execution_count": 52, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 19:> (0 + 1) / 1]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Triton Server PIDs:\n", + " {\n", + " \"cb4ae00-lcedt\": 2897388\n", + "}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] } ], "source": [ - "columns = df.columns\n", - "columns" + "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", + " ports=ports,\n", + " model_name=model_name,\n", + " model_path=triton_model_path)).collectAsMap()\n", + "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + ] + }, + { + "cell_type": "markdown", + "id": "e1477f4b", + "metadata": {}, + "source": [ + "#### Define client function" ] }, { "cell_type": "code", - "execution_count": 53, - "id": "29b0cc0d-c480-4e4a-bd41-207dc314cba5", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "execution_count": 62, + "id": "d590cd25", + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{ports[0]}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "0ad47438", + "metadata": {}, "outputs": [], "source": [ - "def triton_fn(triton_uri, model_name):\n", + "def triton_fn(url, model_name):\n", " import numpy as np\n", - " import tritonclient.grpc as grpcclient\n", - " \n", - " np_types = {\n", - " \"BOOL\": np.dtype(np.bool_),\n", - " \"INT8\": np.dtype(np.int8),\n", - " \"INT16\": np.dtype(np.int16),\n", - " \"INT32\": np.dtype(np.int32),\n", - " \"INT64\": np.dtype(np.int64),\n", - " \"FP16\": np.dtype(np.float16),\n", - " \"FP32\": np.dtype(np.float32),\n", - " \"FP64\": np.dtype(np.float64),\n", - " \"FP64\": np.dtype(np.double),\n", - " \"BYTES\": np.dtype(object)\n", - " }\n", - "\n", - " client = grpcclient.InferenceServerClient(triton_uri)\n", - " model_meta = client.get_model_metadata(model_name)\n", - " \n", - " def predict(inputs):\n", - " if isinstance(inputs, np.ndarray):\n", - " # single ndarray input\n", - " request = [grpcclient.InferInput(model_meta.inputs[0].name, inputs.shape, model_meta.inputs[0].datatype)]\n", - " request[0].set_data_from_numpy(inputs.astype(np_types[model_meta.inputs[0].datatype]))\n", - " else:\n", - " # dict of multiple ndarray inputs\n", - " request = [grpcclient.InferInput(i.name, inputs[i.name].shape, i.datatype) for i in model_meta.inputs]\n", - " for i in request:\n", - " i.set_data_from_numpy(inputs[i.name()].astype(np_types[i.datatype()]))\n", - " \n", - " response = client.infer(model_name, inputs=request)\n", - " \n", - " if len(model_meta.outputs) > 1:\n", - " # return dictionary of numpy arrays\n", - " return {o.name: response.as_numpy(o.name) for o in model_meta.outputs}\n", - " else:\n", - " # return single numpy array\n", - " return response.as_numpy(model_meta.outputs[0].name)\n", - " \n", - " return predict" + " from pytriton.client import ModelClient\n", + "\n", + " print(f\"CLIENT: Connecting to {model_name} at {url}\")\n", + "\n", + " def infer_batch(inputs):\n", + " with ModelClient(url, model_name, inference_timeout_s=240) as client:\n", + " encoded_inputs = np.vectorize(lambda x: x.encode(\"utf-8\"))(inputs).astype(np.bytes_)\n", + " encoded_inputs = np.expand_dims(encoded_inputs, axis=1)\n", + " result_data = client.infer_batch(encoded_inputs)\n", + " \n", + " return result_data[\"preds\"]\n", + " \n", + " return infer_batch" + ] + }, + { + "cell_type": "markdown", + "id": "91974885", + "metadata": {}, + "source": [ + "#### Load and preprocess DataFrame" ] }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 64, + "id": "41106a02-236e-4cb3-ac51-76aa64b663c2", + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.parquet(data_path).limit(512).repartition(8)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e851870b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/01/06 22:02:44 WARN CacheManager: Asked to cache already cached data.\n" + ] + } + ], + "source": [ + "input_df = df.select(preprocess(col(\"text\")).alias(\"lines\")).cache()" + ] + }, + { + "cell_type": "code", + "execution_count": 66, "id": "8e06d33f-5cef-4a48-afc3-5d468f8ec2b4", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [], "source": [ - "from functools import partial\n", - "\n", - "classify = predict_batch_udf(partial(triton_fn, triton_uri=\"localhost:8001\", model_name=\"text_classification\"),\n", - " input_tensor_shapes=[[1]],\n", + "classify = predict_batch_udf(partial(triton_fn, url=url, model_name=\"TextModel\"),\n", " return_type=FloatType(),\n", - " batch_size=2048)" + " batch_size=64)" ] }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 67, "id": "d89e74ad-e551-4bfa-ad08-98725878630a", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 23:> (0 + 8) / 8]\r" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "+--------------------------------------------------------------------------------+----------+\n", - "| lines| preds|\n", - "+--------------------------------------------------------------------------------+----------+\n", - "|i do not understand at all why this movie received such good grades from crit...| 0.5380144|\n", - "| I am a big fan of The ABC Movies of the Week genre|0.59806347|\n", - "| Strangeland is a terrible horror/technological thriller|0.54900867|\n", - "|Sex,Drugs,Rock & Roll is without a doubt the worst product of Western Civiliz...|0.56048334|\n", - "| Not to be mistaken as the highly touted Samuel L|0.56276447|\n", - "|Following the pleasingly atmospheric original and the amusingly silly second ...| 0.5571853|\n", - "| No idea how this is rated as high as it is (5| 0.5637812|\n", - "|When I saw this in the cinema, I remember wincing at the bad acting about a m...|0.66255826|\n", - "| I was shocked at how bad it was and unable to turn away from the disaster| 0.5871666|\n", - "|I don't know if this exceptionally dull movie was intended as an unofficial s...| 0.5578672|\n", - "|Greetings All,

Isn't it amazing the power that films have on you a...|0.56385136|\n", - "| I'm sorry but this guy is not funny| 0.5634932|\n", - "|This movie is so dull I spent half of it on IMDb while it was open in another...|0.58991694|\n", - "| OK, lets start with the best| 0.5795415|\n", - "|This show had a promising start as sort of the opposite of 'Oceans 11' but ha...|0.57494473|\n", - "|The 3rd in the series finds Paul Kersey (Bronson) turning vigilante to get re...| 0.6133918|\n", - "| I'm not sure I've ever seen a film as bad as this| 0.5336116|\n", - "| Steven Seagal has made a really dull, bad and boring movie|0.55780387|\n", - "| You have to acknowledge Cimino's contribution to cinema| 0.5763774|\n", - "|***SPOILER ALERT*** Disjointed and confusing arson drama that has to do with ...|0.56471467|\n", - "+--------------------------------------------------------------------------------+----------+\n", - "only showing top 20 rows\n", - "\n", - "CPU times: user 2.49 ms, sys: 1.47 ms, total: 3.96 ms\n", - "Wall time: 916 ms\n" + "CPU times: user 10.4 ms, sys: 7 ms, total: 17.4 ms\n", + "Wall time: 2.53 s\n" ] }, { @@ -1759,18 +2047,57 @@ ], "source": [ "%%time\n", - "df.withColumn(\"preds\", classify(struct(*columns))).show(truncate=80)" + "predictions = input_df.withColumn(\"preds\", classify(struct(\"lines\")))\n", + "results = predictions.collect()" ] }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 68, "id": "b4fa7fc9-341c-49a6-9af2-e316f2355d67", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 2.59 ms, sys: 631 μs, total: 3.22 ms\n", + "Wall time: 214 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "predictions = input_df.withColumn(\"preds\", classify(\"lines\"))\n", + "results = predictions.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "id": "564f999b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 288 μs, sys: 3.66 ms, total: 3.95 ms\n", + "Wall time: 245 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "predictions = input_df.withColumn(\"preds\", classify(col(\"lines\")))\n", + "results = predictions.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "id": "9222e8a9", + "metadata": {}, "outputs": [ { "name": "stdout", @@ -1779,37 +2106,34 @@ "+--------------------------------------------------------------------------------+----------+\n", "| lines| preds|\n", "+--------------------------------------------------------------------------------+----------+\n", - "|i do not understand at all why this movie received such good grades from crit...| 0.5380144|\n", - "| I am a big fan of The ABC Movies of the Week genre|0.59806347|\n", - "| Strangeland is a terrible horror/technological thriller|0.54900867|\n", - "|Sex,Drugs,Rock & Roll is without a doubt the worst product of Western Civiliz...|0.56048334|\n", - "| Not to be mistaken as the highly touted Samuel L|0.56276447|\n", - "|Following the pleasingly atmospheric original and the amusingly silly second ...| 0.5571853|\n", - "| No idea how this is rated as high as it is (5| 0.5637812|\n", - "|When I saw this in the cinema, I remember wincing at the bad acting about a m...|0.66255826|\n", - "| I was shocked at how bad it was and unable to turn away from the disaster| 0.5871666|\n", - "|I don't know if this exceptionally dull movie was intended as an unofficial s...| 0.5578672|\n", - "|Greetings All,

Isn't it amazing the power that films have on you a...|0.56385136|\n", - "| I'm sorry but this guy is not funny| 0.5634932|\n", - "|This movie is so dull I spent half of it on IMDb while it was open in another...|0.58991694|\n", - "| OK, lets start with the best| 0.5795415|\n", - "|This show had a promising start as sort of the opposite of 'Oceans 11' but ha...|0.57494473|\n", - "|The 3rd in the series finds Paul Kersey (Bronson) turning vigilante to get re...| 0.6133918|\n", - "| I'm not sure I've ever seen a film as bad as this| 0.5336116|\n", - "| Steven Seagal has made a really dull, bad and boring movie|0.55780387|\n", - "| You have to acknowledge Cimino's contribution to cinema| 0.5763774|\n", - "|***SPOILER ALERT*** Disjointed and confusing arson drama that has to do with ...|0.56471467|\n", + "|The only reason I'm even giving this movie a 4 is because it was made in to a...| 0.5441438|\n", + "|Awkward disaster mishmash has a team of scavengers coming across the overturn...|0.58016133|\n", + "|Here is a fantastic concept for a film - a series of meteors crash into a sma...|0.55131954|\n", + "| I walked out of the cinema having suffered this film after 30 mins| 0.542057|\n", + "|A wildly uneven film where the major problem is the uneasy mix of comedy and ...| 0.5196002|\n", + "|Leonard Rossiter and Frances de la Tour carry this film, not without a strugg...|0.53112733|\n", + "| A good cast| 0.5486873|\n", + "|Yet again, I appear to be the only person on planet Earth who is capable of c...| 0.5343111|\n", + "|As a serious horror fan, I get that certain marketing ploys are used to sell ...| 0.5497148|\n", + "|Upon writing this review I have difficulty trying to think of what to write a...| 0.5581456|\n", + "| Simply awful| 0.5701754|\n", + "|I am a fan of Ed Harris' work and I really had high expectations about this film| 0.5510578|\n", + "| Well|0.55721515|\n", + "| This is a new approach to comedy|0.56038314|\n", + "| It's been mentioned by others the inane dialogue in this series and I agree| 0.5451202|\n", + "|One of the most boring movies I've ever had to sit through, it's completely f...|0.56161135|\n", + "|This movie was playing on Lifetime Movie Network last month and I decided to ...| 0.5555233|\n", + "| 1983's \"Frightmare\" is an odd little film| 0.5363368|\n", + "| 'Felony' is a B-movie|0.55682427|\n", + "| This movie defines the word \"confused\"| 0.5630136|\n", "+--------------------------------------------------------------------------------+----------+\n", "only showing top 20 rows\n", - "\n", - "CPU times: user 571 μs, sys: 2.22 ms, total: 2.79 ms\n", - "Wall time: 528 ms\n" + "\n" ] } ], "source": [ - "%%time\n", - "df.withColumn(\"preds\", classify(*columns)).show(truncate=80)" + "predictions.show(truncate=80)" ] }, { @@ -1824,14 +2148,17 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 71, "id": "a71ac9b6-47a2-4306-bc40-9ce7b4e968ec", - "metadata": { - "tags": [ - "TRITON" - ] - }, + "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + }, { "name": "stderr", "output_type": "stream", @@ -1845,36 +2172,26 @@ "[True]" ] }, - "execution_count": 57, + "execution_count": 71, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "def stop_triton(it):\n", - " import docker\n", - " import time\n", - " \n", - " client=docker.from_env()\n", - " containers=client.containers.list(filters={\"name\": \"spark-triton\"})\n", - " print(\">>>> stopping containers: {}\".format([c.short_id for c in containers]))\n", - " if containers:\n", - " container=containers[0]\n", - " container.stop(timeout=120)\n", - "\n", - " return [True]\n", - "\n", - "nodeRDD.barrier().mapPartitions(stop_triton).collect()" + "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", + "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", + "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" ] }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 73, "id": "54a90574-7cbb-487b-b7a8-dcda0e6e301f", "metadata": {}, "outputs": [], "source": [ - "spark.stop()" + "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n", + " spark.stop()" ] }, { From 7561c5f060e0a4a0fa5e9b1198e0c97645573c89 Mon Sep 17 00:00:00 2001 From: "Rishi C." <77904151+rishic3@users.noreply.github.com> Date: Thu, 6 Feb 2025 12:39:00 -0800 Subject: [PATCH 06/12] Fix torch-tensorrt install, libstdc++ troubleshooting (#487) Pin torch<=2.5.1 until torch-tensorrt has a compatible version for torch 2.6. Add troubleshooting to update libstdc++ for Triton server binary requirements; can occur if default conda channel is used and not up to date. Removed ipykernel from requirements since it's already pulled in by jupyterlab (and slows pip down a lot). --------- Signed-off-by: Rishi Chandra --- examples/ML+DL-Examples/Spark-DL/dl_inference/README.md | 8 ++++++-- .../dl_inference/databricks/setup/init_spark_dl.sh | 2 +- .../Spark-DL/dl_inference/dataproc/setup/start_cluster.sh | 2 +- .../ML+DL-Examples/Spark-DL/dl_inference/requirements.txt | 1 - .../Spark-DL/dl_inference/torch_requirements.txt | 4 ++-- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md b/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md index 034ee2af..b396b662 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md @@ -63,13 +63,13 @@ Each notebook has a suffix `_torch` or `_tf` specifying the environment used. **For PyTorch:** ``` -conda create -n spark-dl-torch python=3.11 +conda create -n spark-dl-torch -c conda-forge python=3.11 conda activate spark-dl-torch pip install -r torch_requirements.txt ``` **For TensorFlow:** ``` -conda create -n spark-dl-tf python=3.11 +conda create -n spark-dl-tf -c conda-forge python=3.11 conda activate spark-dl-tf pip install -r tf_requirements.txt ``` @@ -105,6 +105,10 @@ If you encounter issues starting the Triton server, you may need to link your li ```shell ln -sf /usr/lib/x86_64-linux-gnu/libstdc++.so.6 ${CONDA_PREFIX}/lib/libstdc++.so.6 ``` +If the issue persists with the message `libstdc++.so.6: version 'GLIBCXX_3.4.30' not found`, you may need to update libstdc++ in your conda environment: +```shell +conda install -c conda-forge libstdcxx-ng +``` ## Running on Cloud Environments diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/init_spark_dl.sh b/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/init_spark_dl.sh index 39673bf6..9515f435 100755 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/init_spark_dl.sh +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/init_spark_dl.sh @@ -12,7 +12,7 @@ datasets==3.* transformers urllib3<2 nvidia-pytriton -torch +torch<=2.5.1 torchvision --extra-index-url https://download.pytorch.org/whl/cu121 torch-tensorrt tensorrt --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/start_cluster.sh b/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/start_cluster.sh index 4e9a7791..b44df43b 100755 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/start_cluster.sh +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/start_cluster.sh @@ -49,7 +49,7 @@ urllib3<2 nvidia-pytriton" TORCH_REQUIREMENTS="${COMMON_REQUIREMENTS} -torch +torch<=2.5.1 torchvision --extra-index-url https://download.pytorch.org/whl/cu121 torch-tensorrt tensorrt --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/requirements.txt b/examples/ML+DL-Examples/Spark-DL/dl_inference/requirements.txt index 2520fb8e..a0afb217 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/requirements.txt +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/requirements.txt @@ -26,6 +26,5 @@ huggingface datasets transformers ipywidgets -ipykernel urllib3<2 nvidia-pytriton diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/torch_requirements.txt b/examples/ML+DL-Examples/Spark-DL/dl_inference/torch_requirements.txt index 5f3b9017..708d1387 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/torch_requirements.txt +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/torch_requirements.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # limitations under the License. -r requirements.txt -torch +torch<=2.5.1 torchvision torch-tensorrt tensorrt --extra-index-url https://download.pytorch.org/whl/cu121 From 59b4920a68188aab0b1002ed4c2dd8c8e77e7db7 Mon Sep 17 00:00:00 2001 From: "Rishi C." <77904151+rishic3@users.noreply.github.com> Date: Thu, 6 Feb 2025 13:39:30 -0800 Subject: [PATCH 07/12] Fix broken predict_batch_udf links (#489) Signed-off-by: Rishi Chandra --- examples/ML+DL-Examples/Spark-DL/dl_inference/README.md | 2 +- .../dl_inference/huggingface/conditional_generation_tf.ipynb | 2 +- .../dl_inference/huggingface/conditional_generation_torch.ipynb | 2 +- .../Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb | 2 +- .../Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb | 2 +- .../dl_inference/huggingface/sentence_transformers_torch.ipynb | 2 +- .../dl_inference/pytorch/housing_regression_torch.ipynb | 2 +- .../dl_inference/pytorch/image_classification_torch.ipynb | 2 +- .../dl_inference/tensorflow/image_classification_tf.ipynb | 2 +- .../dl_inference/tensorflow/keras_preprocessing_tf.ipynb | 2 +- .../Spark-DL/dl_inference/tensorflow/keras_resnet50_tf.ipynb | 2 +- .../dl_inference/tensorflow/text_classification_tf.ipynb | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md b/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md index b396b662..613536f9 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md @@ -1,6 +1,6 @@ # Deep Learning Inference on Spark -Example notebooks demonstrating **distributed deep learning inference** using the [predict_batch_udf](https://developer.nvidia.com/blog/distributed-deep-learning-made-easy-with-spark-3-4/) introduced in Spark 3.4.0. +Example notebooks demonstrating **distributed deep learning inference** using the [predict_batch_udf](https://developer.nvidia.com/blog/distributed-deep-learning-made-easy-with-spark-3-4/#distributed_inference) introduced in Spark 3.4.0. These notebooks also demonstrate integration with [Triton Inference Server](https://docs.nvidia.com/deeplearning/triton-inference-server/user-guide/docs/index.html), an open-source, GPU-accelerated serving solution for DL. ## Contents: diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_tf.ipynb index cdf6dc49..24f9927b 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_tf.ipynb @@ -561,7 +561,7 @@ "source": [ "## Inference using Spark DL API\n", "\n", - "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", "\n", "- predict_batch_fn uses Tensorflow APIs to load the model and return a predict function which operates on numpy arrays \n", "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_torch.ipynb index fe91335e..924c4d41 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_torch.ipynb @@ -622,7 +622,7 @@ "source": [ "## Inference using Spark DL API\n", "\n", - "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", "\n", "- predict_batch_fn uses PyTorch APIs to load the model and return a predict function which operates on numpy arrays \n", "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb index 749ba9d3..58f9bcd7 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb @@ -564,7 +564,7 @@ "source": [ "## Inference using Spark DL API\n", "\n", - "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", "\n", "- predict_batch_fn uses Tensorflow APIs to load the model and return a predict function which operates on numpy arrays \n", "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb index 0a4c8340..d847d26a 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb @@ -461,7 +461,7 @@ "source": [ "## Inference using Spark DL API\n", "\n", - "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", "\n", "- predict_batch_fn uses PyTorch APIs to load the model and return a predict function which operates on numpy arrays \n", "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/sentence_transformers_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/sentence_transformers_torch.ipynb index 46c961de..83d4ecba 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/sentence_transformers_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/sentence_transformers_torch.ipynb @@ -395,7 +395,7 @@ "source": [ "## Inference using Spark DL API\n", "\n", - "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", "\n", "- predict_batch_fn uses PyTorch APIs to load the model and return a predict function which operates on numpy arrays \n", "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/housing_regression_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/housing_regression_torch.ipynb index eb558bb2..99bb06fa 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/housing_regression_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/housing_regression_torch.ipynb @@ -1151,7 +1151,7 @@ "source": [ "## Inference using Spark DL API\n", "\n", - "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", "\n", "- predict_batch_fn uses PyTorch APIs to load the model and return a predict function which operates on numpy arrays \n", "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/image_classification_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/image_classification_torch.ipynb index e22ad389..4d05305b 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/image_classification_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/image_classification_torch.ipynb @@ -1743,7 +1743,7 @@ "source": [ "## Inference using Spark DL API\n", "\n", - "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", "\n", "- predict_batch_fn uses PyTorch APIs to load the model and return a predict function which operates on numpy arrays \n", "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/image_classification_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/image_classification_tf.ipynb index 379a4d91..aec9bc8d 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/image_classification_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/image_classification_tf.ipynb @@ -1000,7 +1000,7 @@ "source": [ "## Inference using Spark DL API\n", "\n", - "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", "\n", "- predict_batch_fn uses Tensorflow APIs to load the model and return a predict function which operates on numpy arrays \n", "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_preprocessing_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_preprocessing_tf.ipynb index a5edd2c6..f2d54e9f 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_preprocessing_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_preprocessing_tf.ipynb @@ -1160,7 +1160,7 @@ "source": [ "## Inference using Spark DL API\n", "\n", - "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", "\n", "- predict_batch_fn uses Tensorflow APIs to load the model and return a predict function which operates on numpy arrays \n", "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_resnet50_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_resnet50_tf.ipynb index 1a9b82e8..c90a835b 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_resnet50_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_resnet50_tf.ipynb @@ -613,7 +613,7 @@ "source": [ "## Inference using Spark DL API\n", "\n", - "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", "\n", "- predict_batch_fn uses Tensorflow APIs to load the model and return a predict function which operates on numpy arrays \n", "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/text_classification_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/text_classification_tf.ipynb index 29230de5..bfa5affb 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/text_classification_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/text_classification_tf.ipynb @@ -1374,7 +1374,7 @@ "source": [ "## Inference using Spark DL API\n", "\n", - "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/3.4.0/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", + "Distributed inference using the PySpark [predict_batch_udf](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.functions.predict_batch_udf.html#pyspark.ml.functions.predict_batch_udf):\n", "\n", "- predict_batch_fn uses Tensorflow APIs to load the model and return a predict function which operates on numpy arrays \n", "- predict_batch_udf will convert the Spark DataFrame columns into numpy input batches for the predict function" From e5224045b7380d06e649a7781f08a024beb941a1 Mon Sep 17 00:00:00 2001 From: "Rishi C." <77904151+rishic3@users.noreply.github.com> Date: Fri, 7 Feb 2025 15:47:34 -0800 Subject: [PATCH 08/12] Implement and use TritonServerManager class (#488) Consolidate util functions into a server manager class to simplify usage. (Note that notebooks were rerun but only the Triton utility invocations are changed). Also on Dataproc, the utils file needs to be copied driver root dir instead of the same directory as the notebooks, since sc.addPyFile on Dataproc only accepts absolute paths from root. --------- Signed-off-by: Rishi Chandra --- .../Spark-DL/dl_inference/README.md | 5 + .../dataproc/setup/init_spark_dl.sh | 2 +- .../conditional_generation_tf.ipynb | 289 ++++---- .../conditional_generation_torch.ipynb | 289 +++----- .../huggingface/pipelines_tf.ipynb | 284 ++++---- .../huggingface/pipelines_torch.ipynb | 267 ++++--- .../sentence_transformers_torch.ipynb | 243 +++---- .../pytorch/housing_regression_torch.ipynb | 652 +++++++----------- .../pytorch/image_classification_torch.ipynb | 583 ++++++---------- .../Spark-DL/dl_inference/pytriton_utils.py | 319 ++++++--- .../tensorflow/image_classification_tf.ipynb | 541 +++++++-------- .../tensorflow/keras_preprocessing_tf.ipynb | 458 ++++++------ .../tensorflow/keras_resnet50_tf.ipynb | 368 +++++----- .../tensorflow/text_classification_tf.ipynb | 438 ++++++------ 14 files changed, 2155 insertions(+), 2583 deletions(-) diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md b/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md index 613536f9..8662ea3d 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md @@ -99,6 +99,11 @@ The notebooks are ready to run! Each notebook has a cell to connect to the stand - `requirements.txt` installs pyspark>=3.4.0. Make sure the installed PySpark version is compatible with your system's Spark installation. - The notebooks require a GPU environment for the executors. - The PyTorch notebooks include model compilation and accelerated inference with TensorRT. While not included in the notebooks, Tensorflow also supports [integration with TensorRT](https://docs.nvidia.com/deeplearning/frameworks/tf-trt-user-guide/index.html), but as of writing it is not supported in TF==2.17.0. +- Note that some Huggingface models may be gated and will require a login, e.g.,: + ```python + from huggingface_hub import login + login() + ``` **Troubleshooting:** If you encounter issues starting the Triton server, you may need to link your libstdc++ file to the conda environment, e.g.: diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/init_spark_dl.sh b/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/init_spark_dl.sh index 264e3d5b..dca04d97 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/init_spark_dl.sh +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/init_spark_dl.sh @@ -49,7 +49,7 @@ if [[ "${ROLE}" == 'Master' ]]; then if gsutil -q stat gs://${SPARK_DL_HOME}/notebooks/**; then mkdir spark-dl-notebooks gcloud storage cp -r gs://${SPARK_DL_HOME}/notebooks/* spark-dl-notebooks - gcloud storage cp gs://${SPARK_DL_HOME}/pytriton_utils.py spark-dl-notebooks/ + gcloud storage cp gs://${SPARK_DL_HOME}/pytriton_utils.py . else echo "Failed to retrieve notebooks from gs://${SPARK_DL_HOME}/notebooks/" exit 1 diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_tf.ipynb index 24f9927b..7a8316c2 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_tf.ipynb @@ -32,13 +32,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-01-27 12:01:14.829270: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2025-01-27 12:01:14.836118: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2025-01-27 12:01:14.843723: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2025-01-27 12:01:14.845951: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2025-01-27 12:01:14.851831: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "2025-02-04 13:53:50.831324: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2025-02-04 13:53:50.838528: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2025-02-04 13:53:50.846226: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2025-02-04 13:53:50.848585: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2025-02-04 13:53:50.854859: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2025-01-27 12:01:15.226235: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" + "2025-02-04 13:53:51.229622: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" ] } ], @@ -69,9 +69,9 @@ "output_type": "stream", "text": [ "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", - "I0000 00:00:1738008075.848429 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008075.871583 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008075.875015 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" + "I0000 00:00:1738706031.770264 3625306 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706031.793270 3625306 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706031.796251 3625306 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" ] } ], @@ -100,13 +100,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "I0000 00:00:1738008076.235518 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008076.238670 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008076.241422 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008076.340179 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008076.341199 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008076.342108 2973032 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "2025-01-27 12:01:16.343039: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 43844 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n", + "I0000 00:00:1738706032.132191 3625306 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706032.134996 3625306 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706032.137528 3625306 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706032.251302 3625306 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706032.252345 3625306 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706032.253281 3625306 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2025-02-04 13:53:52.254192: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 43462 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n", "All PyTorch model weights were used when initializing TFT5ForConditionalGeneration.\n", "\n", "All the weights of TFT5ForConditionalGeneration were initialized from the PyTorch model.\n", @@ -140,11 +140,11 @@ "output_type": "stream", "text": [ "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", - "I0000 00:00:1738008077.718863 2973172 service.cc:146] XLA service 0x7370c0003170 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n", - "I0000 00:00:1738008077.718882 2973172 service.cc:154] StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n", - "2025-01-27 12:01:17.726803: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", - "2025-01-27 12:01:17.754699: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8907\n", - "I0000 00:00:1738008077.795877 2973172 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n" + "I0000 00:00:1738706033.555987 3625654 service.cc:146] XLA service 0x712d300025f0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n", + "I0000 00:00:1738706033.556005 3625654 service.cc:154] StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n", + "2025-02-04 13:53:53.558887: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", + "2025-02-04 13:53:53.569767: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8907\n", + "I0000 00:00:1738706033.604327 3625654 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n" ] } ], @@ -255,12 +255,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 20:01:19 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "25/01/27 20:01:19 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "25/02/04 13:53:54 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/02/04 13:53:54 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "25/01/27 20:01:19 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", - "25/01/27 20:01:19 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + "25/02/04 13:53:55 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" ] } ], @@ -384,7 +383,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 20:01:27 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + "25/02/04 13:54:01 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] }, { @@ -420,7 +419,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 20:01:27 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + "25/02/04 13:54:02 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] } ], @@ -547,6 +546,13 @@ "only showing top 20 rows\n", "\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] } ], "source": [ @@ -629,14 +635,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 24:===========================================> (6 + 2) / 8]\r" + "[Stage 24:====================================> (5 + 3) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 10.9 ms, sys: 10.1 ms, total: 21 ms\n", + "CPU times: user 9.07 ms, sys: 8.83 ms, total: 17.9 ms\n", "Wall time: 19.3 s\n" ] }, @@ -672,8 +678,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 7.89 ms, sys: 4.57 ms, total: 12.5 ms\n", - "Wall time: 12 s\n" + "CPU times: user 7.51 ms, sys: 4.96 ms, total: 12.5 ms\n", + "Wall time: 12.4 s\n" ] }, { @@ -700,15 +706,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 30:==================================================> (7 + 1) / 8]\r" + "[Stage 30:=====================> (3 + 5) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 6.32 ms, sys: 5.64 ms, total: 12 ms\n", - "Wall time: 12.2 s\n" + "CPU times: user 5.46 ms, sys: 5.98 ms, total: 11.4 ms\n", + "Wall time: 11.4 s\n" ] }, { @@ -809,8 +815,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 4.68 ms, sys: 8.71 ms, total: 13.4 ms\n", - "Wall time: 16.1 s\n" + "CPU times: user 9.1 ms, sys: 4.04 ms, total: 13.1 ms\n", + "Wall time: 14.9 s\n" ] }, { @@ -838,15 +844,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 39:==============> (2 + 6) / 8]\r" + "[Stage 39:==================================================> (7 + 1) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 5.51 ms, sys: 5.43 ms, total: 10.9 ms\n", - "Wall time: 11.6 s\n" + "CPU times: user 6.62 ms, sys: 5.23 ms, total: 11.9 ms\n", + "Wall time: 11.9 s\n" ] }, { @@ -880,8 +886,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 8.25 ms, sys: 3.17 ms, total: 11.4 ms\n", - "Wall time: 11.6 s\n" + "CPU times: user 8.67 ms, sys: 3.27 ms, total: 11.9 ms\n", + "Wall time: 11.7 s\n" ] }, { @@ -988,7 +994,7 @@ "id": "2964ffee", "metadata": {}, "source": [ - "Import the utility functions from pytriton_utils.py:" + "Import the helper class from pytriton_utils.py:" ] }, { @@ -1000,12 +1006,7 @@ "source": [ "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "from pytriton_utils import (\n", - " use_stage_level_scheduling,\n", - " find_ports,\n", - " start_triton,\n", - " stop_triton\n", - ")" + "from pytriton_utils import TritonServerManager" ] }, { @@ -1084,6 +1085,7 @@ " )\n", "\n", " def _stop_triton(signum, frame):\n", + " # The server manager sends SIGTERM to stop the server; this function ensures graceful cleanup.\n", " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", " triton.stop()\n", "\n", @@ -1123,82 +1125,38 @@ }, { "cell_type": "markdown", - "id": "52f7e397", + "id": "4142ebfc", "metadata": {}, "source": [ - "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + "The `TritonClusterManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n", + "- Find available ports for HTTP/gRPC/metrics\n", + "- Deploy a server on each node via stage-level scheduling\n", + "- Gracefully shutdown servers across nodes" ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "id": "3d522f30", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - } - ], - "source": [ - "sc = spark.sparkContext\n", - "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" - ] - }, - { - "cell_type": "markdown", - "id": "f9cc80bf", - "metadata": {}, - "source": [ - "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "id": "3487a85d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using ports [7000, 7001, 7002]\n" - ] - } - ], + "outputs": [], "source": [ "model_name = \"ConditionalGeneration\"\n", - "ports = find_ports()\n", - "assert len(ports) == 3\n", - "print(f\"Using ports {ports}\")" + "server_manager = TritonServerManager(num_nodes=num_nodes, model_name=model_name)" ] }, { "cell_type": "code", - "execution_count": 35, - "id": "c605ab40", + "execution_count": null, + "id": "7c18994c", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 46:> (0 + 1) / 1]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Triton Server PIDs:\n", - " {\n", - " \"cb4ae00-lcedt\": 3010304\n", - "}\n" + "2025-02-07 11:03:44,809 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-07 11:03:44,810 - INFO - Starting 1 servers.\n" ] }, { @@ -1207,13 +1165,20 @@ "text": [ " \r" ] + }, + { + "data": { + "text/plain": [ + "{'cb4ae00-lcedt': (2020631, [7000, 7001, 7002])}" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", - " ports=ports,\n", - " model_name=model_name)).collectAsMap()\n", - "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n", + "server_manager.start_servers(triton_server)" ] }, { @@ -1224,27 +1189,45 @@ "#### Define client function" ] }, + { + "cell_type": "markdown", + "id": "237e56dd", + "metadata": {}, + "source": [ + "Get the hostname -> url mapping from the server manager:" + ] + }, { "cell_type": "code", - "execution_count": 36, - "id": "404c5091", + "execution_count": null, + "id": "826db582", "metadata": {}, "outputs": [], "source": [ - "url = f\"http://localhost:{ports[0]}\"" + "host_to_http_url = server_manager.host_to_http_url # or server_manager.host_to_grpc_url" + ] + }, + { + "cell_type": "markdown", + "id": "f3f58e7b", + "metadata": {}, + "source": [ + "Define the Triton inference function, which returns a predict function for batch inference through the server:" ] }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 36, "id": "aff88b3f", "metadata": {}, "outputs": [], "source": [ - "def triton_fn(url, model_name):\n", + "def triton_fn(model_name, host_to_url):\n", + " import socket\n", " import numpy as np\n", " from pytriton.client import ModelClient\n", "\n", + " url = host_to_url[socket.gethostname()]\n", " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", "\n", " def infer_batch(inputs):\n", @@ -1261,6 +1244,19 @@ " return infer_batch" ] }, + { + "cell_type": "code", + "execution_count": 40, + "id": "5d10c61c-6102-4d19-8dd6-0c7b5b65343e", + "metadata": {}, + "outputs": [], + "source": [ + "generate = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_http_url),\n", + " return_type=StringType(),\n", + " input_tensor_shapes=[[1]],\n", + " batch_size=32)" + ] + }, { "cell_type": "markdown", "id": "a85e2ceb", @@ -1271,7 +1267,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 37, "id": "2fa3664e", "metadata": {}, "outputs": [], @@ -1285,7 +1281,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 38, "id": "5d6c54e7-534d-406f-b8e6-fd592efd0ab2", "metadata": {}, "outputs": [], @@ -1295,7 +1291,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 39, "id": "dc1bbbe3-4232-49e5-80f6-99976524b73b", "metadata": {}, "outputs": [ @@ -1303,7 +1299,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 20:03:03 WARN CacheManager: Asked to cache already cached data.\n" + "25/02/04 13:55:37 WARN CacheManager: Asked to cache already cached data.\n" ] } ], @@ -1322,19 +1318,6 @@ { "cell_type": "code", "execution_count": 41, - "id": "5d10c61c-6102-4d19-8dd6-0c7b5b65343e", - "metadata": {}, - "outputs": [], - "source": [ - "generate = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", - " return_type=StringType(),\n", - " input_tensor_shapes=[[1]],\n", - " batch_size=32)" - ] - }, - { - "cell_type": "code", - "execution_count": 42, "id": "2e0907da-a5d9-4c3b-9db4-ce5e70ca9bb4", "metadata": {}, "outputs": [ @@ -1342,15 +1325,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 50:===========================================> (6 + 2) / 8]\r" + "[Stage 51:==================================================> (7 + 1) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 9.24 ms, sys: 8.47 ms, total: 17.7 ms\n", - "Wall time: 27.2 s\n" + "CPU times: user 10.8 ms, sys: 8.12 ms, total: 18.9 ms\n", + "Wall time: 30 s\n" ] }, { @@ -1370,7 +1353,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 42, "id": "9308bdd7-6f67-484d-8b51-dd1e1b2960ba", "metadata": {}, "outputs": [ @@ -1378,15 +1361,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 53:===========================================> (6 + 2) / 8]\r" + "[Stage 54:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 9.13 ms, sys: 4.18 ms, total: 13.3 ms\n", - "Wall time: 20.7 s\n" + "CPU times: user 7.23 ms, sys: 3.43 ms, total: 10.7 ms\n", + "Wall time: 21.2 s\n" ] }, { @@ -1405,7 +1388,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 43, "id": "38484ffd-370d-492b-8ca4-9eff9f242a9f", "metadata": {}, "outputs": [ @@ -1413,15 +1396,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 56:===========================================> (6 + 2) / 8]\r" + "[Stage 57:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 8.88 ms, sys: 5.1 ms, total: 14 ms\n", - "Wall time: 21 s\n" + "CPU times: user 2.81 ms, sys: 12.7 ms, total: 15.5 ms\n", + "Wall time: 22.3 s\n" ] }, { @@ -1440,7 +1423,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 44, "id": "ebcb6699-3ac2-4529-ab0f-fab0a5e792da", "metadata": {}, "outputs": [ @@ -1448,7 +1431,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 59:> (0 + 1) / 1]\r" + "[Stage 60:> (0 + 1) / 1]\r" ] }, { @@ -1507,22 +1490,16 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 45, "id": "425d3b28-7705-45ba-8a18-ad34fc895219", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "2025-02-04 13:56:54,506 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-04 13:56:59,695 - INFO - Sucessfully stopped 1 servers. \n" ] }, { @@ -1531,20 +1508,18 @@ "[True]" ] }, - "execution_count": 46, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", - "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" + "server_manager.stop_servers()" ] }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 46, "id": "2dec80ca-7a7c-46a9-97c0-7afb1572f5b9", "metadata": {}, "outputs": [], diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_torch.ipynb index 924c4d41..e7dd198b 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/conditional_generation_torch.ipynb @@ -196,12 +196,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:35:04 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "25/01/27 19:35:05 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "25/02/04 13:34:55 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/02/04 13:34:55 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "25/01/27 19:35:05 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", - "25/01/27 19:35:05 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + "25/02/04 13:34:55 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" ] } ], @@ -345,13 +344,6 @@ } }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, { "data": { "text/plain": [ @@ -388,7 +380,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:35:12 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + "25/02/04 13:35:02 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] }, { @@ -427,7 +419,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:35:12 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + "25/02/04 13:35:02 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] } ], @@ -716,15 +708,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 24:====================================> (5 + 3) / 8]\r" + "[Stage 24:=============================> (4 + 4) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 8.37 ms, sys: 1.12 ms, total: 9.48 ms\n", - "Wall time: 6.8 s\n" + "CPU times: user 10.2 ms, sys: 5.05 ms, total: 15.2 ms\n", + "Wall time: 7.41 s\n" ] }, { @@ -763,15 +755,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 27:==================================================> (7 + 1) / 8]\r" + "[Stage 27:=============================> (4 + 4) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 6.1 ms, sys: 1.19 ms, total: 7.29 ms\n", - "Wall time: 3.9 s\n" + "CPU times: user 3.93 ms, sys: 1.98 ms, total: 5.91 ms\n", + "Wall time: 4.08 s\n" ] }, { @@ -809,15 +801,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 30:===========================================> (6 + 2) / 8]\r" + "[Stage 30:==============> (2 + 6) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 1.79 ms, sys: 4.62 ms, total: 6.4 ms\n", - "Wall time: 3.88 s\n" + "CPU times: user 3.85 ms, sys: 1.75 ms, total: 5.6 ms\n", + "Wall time: 4.08 s\n" ] }, { @@ -920,8 +912,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 6.63 ms, sys: 618 μs, total: 7.25 ms\n", - "Wall time: 4.04 s\n" + "CPU times: user 6.02 ms, sys: 705 μs, total: 6.73 ms\n", + "Wall time: 4.24 s\n" ] }, { @@ -948,15 +940,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 39:====================================> (5 + 3) / 8]\r" + "[Stage 39:==============> (2 + 6) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 6.29 ms, sys: 0 ns, total: 6.29 ms\n", - "Wall time: 3.69 s\n" + "CPU times: user 6.12 ms, sys: 319 μs, total: 6.43 ms\n", + "Wall time: 3.88 s\n" ] }, { @@ -982,15 +974,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 42:==================================================> (7 + 1) / 8]\r" + "[Stage 42:==============> (2 + 6) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 5.03 ms, sys: 1.58 ms, total: 6.61 ms\n", - "Wall time: 3.7 s\n" + "CPU times: user 7.03 ms, sys: 16 μs, total: 7.05 ms\n", + "Wall time: 3.9 s\n" ] }, { @@ -1103,7 +1095,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Import the utility functions from pytriton_utils.py:" + "Import the helper class from pytriton_utils.py:" ] }, { @@ -1114,12 +1106,7 @@ "source": [ "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "from pytriton_utils import (\n", - " use_stage_level_scheduling,\n", - " find_ports,\n", - " start_triton,\n", - " stop_triton\n", - ")" + "from pytriton_utils import TritonServerManager" ] }, { @@ -1202,6 +1189,7 @@ " )\n", "\n", " def _stop_triton(signum, frame):\n", + " # The server manager sends SIGTERM to stop the server; this function ensures graceful cleanup.\n", " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", " triton.stop()\n", "\n", @@ -1252,7 +1240,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + "The `TritonClusterManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n", + "- Find available ports for HTTP/gRPC/metrics\n", + "- Deploy a server on each node via stage-level scheduling\n", + "- Gracefully shutdown servers across nodes" ] }, { @@ -1271,80 +1262,23 @@ "title": "" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - } - ], - "source": [ - "sc = spark.sparkContext\n", - "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using ports [7000, 7001, 7002]\n" - ] - } - ], + "outputs": [], "source": [ "model_name = \"ConditionalGeneration\"\n", - "ports = find_ports()\n", - "assert len(ports) == 3\n", - "print(f\"Using ports {ports}\")" + "server_manager = TritonServerManager(num_nodes=num_nodes, model_name=model_name)" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "289b08fa-7916-44ea-8fe5-28821451db6b", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 46:> (0 + 1) / 1]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Triton Server PIDs:\n", - " {\n", - " \"cb4ae00-lcedt\": 2950716\n", - "}\n" + "2025-02-07 11:03:44,809 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-07 11:03:44,810 - INFO - Starting 1 servers.\n" ] }, { @@ -1353,13 +1287,20 @@ "text": [ " \r" ] + }, + { + "data": { + "text/plain": [ + "{'cb4ae00-lcedt': (2020631, [7000, 7001, 7002])}" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", - " ports=ports,\n", - " model_name=model_name)).collectAsMap()\n", - "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n", + "server_manager.start_servers(triton_server)" ] }, { @@ -1369,18 +1310,32 @@ "#### Define client function" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Get the hostname -> url mapping from the server manager:" + ] + }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "url = f\"http://localhost:{ports[0]}\"" + "host_to_http_url = server_manager.host_to_http_url # or server_manager.host_to_grpc_url" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define the Triton inference function, which returns a predict function for batch inference through the server:" ] }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 35, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -1396,10 +1351,12 @@ }, "outputs": [], "source": [ - "def triton_fn(url, model_name):\n", + "def triton_fn(model_name, host_to_url):\n", + " import socket\n", " import numpy as np\n", " from pytriton.client import ModelClient\n", "\n", + " url = host_to_url[socket.gethostname()]\n", " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", "\n", " def infer_batch(inputs):\n", @@ -1416,6 +1373,30 @@ " return infer_batch" ] }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "be692f4a-cf86-4cf4-9530-7c62e479cacd", + "showTitle": false, + "tableResultSettingsMap": {}, + "title": "" + } + }, + "outputs": [], + "source": [ + "generate = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_http_url),\n", + " return_type=StringType(),\n", + " input_tensor_shapes=[[1]],\n", + " batch_size=32)" + ] + }, { "cell_type": "markdown", "metadata": { @@ -1437,7 +1418,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 36, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -1462,7 +1443,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 37, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -1483,7 +1464,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 38, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -1502,7 +1483,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:35:45 WARN CacheManager: Asked to cache already cached data.\n" + "25/02/04 13:35:39 WARN CacheManager: Asked to cache already cached data.\n" ] } ], @@ -1520,30 +1501,6 @@ { "cell_type": "code", "execution_count": 40, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "be692f4a-cf86-4cf4-9530-7c62e479cacd", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, - "outputs": [], - "source": [ - "generate = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", - " return_type=StringType(),\n", - " input_tensor_shapes=[[1]],\n", - " batch_size=32)" - ] - }, - { - "cell_type": "code", - "execution_count": 41, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -1562,15 +1519,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 50:===========================================> (6 + 2) / 8]\r" + "[Stage 51:====================================> (5 + 3) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 8.21 ms, sys: 3.06 ms, total: 11.3 ms\n", - "Wall time: 4.43 s\n" + "CPU times: user 5.09 ms, sys: 4.41 ms, total: 9.5 ms\n", + "Wall time: 4.96 s\n" ] }, { @@ -1590,7 +1547,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 41, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -1609,15 +1566,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 53:===========================================> (6 + 2) / 8]\r" + "[Stage 54:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 6.47 ms, sys: 910 μs, total: 7.38 ms\n", - "Wall time: 4.24 s\n" + "CPU times: user 5.4 ms, sys: 1.12 ms, total: 6.52 ms\n", + "Wall time: 4.41 s\n" ] }, { @@ -1636,7 +1593,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 42, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -1655,15 +1612,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 56:===========================================> (6 + 2) / 8]\r" + "[Stage 57:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 5.84 ms, sys: 1.27 ms, total: 7.11 ms\n", - "Wall time: 4.29 s\n" + "CPU times: user 4.59 ms, sys: 1.79 ms, total: 6.38 ms\n", + "Wall time: 4.55 s\n" ] }, { @@ -1682,7 +1639,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 43, "metadata": { "application/vnd.databricks.v1+cell": { "cellMetadata": { @@ -1755,33 +1712,15 @@ }, { "cell_type": "code", - "execution_count": 45, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "16fd4601-f6d5-4ddf-9b5e-d918ab0adf3a", - "showTitle": false, - "tableResultSettingsMap": {}, - "title": "" - } - }, + "execution_count": 44, + "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "2025-02-04 13:35:53,794 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-04 13:35:58,983 - INFO - Sucessfully stopped 1 servers. \n" ] }, { @@ -1790,20 +1729,18 @@ "[True]" ] }, - "execution_count": 45, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", - "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" + "server_manager.stop_servers()" ] }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -1863,7 +1800,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.11.11" } }, "nbformat": 4, diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb index 58f9bcd7..1d3fe6ff 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_tf.ipynb @@ -32,13 +32,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-01-27 12:06:26.417984: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2025-01-27 12:06:26.426005: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2025-01-27 12:06:26.434857: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2025-01-27 12:06:26.437414: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2025-01-27 12:06:26.444254: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "2025-02-04 13:57:08.242673: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2025-02-04 13:57:08.249833: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2025-02-04 13:57:08.257735: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2025-02-04 13:57:08.259994: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2025-02-04 13:57:08.266655: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2025-01-27 12:06:26.880520: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" + "2025-02-04 13:57:08.649929: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" ] } ], @@ -63,9 +63,9 @@ "output_type": "stream", "text": [ "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", - "I0000 00:00:1738008387.629698 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008387.653432 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008387.656121 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" + "I0000 00:00:1738706229.309141 3668091 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706229.333555 3668091 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706229.336487 3668091 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" ] } ], @@ -118,13 +118,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "I0000 00:00:1738008387.957197 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008387.960022 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008387.962670 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008388.067491 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008388.068546 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008388.069484 3016848 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "2025-01-27 12:06:28.070408: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 43043 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n", + "I0000 00:00:1738706229.617170 3668091 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706229.620218 3668091 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706229.622781 3668091 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706229.732012 3668091 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706229.733045 3668091 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706229.733965 3668091 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2025-02-04 13:57:09.734873: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 43096 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n", "All PyTorch model weights were used when initializing TFDistilBertForSequenceClassification.\n", "\n", "All the weights of TFDistilBertForSequenceClassification were initialized from the PyTorch model.\n", @@ -138,7 +138,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "3b91fe91-b725-4564-ae93-56e3fb51e47c", "metadata": {}, "outputs": [ @@ -159,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "0be39eb3-462c-42ff-b8f4-09f4e4fe3a3c", "metadata": {}, "outputs": [ @@ -222,7 +222,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "31079133", "metadata": {}, "outputs": [ @@ -320,12 +320,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 20:06:30 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "25/01/27 20:06:30 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "25/02/04 13:57:12 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/02/04 13:57:12 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "25/01/27 20:06:30 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", - "25/01/27 20:06:31 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + "25/02/04 13:57:12 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" ] } ], @@ -416,13 +415,6 @@ "id": "1db4db3a", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, { "data": { "text/plain": [ @@ -448,7 +440,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 20:06:38 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + "25/02/04 13:57:20 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] }, { @@ -476,7 +468,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 20:06:38 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + "25/02/04 13:57:20 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] } ], @@ -629,8 +621,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 6.91 ms, sys: 3.39 ms, total: 10.3 ms\n", - "Wall time: 4.75 s\n" + "CPU times: user 8.06 ms, sys: 2.92 ms, total: 11 ms\n", + "Wall time: 4.86 s\n" ] }, { @@ -666,8 +658,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.93 ms, sys: 3.4 ms, total: 6.33 ms\n", - "Wall time: 1.24 s\n" + "CPU times: user 4.5 ms, sys: 1.43 ms, total: 5.93 ms\n", + "Wall time: 1.19 s\n" ] }, { @@ -701,8 +693,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.63 ms, sys: 2.03 ms, total: 5.66 ms\n", - "Wall time: 1.31 s\n" + "CPU times: user 5.9 ms, sys: 605 μs, total: 6.5 ms\n", + "Wall time: 1.37 s\n" ] }, { @@ -725,6 +717,13 @@ "id": "c01761b3-c766-46b0-ae0b-fcf968ffb3a1", "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 27:> (0 + 1) / 1]\r" + ] + }, { "name": "stdout", "output_type": "stream", @@ -756,6 +755,13 @@ "only showing top 20 rows\n", "\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] } ], "source": [ @@ -795,7 +801,7 @@ "id": "4f15dfcb", "metadata": {}, "source": [ - "Import the utility functions from pytriton_utils.py:" + "Import the helper class from pytriton_utils.py:" ] }, { @@ -807,12 +813,7 @@ "source": [ "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "from pytriton_utils import (\n", - " use_stage_level_scheduling,\n", - " find_ports,\n", - " start_triton,\n", - " stop_triton\n", - ")" + "from pytriton_utils import TritonServerManager" ] }, { @@ -924,82 +925,38 @@ }, { "cell_type": "markdown", - "id": "c10905de", + "id": "5354c597", "metadata": {}, "source": [ - "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + "The `TritonClusterManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n", + "- Find available ports for HTTP/gRPC/metrics\n", + "- Deploy a server on each node via stage-level scheduling\n", + "- Gracefully shutdown servers across nodes" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "id": "156de815", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - } - ], - "source": [ - "sc = spark.sparkContext\n", - "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" - ] - }, - { - "cell_type": "markdown", - "id": "736ac5f4", - "metadata": {}, - "source": [ - "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "id": "4b6044f9", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using ports [7000, 7001, 7002]\n" - ] - } - ], + "outputs": [], "source": [ "model_name = \"SentimentAnalysis\"\n", - "ports = find_ports()\n", - "assert len(ports) == 3\n", - "print(f\"Using ports {ports}\")" + "server_manager = TritonServerManager(num_nodes=num_nodes, model_name=model_name)" ] }, { "cell_type": "code", - "execution_count": 33, - "id": "f75c30c5", + "execution_count": null, + "id": "d003a862", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 28:> (0 + 1) / 1]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Triton Server PIDs:\n", - " {\n", - " \"cb4ae00-lcedt\": 3019330\n", - "}\n" + "2025-02-07 11:03:44,809 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-07 11:03:44,810 - INFO - Starting 1 servers.\n" ] }, { @@ -1008,13 +965,20 @@ "text": [ " \r" ] + }, + { + "data": { + "text/plain": [ + "{'cb4ae00-lcedt': (2020631, [7000, 7001, 7002])}" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", - " ports=ports,\n", - " model_name=model_name)).collectAsMap()\n", - "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n", + "server_manager.start_servers(triton_server)" ] }, { @@ -1025,27 +989,45 @@ "#### Define client function" ] }, + { + "cell_type": "markdown", + "id": "405edc49", + "metadata": {}, + "source": [ + "Get the hostname -> url mapping from the server manager:" + ] + }, { "cell_type": "code", - "execution_count": 34, - "id": "35bf6939", + "execution_count": null, + "id": "19768ddb", "metadata": {}, "outputs": [], "source": [ - "url = f\"http://localhost:{ports[0]}\"" + "host_to_http_url = server_manager.host_to_http_url # or server_manager.host_to_grpc_url" + ] + }, + { + "cell_type": "markdown", + "id": "eb5dbb89", + "metadata": {}, + "source": [ + "Define the Triton inference function, which returns a predict function for batch inference through the server:" ] }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 34, "id": "431b864c", "metadata": {}, "outputs": [], "source": [ - "def triton_fn(url, model_name):\n", + "def triton_fn(model_name, host_to_url):\n", + " import socket\n", " import numpy as np\n", " from pytriton.client import ModelClient\n", "\n", + " url = host_to_url[socket.gethostname()]\n", " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", "\n", " def infer_batch(inputs):\n", @@ -1062,6 +1044,22 @@ " return infer_batch" ] }, + { + "cell_type": "code", + "execution_count": 37, + "id": "3930cfcd-3284-4c6a-a9b5-36b8053fe899", + "metadata": {}, + "outputs": [], + "source": [ + "classify = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_http_url),\n", + " return_type=StructType([\n", + " StructField(\"label\", StringType(), True),\n", + " StructField(\"score\", FloatType(), True)\n", + " ]),\n", + " input_tensor_shapes=[[1]],\n", + " batch_size=32)" + ] + }, { "cell_type": "markdown", "id": "5a8ec7be", @@ -1072,7 +1070,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 35, "id": "d53fb283-bf9e-4571-8c68-b75a41f1f067", "metadata": {}, "outputs": [], @@ -1084,7 +1082,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 36, "id": "29b0cc0d-c480-4e4a-bd41-207dc314cba5", "metadata": {}, "outputs": [ @@ -1092,7 +1090,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 20:06:54 WARN CacheManager: Asked to cache already cached data.\n" + "25/02/04 13:57:36 WARN CacheManager: Asked to cache already cached data.\n" ] } ], @@ -1112,22 +1110,6 @@ { "cell_type": "code", "execution_count": 38, - "id": "3930cfcd-3284-4c6a-a9b5-36b8053fe899", - "metadata": {}, - "outputs": [], - "source": [ - "classify = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", - " return_type=StructType([\n", - " StructField(\"label\", StringType(), True),\n", - " StructField(\"score\", FloatType(), True)\n", - " ]),\n", - " input_tensor_shapes=[[1]],\n", - " batch_size=32)" - ] - }, - { - "cell_type": "code", - "execution_count": 39, "id": "8eecbf23-4e9e-4d4c-8645-98209b25db2c", "metadata": {}, "outputs": [ @@ -1135,15 +1117,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 32:===========================================> (6 + 2) / 8]\r" + "[Stage 33:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 7.31 ms, sys: 4 ms, total: 11.3 ms\n", - "Wall time: 7.44 s\n" + "CPU times: user 13.1 ms, sys: 8.29 ms, total: 21.4 ms\n", + "Wall time: 7.54 s\n" ] }, { @@ -1164,7 +1146,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 39, "id": "566ba28c-0ca4-4479-a24a-c8a362228b89", "metadata": {}, "outputs": [ @@ -1172,15 +1154,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 35:===========================================> (6 + 2) / 8]\r" + "[Stage 36:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 6.68 ms, sys: 1.86 ms, total: 8.54 ms\n", - "Wall time: 7.33 s\n" + "CPU times: user 7.54 ms, sys: 3.13 ms, total: 10.7 ms\n", + "Wall time: 7.02 s\n" ] }, { @@ -1199,7 +1181,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 40, "id": "44c7e776-08da-484a-ba07-9d6add1a0f15", "metadata": {}, "outputs": [ @@ -1207,15 +1189,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 38:===========================================> (6 + 2) / 8]\r" + "[Stage 39:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 7.34 ms, sys: 1.15 ms, total: 8.49 ms\n", - "Wall time: 6.88 s\n" + "CPU times: user 6.26 ms, sys: 3 ms, total: 9.26 ms\n", + "Wall time: 7.03 s\n" ] }, { @@ -1234,7 +1216,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 41, "id": "f61d79f8-661e-4d9e-a3aa-c0754b854603", "metadata": {}, "outputs": [ @@ -1292,22 +1274,16 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 42, "id": "425d3b28-7705-45ba-8a18-ad34fc895219", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "2025-02-04 13:57:58,747 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-04 13:58:03,931 - INFO - Sucessfully stopped 1 servers. \n" ] }, { @@ -1316,20 +1292,18 @@ "[True]" ] }, - "execution_count": 43, + "execution_count": 42, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", - "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" + "server_manager.stop_servers()" ] }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 43, "id": "9f19643c-4ee4-44f2-b762-2078c0c8eba9", "metadata": {}, "outputs": [], diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb index d847d26a..476168bc 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/pipelines_torch.ipynb @@ -46,7 +46,8 @@ "output_type": "stream", "text": [ "No model was supplied, defaulted to distilbert/distilbert-base-uncased-finetuned-sst-2-english and revision 714eb0f (https://huggingface.co/distilbert/distilbert-base-uncased-finetuned-sst-2-english).\n", - "Using a pipeline without specifying a model name and revision in production is not recommended.\n" + "Using a pipeline without specifying a model name and revision in production is not recommended.\n", + "Device set to use cuda\n" ] } ], @@ -133,6 +134,13 @@ "id": "312017fc", "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Device set to use cuda\n" + ] + }, { "data": { "text/plain": [ @@ -227,12 +235,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:43:21 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "25/01/27 19:43:21 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "25/02/04 13:23:47 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/02/04 13:23:47 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "25/01/27 19:43:21 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", - "25/01/27 19:43:22 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + "25/02/04 13:23:47 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" ] } ], @@ -313,13 +320,6 @@ "id": "b0d1876b", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, { "data": { "text/plain": [ @@ -345,7 +345,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:43:28 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + "25/02/04 13:23:54 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] }, { @@ -373,7 +373,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:43:29 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + "25/02/04 13:23:54 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] } ], @@ -510,15 +510,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 18:=====================> (3 + 5) / 8]\r" + "[Stage 18:====================================> (5 + 3) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 8.71 ms, sys: 5.88 ms, total: 14.6 ms\n", - "Wall time: 3.03 s\n" + "CPU times: user 8.82 ms, sys: 2.5 ms, total: 11.3 ms\n", + "Wall time: 3.59 s\n" ] }, { @@ -547,8 +547,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.41 ms, sys: 1.2 ms, total: 4.61 ms\n", - "Wall time: 391 ms\n" + "CPU times: user 3.19 ms, sys: 1.65 ms, total: 4.84 ms\n", + "Wall time: 392 ms\n" ] } ], @@ -568,8 +568,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 4.57 ms, sys: 291 μs, total: 4.87 ms\n", - "Wall time: 409 ms\n" + "CPU times: user 3.43 ms, sys: 2.33 ms, total: 5.77 ms\n", + "Wall time: 403 ms\n" ] } ], @@ -594,10 +594,10 @@ "+--------------------------------------------------------------------------------+--------+----------+\n", "|Doesn't anyone bother to check where this kind of sludge comes from before bl...|NEGATIVE| 0.9984042|\n", "| There were two things I hated about WASTED : The directing and the script |NEGATIVE| 0.9979019|\n", - "| I'm rather surprised that anybody found this film touching or moving|POSITIVE| 0.8392794|\n", + "| I'm rather surprised that anybody found this film touching or moving|POSITIVE| 0.839279|\n", "|Cultural Vandalism Is the new Hallmark production of Gulliver's Travels an ac...|NEGATIVE|0.99726933|\n", - "|I was at Wrestlemania VI in Toronto as a 10 year old, and the event I saw the...|POSITIVE|0.98212516|\n", - "| This movie has been done before|NEGATIVE|0.94194806|\n", + "|I was at Wrestlemania VI in Toronto as a 10 year old, and the event I saw the...|POSITIVE|0.98212504|\n", + "| This movie has been done before|NEGATIVE| 0.9419482|\n", "|[ as a new resolution for this year 2005, i decide to write a comment for eac...|NEGATIVE|0.99678314|\n", "|This movie is over hyped!! I am sad to say that I manage to watch the first 1...|NEGATIVE| 0.9985846|\n", "|This show had a promising start as sort of the opposite of 'Oceans 11' but ha...|NEGATIVE|0.99926823|\n", @@ -652,27 +652,22 @@ }, { "cell_type": "markdown", - "id": "42867a0b", + "id": "ab52381b", "metadata": {}, "source": [ - "Import the utility functions from pytriton_utils.py:" + "Import the helper class from pytriton_utils.py:" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 32, "id": "4e6764c4", "metadata": {}, "outputs": [], "source": [ "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "from pytriton_utils import (\n", - " use_stage_level_scheduling,\n", - " find_ports,\n", - " start_triton,\n", - " stop_triton\n", - ")" + "from pytriton_utils import TritonServerManager" ] }, { @@ -685,7 +680,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 33, "id": "7e53df9f-43cb-4c38-b8ac-dc2cbad99815", "metadata": {}, "outputs": [], @@ -735,6 +730,7 @@ " )\n", "\n", " def _stop_triton(signum, frame):\n", + " # The server manager sends SIGTERM to stop the server; this function ensures graceful cleanup.\n", " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", " triton.stop()\n", "\n", @@ -763,7 +759,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 34, "id": "a4757163", "metadata": {}, "outputs": [], @@ -774,83 +770,38 @@ }, { "cell_type": "markdown", - "id": "767c40e6", + "id": "b5ef160a", "metadata": {}, "source": [ - "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + "The `TritonClusterManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n", + "- Find available ports for HTTP/gRPC/metrics\n", + "- Deploy a server on each node via stage-level scheduling\n", + "- Gracefully shutdown servers across nodes" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 35, "id": "ad13db78", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - } - ], - "source": [ - "sc = spark.sparkContext\n", - "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" - ] - }, - { - "cell_type": "markdown", - "id": "5febf6e8", - "metadata": {}, - "source": [ - "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "id": "79a4e9d7", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using ports [7000, 7001, 7002]\n" - ] - } - ], + "outputs": [], "source": [ "model_name = \"SentimentAnalysis\"\n", - "\n", - "ports = find_ports()\n", - "assert len(ports) == 3\n", - "print(f\"Using ports {ports}\")" + "server_manager = TritonServerManager(num_nodes=num_nodes, model_name=model_name)" ] }, { "cell_type": "code", "execution_count": null, - "id": "7a1a4c4c", + "id": "e62d9739", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 28:> (0 + 1) / 1]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Triton Server PIDs:\n", - " {\n", - " \"cb4ae00-lcedt\": 2956583\n", - "}\n" + "2025-02-07 11:03:44,809 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-07 11:03:44,810 - INFO - Starting 1 servers.\n" ] }, { @@ -859,13 +810,20 @@ "text": [ " \r" ] + }, + { + "data": { + "text/plain": [ + "{'cb4ae00-lcedt': (2020631, [7000, 7001, 7002])}" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", - " ports=ports,\n", - " model_name=model_name)).collectAsMap()\n", - "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n", + "server_manager.start_servers(triton_server)" ] }, { @@ -876,27 +834,45 @@ "#### Define client function" ] }, + { + "cell_type": "markdown", + "id": "9e2059f9", + "metadata": {}, + "source": [ + "Get the hostname -> url mapping from the server manager:" + ] + }, { "cell_type": "code", - "execution_count": 33, - "id": "f6899d96", + "execution_count": null, + "id": "7ede428b", "metadata": {}, "outputs": [], "source": [ - "url = f\"http://localhost:{ports[0]}\"" + "host_to_http_url = server_manager.host_to_http_url # or server_manager.host_to_grpc_url" + ] + }, + { + "cell_type": "markdown", + "id": "72f16ff5", + "metadata": {}, + "source": [ + "Define the Triton inference function, which returns a predict function for batch inference through the server:" ] }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 38, "id": "14760940", "metadata": {}, "outputs": [], "source": [ - "def triton_fn(url, model_name):\n", + "def triton_fn(model_name, host_to_url):\n", + " import socket\n", " import numpy as np\n", " from pytriton.client import ModelClient\n", "\n", + " url = host_to_url[socket.gethostname()]\n", " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", "\n", " def infer_batch(inputs):\n", @@ -913,6 +889,22 @@ " return infer_batch" ] }, + { + "cell_type": "code", + "execution_count": 41, + "id": "3930cfcd-3284-4c6a-a9b5-36b8053fe899", + "metadata": {}, + "outputs": [], + "source": [ + "classify = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_http_url),\n", + " return_type=StructType([\n", + " StructField(\"label\", StringType(), True),\n", + " StructField(\"score\", FloatType(), True)\n", + " ]),\n", + " input_tensor_shapes=[[1]],\n", + " batch_size=32)" + ] + }, { "cell_type": "markdown", "id": "a741e23a", @@ -923,7 +915,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 39, "id": "ccc884a4", "metadata": {}, "outputs": [], @@ -935,7 +927,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 40, "id": "c426fdbe", "metadata": {}, "outputs": [ @@ -943,7 +935,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:43:39 WARN CacheManager: Asked to cache already cached data.\n" + "25/02/04 13:24:35 WARN CacheManager: Asked to cache already cached data.\n" ] } ], @@ -962,23 +954,7 @@ }, { "cell_type": "code", - "execution_count": 37, - "id": "3930cfcd-3284-4c6a-a9b5-36b8053fe899", - "metadata": {}, - "outputs": [], - "source": [ - "classify = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", - " return_type=StructType([\n", - " StructField(\"label\", StringType(), True),\n", - " StructField(\"score\", FloatType(), True)\n", - " ]),\n", - " input_tensor_shapes=[[1]],\n", - " batch_size=32)" - ] - }, - { - "cell_type": "code", - "execution_count": 38, + "execution_count": 42, "id": "8eecbf23-4e9e-4d4c-8645-98209b25db2c", "metadata": {}, "outputs": [ @@ -986,8 +962,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.67 ms, sys: 1.81 ms, total: 5.48 ms\n", - "Wall time: 647 ms\n" + "CPU times: user 10.5 ms, sys: 2.2 ms, total: 12.7 ms\n", + "Wall time: 671 ms\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" ] } ], @@ -1001,7 +984,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 43, "id": "566ba28c-0ca4-4479-a24a-c8a362228b89", "metadata": {}, "outputs": [ @@ -1009,8 +992,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 436 μs, sys: 3.23 ms, total: 3.66 ms\n", - "Wall time: 477 ms\n" + "CPU times: user 1.68 ms, sys: 1.87 ms, total: 3.55 ms\n", + "Wall time: 396 ms\n" ] } ], @@ -1022,7 +1005,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 44, "id": "44c7e776-08da-484a-ba07-9d6add1a0f15", "metadata": {}, "outputs": [ @@ -1030,8 +1013,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.62 ms, sys: 792 μs, total: 4.41 ms\n", - "Wall time: 459 ms\n" + "CPU times: user 3.06 ms, sys: 5.02 ms, total: 8.08 ms\n", + "Wall time: 408 ms\n" ] } ], @@ -1043,7 +1026,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 45, "id": "f61d79f8-661e-4d9e-a3aa-c0754b854603", "metadata": {}, "outputs": [ @@ -1056,10 +1039,10 @@ "+----------------------------------------------------------------------+--------+----------+\n", "|Doesn't anyone bother to check where this kind of sludge comes from...|NEGATIVE| 0.9984042|\n", "|There were two things I hated about WASTED : The directing and the ...|NEGATIVE| 0.9979019|\n", - "| I'm rather surprised that anybody found this film touching or moving|POSITIVE| 0.8392794|\n", + "| I'm rather surprised that anybody found this film touching or moving|POSITIVE| 0.839279|\n", "|Cultural Vandalism Is the new Hallmark production of Gulliver's Tra...|NEGATIVE|0.99726933|\n", - "|I was at Wrestlemania VI in Toronto as a 10 year old, and the event...|POSITIVE|0.98212516|\n", - "| This movie has been done before|NEGATIVE|0.94194806|\n", + "|I was at Wrestlemania VI in Toronto as a 10 year old, and the event...|POSITIVE|0.98212504|\n", + "| This movie has been done before|NEGATIVE| 0.9419482|\n", "|[ as a new resolution for this year 2005, i decide to write a comme...|NEGATIVE|0.99678314|\n", "|This movie is over hyped!! I am sad to say that I manage to watch t...|NEGATIVE| 0.9985846|\n", "|This show had a promising start as sort of the opposite of 'Oceans ...|NEGATIVE|0.99926823|\n", @@ -1094,22 +1077,22 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 46, "id": "e3a4e51f", "metadata": {}, "outputs": [ { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" + "2025-02-04 13:24:40,325 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "2025-02-04 13:24:45,576 - INFO - Sucessfully stopped 1 servers. \n" ] }, { @@ -1118,20 +1101,18 @@ "[True]" ] }, - "execution_count": 42, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", - "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" + "server_manager.stop_servers()" ] }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 47, "id": "9f19643c-4ee4-44f2-b762-2078c0c8eba9", "metadata": {}, "outputs": [], @@ -1143,7 +1124,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6a538c47-317d-4cac-b9b9-559e88677518", + "id": "a8b03e1e", "metadata": {}, "outputs": [], "source": [] @@ -1165,7 +1146,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.11.11" } }, "nbformat": 4, diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/sentence_transformers_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/sentence_transformers_torch.ipynb index 83d4ecba..9e8d6d48 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/sentence_transformers_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/sentence_transformers_torch.ipynb @@ -16,19 +16,10 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "c5f0d0a8", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:13: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n", - " from tqdm.autonotebook import tqdm, trange\n" - ] - } - ], + "outputs": [], "source": [ "import torch\n", "from sentence_transformers import SentenceTransformer\n", @@ -41,19 +32,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "731faab7-a700-46f8-bba5-1c8764e5eacb", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/transformers/tokenization_utils_base.py:1617: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be deprecated in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n", - " warnings.warn(\n" - ] - } - ], + "outputs": [], "source": [ "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "model = SentenceTransformer(\"paraphrase-MiniLM-L6-v2\", device=device)\n", @@ -160,12 +142,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:47:12 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "25/01/27 19:47:12 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "25/02/04 13:40:01 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/02/04 13:40:01 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "25/01/27 19:47:12 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", - "25/01/27 19:47:12 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + "25/02/04 13:40:01 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" ] } ], @@ -279,7 +260,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:47:19 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + "25/02/04 13:40:08 WARN TaskSetManager: Stage 6 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] }, { @@ -307,7 +288,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:47:19 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + "25/02/04 13:40:08 WARN TaskSetManager: Stage 9 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] } ], @@ -441,15 +422,22 @@ "name": "stderr", "output_type": "stream", "text": [ - " \r" + "[Stage 18:=====================> (3 + 5) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 8.98 ms, sys: 4.45 ms, total: 13.4 ms\n", - "Wall time: 3.67 s\n" + "CPU times: user 10.6 ms, sys: 4.83 ms, total: 15.4 ms\n", + "Wall time: 4.23 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" ] } ], @@ -470,8 +458,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 8.72 ms, sys: 873 μs, total: 9.59 ms\n", - "Wall time: 154 ms\n" + "CPU times: user 6.7 ms, sys: 2.44 ms, total: 9.15 ms\n", + "Wall time: 163 ms\n" ] } ], @@ -491,8 +479,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.61 ms, sys: 3.82 ms, total: 7.43 ms\n", - "Wall time: 180 ms\n" + "CPU times: user 5.37 ms, sys: 2.73 ms, total: 8.1 ms\n", + "Wall time: 232 ms\n" ] } ], @@ -578,7 +566,7 @@ "id": "759385ac", "metadata": {}, "source": [ - "Import the utility functions from pytriton_utils.py:" + "Import the helper class from pytriton_utils.py:" ] }, { @@ -590,12 +578,7 @@ "source": [ "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "from pytriton_utils import (\n", - " use_stage_level_scheduling,\n", - " find_ports,\n", - " start_triton,\n", - " stop_triton\n", - ")" + "from pytriton_utils import TritonServerManager" ] }, { @@ -659,6 +642,7 @@ " )\n", "\n", " def _stop_triton(signum, frame):\n", + " # The server manager sends SIGTERM to stop the server; this function ensures graceful cleanup.\n", " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", " triton.stop()\n", "\n", @@ -698,82 +682,38 @@ }, { "cell_type": "markdown", - "id": "642d9f8b", + "id": "1b0371c8", "metadata": {}, "source": [ - "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + "The `TritonClusterManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n", + "- Find available ports for HTTP/gRPC/metrics\n", + "- Deploy a server on each node via stage-level scheduling\n", + "- Gracefully shutdown servers across nodes" ] }, { "cell_type": "code", "execution_count": 25, - "id": "69015ae1", + "id": "e66e8927", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Reqesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - } - ], - "source": [ - "sc = spark.sparkContext\n", - "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" - ] - }, - { - "cell_type": "markdown", - "id": "32d5e8e9", - "metadata": {}, - "source": [ - "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "012b2d60", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using ports [7000, 7001, 7002]\n" - ] - } - ], + "outputs": [], "source": [ "model_name = \"SentenceTransformer\"\n", - "ports = find_ports()\n", - "assert len(ports) == 3\n", - "print(f\"Using ports {ports}\")" + "server_manager = TritonServerManager(num_nodes=num_nodes, model_name=model_name)" ] }, { "cell_type": "code", "execution_count": null, - "id": "ea38ac6b", + "id": "040df0dd", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 28:> (0 + 1) / 1]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Triton Server PIDs:\n", - " {\n", - " \"cb4ae00-lcedt\": 2960340\n", - "}\n" + "2025-02-07 11:03:44,809 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-07 11:03:44,810 - INFO - Starting 1 servers.\n" ] }, { @@ -782,13 +722,20 @@ "text": [ " \r" ] + }, + { + "data": { + "text/plain": [ + "{'cb4ae00-lcedt': (2020631, [7000, 7001, 7002])}" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", - " ports=ports,\n", - " model_name=model_name)).collectAsMap()\n", - "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n", + "server_manager.start_servers(triton_server)" ] }, { @@ -799,27 +746,37 @@ "#### Define client function" ] }, + { + "cell_type": "markdown", + "id": "ddeadc74", + "metadata": {}, + "source": [ + "Get the hostname -> url mapping from the server manager:" + ] + }, { "cell_type": "code", - "execution_count": 28, - "id": "00d82bfe", + "execution_count": null, + "id": "c42d1578", "metadata": {}, "outputs": [], "source": [ - "url = f\"http://localhost:{ports[0]}\"" + "host_to_http_url = server_manager.host_to_http_url # or server_manager.host_to_grpc_url" ] }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 28, "id": "807dbc45", "metadata": {}, "outputs": [], "source": [ - "def triton_fn(url, model_name):\n", + "def triton_fn(model_name, host_to_url):\n", + " import socket\n", " import numpy as np\n", " from pytriton.client import ModelClient\n", "\n", + " url = host_to_url[socket.gethostname()]\n", " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", "\n", " def infer_batch(inputs):\n", @@ -835,6 +792,19 @@ " return infer_batch" ] }, + { + "cell_type": "code", + "execution_count": 31, + "id": "9c712b8f-6eb4-4fb8-9f0a-04feef847fea", + "metadata": {}, + "outputs": [], + "source": [ + "encode = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_http_url),\n", + " return_type=ArrayType(FloatType()),\n", + " input_tensor_shapes=[[1]],\n", + " batch_size=32)" + ] + }, { "cell_type": "markdown", "id": "af174106", @@ -845,7 +815,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 29, "id": "2969d502-e97b-49d6-bf80-7d177ae867cf", "metadata": {}, "outputs": [], @@ -857,7 +827,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 30, "id": "c8f1e6d6-6519-49e7-8465-4419547633b8", "metadata": {}, "outputs": [ @@ -865,7 +835,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:47:31 WARN CacheManager: Asked to cache already cached data.\n" + "25/02/04 13:40:22 WARN CacheManager: Asked to cache already cached data.\n" ] } ], @@ -885,19 +855,6 @@ { "cell_type": "code", "execution_count": 32, - "id": "9c712b8f-6eb4-4fb8-9f0a-04feef847fea", - "metadata": {}, - "outputs": [], - "source": [ - "encode = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", - " return_type=ArrayType(FloatType()),\n", - " input_tensor_shapes=[[1]],\n", - " batch_size=32)" - ] - }, - { - "cell_type": "code", - "execution_count": 33, "id": "934c1a1f-b126-45b0-9c15-265236820ad3", "metadata": {}, "outputs": [ @@ -905,8 +862,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 11.6 ms, sys: 2.74 ms, total: 14.3 ms\n", - "Wall time: 577 ms\n" + "CPU times: user 5.59 ms, sys: 5.1 ms, total: 10.7 ms\n", + "Wall time: 605 ms\n" ] } ], @@ -919,7 +876,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 33, "id": "f84cd3f6-b6a8-4142-859a-91f3c183457b", "metadata": {}, "outputs": [ @@ -927,8 +884,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 5.36 ms, sys: 3.16 ms, total: 8.52 ms\n", - "Wall time: 205 ms\n" + "CPU times: user 2.57 ms, sys: 4.36 ms, total: 6.93 ms\n", + "Wall time: 161 ms\n" ] } ], @@ -940,7 +897,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 34, "id": "921a4c01-e296-4406-be90-86f20c8c582d", "metadata": {}, "outputs": [ @@ -948,8 +905,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 5.75 ms, sys: 430 μs, total: 6.18 ms\n", - "Wall time: 180 ms\n" + "CPU times: user 7.06 ms, sys: 605 μs, total: 7.67 ms\n", + "Wall time: 191 ms\n" ] } ], @@ -961,7 +918,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 35, "id": "9f67584e-9c4e-474f-b6ea-7811b14d116e", "metadata": {}, "outputs": [ @@ -1014,22 +971,16 @@ }, { "cell_type": "code", - "execution_count": 37, - "id": "d8e5466b-b5dc-4fe1-9012-0c87cdd72962", + "execution_count": 36, + "id": "ef780e30", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Reqesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "2025-02-04 13:40:23,196 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-04 13:40:28,390 - INFO - Sucessfully stopped 1 servers. \n" ] }, { @@ -1038,20 +989,18 @@ "[True]" ] }, - "execution_count": 37, + "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", - "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" + "server_manager.stop_servers()" ] }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 37, "id": "e82b9518-da7b-4ebc-8990-c8ab909bec18", "metadata": {}, "outputs": [], @@ -1085,7 +1034,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.11.11" } }, "nbformat": 4, diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/housing_regression_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/housing_regression_torch.ipynb index 99bb06fa..ca1cc8de 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/housing_regression_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/housing_regression_torch.ipynb @@ -54,7 +54,7 @@ { "data": { "text/plain": [ - "'2.4.1+cu121'" + "'2.5.1+cu124'" ] }, "execution_count": 3, @@ -124,25 +124,35 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "d868f39d-4695-4110-91d2-6f7a09d73b93", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[tensor([[-0.5241, -0.8454, -0.2556, 0.3256, 0.8226, 0.0041, 0.7623, -1.1132],\n", - " [-0.3852, -1.1632, -0.3439, -0.0999, -0.3060, -0.0110, -0.5767, 0.0198],\n", - " [ 0.0802, 0.5054, -0.0854, -0.0261, 0.1170, 0.0557, -0.8062, 0.7485],\n", - " [-0.7215, -0.5276, -0.2073, -0.1787, -0.6115, -0.0134, 1.3756, -0.8686],\n", - " [-0.9009, -0.4481, -0.4374, -0.1855, -0.1788, 0.0477, 1.0760, -0.8537],\n", - " [ 0.6685, 1.3794, 0.2930, -0.1939, -0.8066, -0.0586, 0.9402, -1.1731],\n", - " [-0.7873, -0.9249, 0.1057, -0.0384, 0.3025, -0.0482, 0.1817, 0.2794],\n", - " [ 0.2764, 1.8562, -1.3384, -0.3592, -0.3219, 1.2067, 1.0010, -1.4127],\n", - " [-0.2856, -1.7194, -0.1692, 0.0904, 1.0177, -0.0792, -0.6938, 1.1279],\n", - " [ 1.3093, -0.2097, 0.5685, 0.0202, -0.2477, -0.0660, 0.9121, -1.4127]]),\n", - " tensor([1.8440, 2.4870, 1.5770, 1.0160, 0.6780, 3.1560, 0.7980, 2.2500, 1.2220,\n", - " 5.0000])]" + "[tensor([[ 6.5799e-01, 4.2594e-01, -1.4755e-01, -2.3638e-01, -4.0221e-01,\n", + " -5.6793e-02, 8.8868e-01, -1.3528e+00],\n", + " [ 6.7288e-01, -1.0043e+00, 5.7486e-01, -1.6537e-01, -3.3422e-01,\n", + " -6.4971e-02, -1.2790e+00, 1.2327e+00],\n", + " [-1.1616e-01, 2.8646e-02, -1.7830e-01, -2.3817e-01, -6.7154e-01,\n", + " -3.6429e-02, -1.3258e+00, 1.2726e+00],\n", + " [-3.2513e-01, -6.8648e-01, -3.4226e-01, -8.2805e-02, 5.1239e+00,\n", + " 2.6689e-02, -7.7338e-01, 8.3340e-01],\n", + " [ 1.0892e-01, -1.2427e+00, 2.7819e-01, -8.7150e-02, 3.0158e-01,\n", + " -1.8564e-02, -1.1245e+00, 1.1628e+00],\n", + " [-8.6416e-02, 5.8485e-01, -7.8085e-02, 8.1655e-02, -6.7154e-01,\n", + " -1.6053e-02, -3.4733e-01, 1.2577e+00],\n", + " [-1.2463e-01, 1.0810e-01, 2.6662e-01, -1.0883e-01, 3.4839e-01,\n", + " -2.3125e-02, -7.7338e-01, 1.3325e+00],\n", + " [-9.2662e-01, -1.6400e+00, -2.4824e-01, 6.0041e-01, 6.3361e-01,\n", + " -1.0926e-01, -8.8574e-01, 1.2826e+00],\n", + " [ 2.0038e+00, -6.0702e-01, 8.4770e-01, -2.1254e-01, 1.3745e+00,\n", + " -5.0489e-03, -6.5165e-01, 2.5441e-01],\n", + " [-3.9250e-01, 1.0616e+00, -1.8614e-01, -1.7073e-01, -3.8543e-01,\n", + " -8.1186e-02, 1.0806e+00, -1.3827e+00]]),\n", + " tensor([3.1090, 1.8430, 1.6890, 1.8670, 1.9600, 0.6200, 0.9860, 0.9440, 3.9120,\n", + " 1.4390])]" ] }, "execution_count": 7, @@ -186,7 +196,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "15cff2b4-9d23-4d2b-808a-a5edb8eda135", "metadata": { "scrolled": true, @@ -226,65 +236,65 @@ "output_type": "stream", "text": [ "Starting epoch 1\n", - "Loss after mini-batch 1: 0.003\n", - "Loss after mini-batch 201: 0.709\n", - "Loss after mini-batch 401: 0.505\n", - "Loss after mini-batch 601: 0.379\n", - "Loss after mini-batch 801: 0.305\n", - "Loss after mini-batch 1001: 0.269\n", - "Loss after mini-batch 1201: 0.257\n", - "Loss after mini-batch 1401: 0.227\n", - "Loss after mini-batch 1601: 0.214\n", - "Loss after mini-batch 1801: 0.223\n", - "Loss after mini-batch 2001: 0.223\n", + "Loss after mini-batch 1: 0.004\n", + "Loss after mini-batch 201: 0.701\n", + "Loss after mini-batch 401: 0.463\n", + "Loss after mini-batch 601: 0.329\n", + "Loss after mini-batch 801: 0.285\n", + "Loss after mini-batch 1001: 0.253\n", + "Loss after mini-batch 1201: 0.247\n", + "Loss after mini-batch 1401: 0.234\n", + "Loss after mini-batch 1601: 0.232\n", + "Loss after mini-batch 1801: 0.217\n", + "Loss after mini-batch 2001: 0.211\n", "Starting epoch 2\n", "Loss after mini-batch 1: 0.001\n", - "Loss after mini-batch 201: 0.214\n", - "Loss after mini-batch 401: 0.206\n", - "Loss after mini-batch 601: 0.200\n", - "Loss after mini-batch 801: 0.199\n", - "Loss after mini-batch 1001: 0.203\n", - "Loss after mini-batch 1201: 0.197\n", - "Loss after mini-batch 1401: 0.197\n", - "Loss after mini-batch 1601: 0.197\n", + "Loss after mini-batch 201: 0.205\n", + "Loss after mini-batch 401: 0.212\n", + "Loss after mini-batch 601: 0.206\n", + "Loss after mini-batch 801: 0.205\n", + "Loss after mini-batch 1001: 0.202\n", + "Loss after mini-batch 1201: 0.202\n", + "Loss after mini-batch 1401: 0.204\n", + "Loss after mini-batch 1601: 0.198\n", "Loss after mini-batch 1801: 0.188\n", - "Loss after mini-batch 2001: 0.193\n", + "Loss after mini-batch 2001: 0.188\n", "Starting epoch 3\n", "Loss after mini-batch 1: 0.001\n", - "Loss after mini-batch 201: 0.182\n", - "Loss after mini-batch 401: 0.186\n", - "Loss after mini-batch 601: 0.195\n", - "Loss after mini-batch 801: 0.195\n", - "Loss after mini-batch 1001: 0.190\n", - "Loss after mini-batch 1201: 0.186\n", - "Loss after mini-batch 1401: 0.185\n", - "Loss after mini-batch 1601: 0.185\n", - "Loss after mini-batch 1801: 0.190\n", - "Loss after mini-batch 2001: 0.186\n", + "Loss after mini-batch 201: 0.197\n", + "Loss after mini-batch 401: 0.193\n", + "Loss after mini-batch 601: 0.196\n", + "Loss after mini-batch 801: 0.189\n", + "Loss after mini-batch 1001: 0.183\n", + "Loss after mini-batch 1201: 0.191\n", + "Loss after mini-batch 1401: 0.193\n", + "Loss after mini-batch 1601: 0.181\n", + "Loss after mini-batch 1801: 0.185\n", + "Loss after mini-batch 2001: 0.181\n", "Starting epoch 4\n", "Loss after mini-batch 1: 0.001\n", - "Loss after mini-batch 201: 0.179\n", - "Loss after mini-batch 401: 0.182\n", - "Loss after mini-batch 601: 0.187\n", - "Loss after mini-batch 801: 0.183\n", - "Loss after mini-batch 1001: 0.182\n", - "Loss after mini-batch 1201: 0.187\n", - "Loss after mini-batch 1401: 0.179\n", - "Loss after mini-batch 1601: 0.183\n", - "Loss after mini-batch 1801: 0.177\n", - "Loss after mini-batch 2001: 0.184\n", - "Starting epoch 5\n", - "Loss after mini-batch 1: 0.001\n", - "Loss after mini-batch 201: 0.174\n", + "Loss after mini-batch 201: 0.190\n", "Loss after mini-batch 401: 0.181\n", - "Loss after mini-batch 601: 0.183\n", - "Loss after mini-batch 801: 0.176\n", - "Loss after mini-batch 1001: 0.179\n", - "Loss after mini-batch 1201: 0.176\n", - "Loss after mini-batch 1401: 0.183\n", + "Loss after mini-batch 601: 0.189\n", + "Loss after mini-batch 801: 0.180\n", + "Loss after mini-batch 1001: 0.184\n", + "Loss after mini-batch 1201: 0.180\n", + "Loss after mini-batch 1401: 0.180\n", + "Loss after mini-batch 1601: 0.184\n", + "Loss after mini-batch 1801: 0.186\n", + "Loss after mini-batch 2001: 0.179\n", + "Starting epoch 5\n", + "Loss after mini-batch 1: 0.000\n", + "Loss after mini-batch 201: 0.181\n", + "Loss after mini-batch 401: 0.177\n", + "Loss after mini-batch 601: 0.185\n", + "Loss after mini-batch 801: 0.179\n", + "Loss after mini-batch 1001: 0.178\n", + "Loss after mini-batch 1201: 0.173\n", + "Loss after mini-batch 1401: 0.185\n", "Loss after mini-batch 1601: 0.177\n", - "Loss after mini-batch 1801: 0.183\n", - "Loss after mini-batch 2001: 0.172\n", + "Loss after mini-batch 1801: 0.181\n", + "Loss after mini-batch 2001: 0.178\n", "Training process has finished.\n" ] } @@ -372,7 +382,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "20fedb5d-c59e-4b0b-ba91-3dd15df1f09e", "metadata": {}, "outputs": [ @@ -432,7 +442,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "d46af47e-db7e-42ee-9bd3-6e7d93850be3", "metadata": {}, "outputs": [ @@ -446,16 +456,16 @@ { "data": { "text/plain": [ - "tensor([[2.0423],\n", - " [1.9258],\n", - " [2.8864],\n", - " [3.2128],\n", - " [2.8639],\n", - " [1.0359],\n", - " [2.8652],\n", - " [1.5528],\n", - " [1.7592],\n", - " [0.7497]], device='cuda:0', grad_fn=)" + "tensor([[2.3652],\n", + " [1.8444],\n", + " [2.4587],\n", + " [3.1243],\n", + " [2.2726],\n", + " [2.1818],\n", + " [1.5222],\n", + " [0.5554],\n", + " [2.2508],\n", + " [3.5971]], device='cuda:0', grad_fn=)" ] }, "execution_count": 15, @@ -484,8 +494,8 @@ { "data": { "text/plain": [ - "tensor([2.2030, 2.1590, 1.9400, 3.4310, 2.4480, 1.1410, 2.8780, 0.8310, 1.6530,\n", - " 1.2380])" + "tensor([2.7370, 2.2110, 2.5360, 2.6330, 1.6540, 2.3360, 1.4600, 0.6590, 2.6380,\n", + " 3.6220])" ] }, "execution_count": 16, @@ -518,7 +528,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "0cda8ec8-644e-4888-bfa0-b79425ece7c3", "metadata": { "tags": [] @@ -534,8 +544,8 @@ { "data": { "text/plain": [ - "tensor([2.0423, 1.9258, 2.8864, 3.2128, 2.8639, 1.0359, 2.8652, 1.5528, 1.7592,\n", - " 0.7497], device='cuda:0', grad_fn=)" + "tensor([2.3652, 1.8444, 2.4587, 3.1243, 2.2726, 2.1818, 1.5222, 0.5554, 2.2508,\n", + " 3.5971], device='cuda:0', grad_fn=)" ] }, "execution_count": 18, @@ -572,15 +582,7 @@ "execution_count": 19, "id": "9ffb27fc", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:torch_tensorrt.dynamo.conversion.aten_ops_converters:Unable to import quantization op. Please install modelopt library (https://github.com/NVIDIA/TensorRT-Model-Optimizer?tab=readme-ov-file#installation) to add support for compiling quantized models\n" - ] - } - ], + "outputs": [], "source": [ "import torch_tensorrt as trt\n", "import time" @@ -629,70 +631,25 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO:torch_tensorrt.dynamo.utils:Using Default Torch-TRT Runtime (as requested by user)\n", - "INFO:torch_tensorrt.dynamo.utils:Device not specified, using Torch default current device - cuda:0. If this is incorrect, please specify an input device, via the device keyword.\n", - "INFO:torch_tensorrt.dynamo.utils:Compilation Settings: CompilationSettings(enabled_precisions={}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, refit=False, engine_capability=, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/timing_cache-1738007491.6462495.bin')\n", - "\n", - "WARNING:torch_tensorrt.dynamo._compiler:Node _param_constant1 of op type get_attr does not have metadata. This could sometimes lead to undefined behavior.\n", - "WARNING:torch_tensorrt.dynamo._compiler:Some nodes do not have metadata (shape and dtype information). This could lead to problems sometimes if the graph has PyTorch and TensorRT segments.\n", - "INFO:torch_tensorrt.dynamo._compiler:Partitioning the graph via the fast partitioner\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init CUDA: CPU +2, GPU +0, now: CPU 581, GPU 2466 (MiB)\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init builder kernel library: CPU +1635, GPU +288, now: CPU 2363, GPU 2754 (MiB)\n" + "WARNING:torch_tensorrt.dynamo._compiler:Node linear_default of op type call_function does not have metadata. This could sometimes lead to undefined behavior.\n", + "WARNING:torch_tensorrt.dynamo._compiler:Some nodes do not have metadata (shape and dtype information). This could lead to problems sometimes if the graph has PyTorch and TensorRT segments.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Predictions:\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/torch_tensorrt/dynamo/conversion/impl/activation/base.py:40: DeprecationWarning: Use Deprecated in TensorRT 10.1. Superseded by explicit quantization. instead.\n", - " if input_val.dynamic_range is not None and dyn_range_fn is not None:\n", - "\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT INetwork construction elapsed time: 0:00:00.008361\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Global timing cache in use. Profiling results in this builder pass will be stored.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Detected 1 inputs and 1 output network tensors.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Host Persistent Memory: 22240\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Device Persistent Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Scratch Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Started assigning block shifts. This will take 10 steps to complete.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Algorithm ShiftNTopDown took 0.147039ms to assign 4 blocks to 10 nodes requiring 7168 bytes.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Activation Memory: 6656\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Weights Memory: 11648\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Engine generation completed in 1.60138 seconds.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage of TRT CPU/GPU memory allocators: CPU 0 MiB, GPU 1 MiB\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage during Engine building and serialization: CPU: 4106 MiB\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:Build TRT engine elapsed time: 0:00:01.606208\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT Engine uses: 390420 bytes of Memory\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 26 bytes of code generator cache.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 52 timing cache entries\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "tensor([[2.0423],\n", - " [1.9258],\n", - " [2.8864],\n", - " [3.2128],\n", - " [2.8639],\n", - " [1.0359],\n", - " [2.8652],\n", - " [1.5528],\n", - " [1.7592],\n", - " [0.7497]], device='cuda:0')\n" + "Predictions:\n", + "tensor([[2.3652],\n", + " [1.8444],\n", + " [2.4587],\n", + " [3.1243],\n", + " [2.2726],\n", + " [2.1818],\n", + " [1.5222],\n", + " [0.5554],\n", + " [2.2508],\n", + " [3.5971]], device='cuda:0')\n" ] } ], @@ -720,39 +677,7 @@ "execution_count": 23, "id": "bf36a50d", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:torch_tensorrt.dynamo._compiler:Compilation Settings: CompilationSettings(enabled_precisions={}, debug=False, workspace_size=1073741824, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, refit=False, engine_capability=, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/timing_cache-1738007491.6462495.bin')\n", - "\n", - "INFO:torch_tensorrt.dynamo._compiler:Partitioning the graph via the fast partitioner\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init CUDA: CPU +0, GPU +0, now: CPU 892, GPU 2468 (MiB)\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init builder kernel library: CPU +1633, GPU +286, now: CPU 2525, GPU 2754 (MiB)\n", - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/torch_tensorrt/dynamo/conversion/impl/activation/base.py:40: DeprecationWarning: Use Deprecated in TensorRT 10.1. Superseded by explicit quantization. instead.\n", - " if input_val.dynamic_range is not None and dyn_range_fn is not None:\n", - "\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT INetwork construction elapsed time: 0:00:00.002784\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Global timing cache in use. Profiling results in this builder pass will be stored.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Detected 1 inputs and 1 output network tensors.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Host Persistent Memory: 18368\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Device Persistent Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Scratch Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Started assigning block shifts. This will take 6 steps to complete.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Algorithm ShiftNTopDown took 0.237457ms to assign 3 blocks to 6 nodes requiring 25088 bytes.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Activation Memory: 24576\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Weights Memory: 12292\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Engine generation completed in 1.33069 seconds.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage of TRT CPU/GPU memory allocators: CPU 0 MiB, GPU 5 MiB\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage during Engine building and serialization: CPU: 4131 MiB\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:Build TRT engine elapsed time: 0:00:01.332388\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT Engine uses: 212892 bytes of Memory\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 26 bytes of code generator cache.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 100 timing cache entries\n" - ] - } - ], + "outputs": [], "source": [ "example_inputs = (torch.randn((10, 8), dtype=torch.float).to(\"cuda\"),)\n", "\n", @@ -799,16 +724,16 @@ "output_type": "stream", "text": [ "Predictions:\n", - "tensor([[2.0425],\n", - " [1.9257],\n", - " [2.8869],\n", - " [3.2127],\n", - " [2.8640],\n", - " [1.0362],\n", - " [2.8658],\n", - " [1.5528],\n", - " [1.7586],\n", - " [0.7496]], device='cuda:0')\n" + "tensor([[2.3653],\n", + " [1.8443],\n", + " [2.4586],\n", + " [3.1242],\n", + " [2.2725],\n", + " [2.1815],\n", + " [1.5221],\n", + " [0.5556],\n", + " [2.2508],\n", + " [3.5971]], device='cuda:0')\n" ] } ], @@ -830,7 +755,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "id": "49f72c14", "metadata": {}, "outputs": [ @@ -868,7 +793,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "876fea4a", "metadata": {}, "outputs": [ @@ -895,20 +820,10 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "bb71dd36", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/pyspark/broadcast.py:38: DeprecationWarning: typing.io is deprecated, import directly from typing instead. typing.io will be removed in Python 3.12.\n", - " from typing.io import BinaryIO # type: ignore[import]\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "from pyspark.sql.functions import col, struct, pandas_udf, array\n", "from pyspark.ml.functions import predict_batch_udf\n", @@ -960,12 +875,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:51:37 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "25/01/27 19:51:37 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "25/02/04 13:46:28 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/02/04 13:46:28 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "25/01/27 19:51:37 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", - "25/01/27 19:51:38 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + "25/02/04 13:46:28 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" ] } ], @@ -1042,16 +956,6 @@ "id": "881afee9", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/pyspark/sql/pandas/serializers.py:229: DeprecationWarning: is_categorical_dtype is deprecated and will be removed in a future version. Use isinstance(dtype, pd.CategoricalDtype) instead\n", - " elif is_categorical_dtype(s.dtype):\n", - "\n", - "[Stage 0:> (0 + 8) / 8]\r" - ] - }, { "name": "stdout", "output_type": "stream", @@ -1083,13 +987,6 @@ "only showing top 20 rows\n", "\n" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] } ], "source": [ @@ -1283,15 +1180,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 7:=======> (1 + 7) / 8]\r" + "[Stage 7:==============> (2 + 6) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 29.2 ms, sys: 4.38 ms, total: 33.6 ms\n", - "Wall time: 8.23 s\n" + "CPU times: user 30.4 ms, sys: 13.1 ms, total: 43.5 ms\n", + "Wall time: 10.1 s\n" ] }, { @@ -1320,8 +1217,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 30.7 ms, sys: 6.15 ms, total: 36.8 ms\n", - "Wall time: 309 ms\n" + "CPU times: user 31.6 ms, sys: 7.39 ms, total: 39 ms\n", + "Wall time: 263 ms\n" ] } ], @@ -1341,8 +1238,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 27.3 ms, sys: 4.6 ms, total: 31.9 ms\n", - "Wall time: 276 ms\n" + "CPU times: user 28.7 ms, sys: 6.67 ms, total: 35.4 ms\n", + "Wall time: 296 ms\n" ] } ], @@ -1365,26 +1262,26 @@ "+------------+------------+-----------+------------+-----------+------------+----------+------------+---------+\n", "| MedInc| HouseAge| AveRooms| AveBedrms| Population| AveOccup| Latitude| Longitude| preds|\n", "+------------+------------+-----------+------------+-----------+------------+----------+------------+---------+\n", - "| 0.20909257| -1.1632254| 0.38946992| 0.04609274| -0.9806099| -0.07099328|0.61245227|-0.020113053|1.4441836|\n", - "|-0.098627955| 0.34647804| 0.27216315| -0.0129226| -0.6953838| -0.05380849| 1.0665938| -1.2479742|1.7245855|\n", - "| -0.66006273| 1.0616008|-0.55292207| -0.48945764|-0.13641118| 0.028952759| 1.1040496| -1.3827378|1.3524103|\n", - "| 0.08218294| 0.5848523|-0.13912922| -0.14707813|-0.19116047| -0.07136432|0.96827507| -1.3028787|2.3009148|\n", - "| 0.0784456| -1.4810578| 0.57265776| 0.32067496| 1.0345173|-0.024157424| 1.4411427| -0.52423614| 1.272077|\n", - "| -0.82318723| -0.36864465| 0.07829511| -0.1808107|-0.67242444|-0.061470542| 1.9374212| -1.0083897|0.6968852|\n", - "| 0.59671736| 0.5848523| 0.19346413| -0.1371872|-0.19645879| 0.009964322|0.96827507| -1.2928978|2.6057932|\n", - "| -0.9612035| -1.5605159|-0.56329846| 0.027148023|-0.71127874| -0.08471591| 0.5328614| -0.13990337|1.0139045|\n", - "| -0.74344087| -1.2426835| 0.27282518| 0.4037246| -0.9841421| -0.05610115| 1.2257773| -0.42940006|0.9931088|\n", - "| 0.9784464| -0.2891866| 0.24374022| -0.24670053| 0.28922042| -0.01102468| 1.1087307| -1.2280084| 2.64501|\n", - "| -0.5070446| -1.0043093|-0.78254056|0.0122275995| 2.8465424|-0.060435444| 0.8980464| -1.2080427|2.1324868|\n", - "| -0.18690155| 1.2205169|0.015323491| 0.12183313|-0.41015765| 0.04452552| 1.010412| -1.3228445|1.9459988|\n", - "| -1.2551856| 1.6178073| -0.3341509|-0.060125165| -0.7554314| -0.08777025| 1.0291398| -1.3477987|1.2558049|\n", - "| 4.9607058| -1.9578062| 1.4854684| -0.03948475| 2.1833694|0.0029250523| 1.024457| -1.1581304|5.8959594|\n", - "| 0.73652315| -1.6399739| 0.7913185| -0.05238397| 1.67738| 0.01944797| 1.0993668| -1.1331724|2.0677836|\n", - "| -0.505834| 0.18756187|-0.47093546| -0.24297306|-0.60619545| -0.10791535| 0.977639| -1.2879055|1.7652202|\n", - "| -0.88477343|-0.050812364| -0.6318951| -0.15244243| -0.5258376| -0.15618815| 0.9823201| -1.2879055|1.5843444|\n", - "| -0.42840376| 0.9821427| -0.2266495| -0.36083496| -0.6883194| -0.08552282| 0.5328614| -0.12493005|0.9752624|\n", - "| 0.9369153| -1.4810578| 0.6722208|-0.121177554| 0.3996021| 0.01291408| 1.1040496| -1.1082181|2.2553492|\n", - "| -0.80702734| -0.92485124|-0.26602685| -0.1560743| 1.4398388| -0.09314839|0.55627036| -0.09498342|1.0328484|\n", + "| 0.20909257| -1.1632254| 0.38946992| 0.04609274| -0.9806099| -0.07099328|0.61245227|-0.020113053|1.3746364|\n", + "|-0.098627955| 0.34647804| 0.27216315| -0.0129226| -0.6953838| -0.05380849| 1.0665938| -1.2479742|1.8087528|\n", + "| -0.66006273| 1.0616008|-0.55292207| -0.48945764|-0.13641118| 0.028952759| 1.1040496| -1.3827378|1.4245079|\n", + "| 0.08218294| 0.5848523|-0.13912922| -0.14707813|-0.19116047| -0.07136432|0.96827507| -1.3028787|2.3895802|\n", + "| 0.0784456| -1.4810578| 0.57265776| 0.32067496| 1.0345173|-0.024157424| 1.4411427| -0.52423614|1.3616933|\n", + "| -0.82318723| -0.36864465| 0.07829511| -0.1808107|-0.67242444|-0.061470542| 1.9374212| -1.0083897|0.7539238|\n", + "| 0.59671736| 0.5848523| 0.19346413| -0.1371872|-0.19645879| 0.009964322|0.96827507| -1.2928978|2.6816423|\n", + "| -0.9612035| -1.5605159|-0.56329846| 0.027148023|-0.71127874| -0.08471591| 0.5328614| -0.13990337|1.1731354|\n", + "| -0.74344087| -1.2426835| 0.27282518| 0.4037246| -0.9841421| -0.05610115| 1.2257773| -0.42940006|1.0198532|\n", + "| 0.9784464| -0.2891866| 0.24374022| -0.24670053| 0.28922042| -0.01102468| 1.1087307| -1.2280084| 2.708211|\n", + "| -0.5070446| -1.0043093|-0.78254056|0.0122275995| 2.8465424|-0.060435444| 0.8980464| -1.2080427|2.0327075|\n", + "| -0.18690155| 1.2205169|0.015323491| 0.12183313|-0.41015765| 0.04452552| 1.010412| -1.3228445|1.9909104|\n", + "| -1.2551856| 1.6178073| -0.3341509|-0.060125165| -0.7554314| -0.08777025| 1.0291398| -1.3477987|1.2702764|\n", + "| 4.9607058| -1.9578062| 1.4854684| -0.03948475| 2.1833694|0.0029250523| 1.024457| -1.1581304| 5.975229|\n", + "| 0.73652315| -1.6399739| 0.7913185| -0.05238397| 1.67738| 0.01944797| 1.0993668| -1.1331724|1.9309721|\n", + "| -0.505834| 0.18756187|-0.47093546| -0.24297306|-0.60619545| -0.10791535| 0.977639| -1.2879055|1.7610806|\n", + "| -0.88477343|-0.050812364| -0.6318951| -0.15244243| -0.5258376| -0.15618815| 0.9823201| -1.2879055| 1.655031|\n", + "| -0.42840376| 0.9821427| -0.2266495| -0.36083496| -0.6883194| -0.08552282| 0.5328614| -0.12493005|1.1175063|\n", + "| 0.9369153| -1.4810578| 0.6722208|-0.121177554| 0.3996021| 0.01291408| 1.1040496| -1.1082181|2.1779811|\n", + "| -0.80702734| -0.92485124|-0.26602685| -0.1560743| 1.4398388| -0.09314839|0.55627036| -0.09498342|0.9102398|\n", "+------------+------------+-----------+------------+-----------+------------+----------+------------+---------+\n", "only showing top 20 rows\n", "\n" @@ -1439,7 +1336,7 @@ "id": "1b77dc96", "metadata": {}, "source": [ - "Import the utility functions from pytriton_utils.py:" + "Import the helper class from pytriton_utils.py:" ] }, { @@ -1451,12 +1348,7 @@ "source": [ "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "from pytriton_utils import (\n", - " use_stage_level_scheduling,\n", - " find_ports,\n", - " start_triton,\n", - " stop_triton\n", - ")" + "from pytriton_utils import TritonServerManager" ] }, { @@ -1530,6 +1422,7 @@ " )\n", "\n", " def _stop_triton(signum, frame):\n", + " # The server manager sends SIGTERM to stop the server; this function ensures graceful cleanup.\n", " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", " triton.stop()\n", "\n", @@ -1569,82 +1462,38 @@ }, { "cell_type": "markdown", - "id": "8bba4f54", + "id": "6d6b7143", "metadata": {}, "source": [ - "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + "The `TritonClusterManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n", + "- Find available ports for HTTP/gRPC/metrics\n", + "- Deploy a server on each node via stage-level scheduling\n", + "- Gracefully shutdown servers across nodes" ] }, { "cell_type": "code", - "execution_count": 52, - "id": "bca2f712", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - } - ], - "source": [ - "sc = spark.sparkContext\n", - "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" - ] - }, - { - "cell_type": "markdown", - "id": "fd76c554", - "metadata": {}, - "source": [ - "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "id": "ba954d7d", + "execution_count": 54, + "id": "2fb22db8", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using ports [7000, 7001, 7002]\n" - ] - } - ], + "outputs": [], "source": [ "model_name = \"HousingModel\"\n", - "ports = find_ports()\n", - "assert len(ports) == 3\n", - "print(f\"Using ports {ports}\")" + "server_manager = TritonServerManager(num_nodes=num_nodes, model_name=model_name, model_path=model_path)" ] }, { "cell_type": "code", - "execution_count": 54, - "id": "5358d5b6", + "execution_count": null, + "id": "e067aa14", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 11:> (0 + 1) / 1]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Triton Server PIDs:\n", - " {\n", - " \"cb4ae00-lcedt\": 2964321\n", - "}\n" + "2025-02-07 11:03:44,809 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-07 11:03:44,810 - INFO - Starting 1 servers.\n" ] }, { @@ -1653,14 +1502,20 @@ "text": [ " \r" ] + }, + { + "data": { + "text/plain": [ + "{'cb4ae00-lcedt': (2020631, [7000, 7001, 7002])}" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", - " ports=ports,\n", - " model_name=model_name,\n", - " model_path=model_path)).collectAsMap()\n", - "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n", + "server_manager.start_servers(triton_server)" ] }, { @@ -1671,26 +1526,44 @@ "#### Define client function" ] }, + { + "cell_type": "markdown", + "id": "d4ac45ef", + "metadata": {}, + "source": [ + "Get the hostname -> url mapping from the server manager:" + ] + }, { "cell_type": "code", - "execution_count": 55, - "id": "3812e5ae", + "execution_count": null, + "id": "92760dac", "metadata": {}, "outputs": [], "source": [ - "url = f\"http://localhost:{ports[0]}\"" + "host_to_http_url = server_manager.host_to_http_url # or server_manager.host_to_grpc_url" + ] + }, + { + "cell_type": "markdown", + "id": "122ebe7c", + "metadata": {}, + "source": [ + "Define the Triton inference function, which returns a predict function for batch inference through the server:" ] }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 57, "id": "1ae91c54", "metadata": {}, "outputs": [], "source": [ - "def triton_fn(url, model_name):\n", + "def triton_fn(model_name, host_to_url):\n", + " import socket\n", " from pytriton.client import ModelClient\n", "\n", + " url = host_to_url[socket.gethostname()]\n", " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", "\n", " def infer_batch(inputs):\n", @@ -1702,49 +1575,49 @@ ] }, { - "cell_type": "markdown", - "id": "20b8514e-01de-481f-86aa-75afd99bcc7c", + "cell_type": "code", + "execution_count": 60, + "id": "d3e64fda-117b-4810-a9a2-dd498239496f", "metadata": {}, + "outputs": [], "source": [ - "### Run Inference" + "regress = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_http_url),\n", + " input_tensor_shapes=[[8]],\n", + " return_type=FloatType(),\n", + " batch_size=50)" ] }, { - "cell_type": "code", - "execution_count": 57, - "id": "5eae04bc-75ca-421a-87c8-ac507ce1f2f5", + "cell_type": "markdown", + "id": "20b8514e-01de-481f-86aa-75afd99bcc7c", "metadata": {}, - "outputs": [], "source": [ - "df = spark.read.parquet(data_path)" + "### Run Inference" ] }, { "cell_type": "code", "execution_count": 58, - "id": "b350bd8e-9b8f-4511-9ddf-76d917b21b5f", + "id": "5eae04bc-75ca-421a-87c8-ac507ce1f2f5", "metadata": {}, "outputs": [], "source": [ - "columns = df.columns" + "df = spark.read.parquet(data_path)" ] }, { "cell_type": "code", "execution_count": 59, - "id": "d3e64fda-117b-4810-a9a2-dd498239496f", + "id": "b350bd8e-9b8f-4511-9ddf-76d917b21b5f", "metadata": {}, "outputs": [], "source": [ - "regress = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", - " input_tensor_shapes=[[8]],\n", - " return_type=FloatType(),\n", - " batch_size=50)" + "columns = df.columns" ] }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 61, "id": "a24149a5-3adc-4089-8769-13cf1e44547a", "metadata": {}, "outputs": [ @@ -1752,15 +1625,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 13:====================================> (5 + 3) / 8]\r" + "[Stage 16:> (0 + 8) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 21.1 ms, sys: 8.1 ms, total: 29.2 ms\n", - "Wall time: 1.01 s\n" + "CPU times: user 25.8 ms, sys: 6.21 ms, total: 32.1 ms\n", + "Wall time: 2.37 s\n" ] }, { @@ -1780,7 +1653,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 62, "id": "df2ce39f-30af-491a-8472-800fb1ce8458", "metadata": {}, "outputs": [ @@ -1788,15 +1661,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 14:> (0 + 8) / 8]\r" + "[Stage 17:> (0 + 8) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 23.7 ms, sys: 4.15 ms, total: 27.8 ms\n", - "Wall time: 973 ms\n" + "CPU times: user 171 ms, sys: 3.76 ms, total: 174 ms\n", + "Wall time: 2.5 s\n" ] }, { @@ -1815,7 +1688,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 63, "id": "ca6f3eaa-9569-45d0-88bf-9aa0757e1ecb", "metadata": {}, "outputs": [ @@ -1823,15 +1696,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 15:> (0 + 8) / 8]\r" + "[Stage 18:> (0 + 8) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 34.3 ms, sys: 6.47 ms, total: 40.7 ms\n", - "Wall time: 1.49 s\n" + "CPU times: user 24.4 ms, sys: 4.83 ms, total: 29.2 ms\n", + "Wall time: 1.97 s\n" ] }, { @@ -1850,7 +1723,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 64, "id": "b79c62c8-e1e8-4467-8aef-8939c31833b8", "metadata": {}, "outputs": [ @@ -1858,30 +1731,30 @@ "name": "stdout", "output_type": "stream", "text": [ - "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", - "| MedInc| HouseAge| AveRooms| AveBedrms| Population| AveOccup| Latitude| Longitude| preds|\n", - "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", - "| 0.20909257| -1.1632254| 0.38946992| 0.04609274| -0.9806099| -0.07099328|0.61245227|-0.020113053| 1.4441836|\n", - "|-0.098627955| 0.34647804| 0.27216315| -0.0129226| -0.6953838| -0.05380849| 1.0665938| -1.2479742| 1.7245854|\n", - "| -0.66006273| 1.0616008|-0.55292207| -0.48945764|-0.13641118| 0.028952759| 1.1040496| -1.3827378| 1.3524104|\n", - "| 0.08218294| 0.5848523|-0.13912922| -0.14707813|-0.19116047| -0.07136432|0.96827507| -1.3028787| 2.300915|\n", - "| 0.0784456| -1.4810578| 0.57265776| 0.32067496| 1.0345173|-0.024157424| 1.4411427| -0.52423614| 1.2720771|\n", - "| -0.82318723| -0.36864465| 0.07829511| -0.1808107|-0.67242444|-0.061470542| 1.9374212| -1.0083897| 0.6968852|\n", - "| 0.59671736| 0.5848523| 0.19346413| -0.1371872|-0.19645879| 0.009964322|0.96827507| -1.2928978| 2.6057932|\n", - "| -0.9612035| -1.5605159|-0.56329846| 0.027148023|-0.71127874| -0.08471591| 0.5328614| -0.13990337| 1.0139045|\n", - "| -0.74344087| -1.2426835| 0.27282518| 0.4037246| -0.9841421| -0.05610115| 1.2257773| -0.42940006|0.99310875|\n", - "| 0.9784464| -0.2891866| 0.24374022| -0.24670053| 0.28922042| -0.01102468| 1.1087307| -1.2280084| 2.6450102|\n", - "| -0.5070446| -1.0043093|-0.78254056|0.0122275995| 2.8465424|-0.060435444| 0.8980464| -1.2080427| 2.1324868|\n", - "| -0.18690155| 1.2205169|0.015323491| 0.12183313|-0.41015765| 0.04452552| 1.010412| -1.3228445| 1.945999|\n", - "| -1.2551856| 1.6178073| -0.3341509|-0.060125165| -0.7554314| -0.08777025| 1.0291398| -1.3477987| 1.255805|\n", - "| 4.9607058| -1.9578062| 1.4854684| -0.03948475| 2.1833694|0.0029250523| 1.024457| -1.1581304| 5.8959594|\n", - "| 0.73652315| -1.6399739| 0.7913185| -0.05238397| 1.67738| 0.01944797| 1.0993668| -1.1331724| 2.0677836|\n", - "| -0.505834| 0.18756187|-0.47093546| -0.24297306|-0.60619545| -0.10791535| 0.977639| -1.2879055| 1.7652202|\n", - "| -0.88477343|-0.050812364| -0.6318951| -0.15244243| -0.5258376| -0.15618815| 0.9823201| -1.2879055| 1.5843443|\n", - "| -0.42840376| 0.9821427| -0.2266495| -0.36083496| -0.6883194| -0.08552282| 0.5328614| -0.12493005|0.97526246|\n", - "| 0.9369153| -1.4810578| 0.6722208|-0.121177554| 0.3996021| 0.01291408| 1.1040496| -1.1082181| 2.2553492|\n", - "| -0.80702734| -0.92485124|-0.26602685| -0.1560743| 1.4398388| -0.09314839|0.55627036| -0.09498342| 1.0328484|\n", - "+------------+------------+-----------+------------+-----------+------------+----------+------------+----------+\n", + "+------------+------------+-----------+------------+-----------+------------+----------+------------+---------+\n", + "| MedInc| HouseAge| AveRooms| AveBedrms| Population| AveOccup| Latitude| Longitude| preds|\n", + "+------------+------------+-----------+------------+-----------+------------+----------+------------+---------+\n", + "| 0.20909257| -1.1632254| 0.38946992| 0.04609274| -0.9806099| -0.07099328|0.61245227|-0.020113053|1.3746364|\n", + "|-0.098627955| 0.34647804| 0.27216315| -0.0129226| -0.6953838| -0.05380849| 1.0665938| -1.2479742|1.8087528|\n", + "| -0.66006273| 1.0616008|-0.55292207| -0.48945764|-0.13641118| 0.028952759| 1.1040496| -1.3827378|1.4245079|\n", + "| 0.08218294| 0.5848523|-0.13912922| -0.14707813|-0.19116047| -0.07136432|0.96827507| -1.3028787|2.3895802|\n", + "| 0.0784456| -1.4810578| 0.57265776| 0.32067496| 1.0345173|-0.024157424| 1.4411427| -0.52423614|1.3616933|\n", + "| -0.82318723| -0.36864465| 0.07829511| -0.1808107|-0.67242444|-0.061470542| 1.9374212| -1.0083897|0.7539238|\n", + "| 0.59671736| 0.5848523| 0.19346413| -0.1371872|-0.19645879| 0.009964322|0.96827507| -1.2928978|2.6816423|\n", + "| -0.9612035| -1.5605159|-0.56329846| 0.027148023|-0.71127874| -0.08471591| 0.5328614| -0.13990337|1.1731354|\n", + "| -0.74344087| -1.2426835| 0.27282518| 0.4037246| -0.9841421| -0.05610115| 1.2257773| -0.42940006|1.0198532|\n", + "| 0.9784464| -0.2891866| 0.24374022| -0.24670053| 0.28922042| -0.01102468| 1.1087307| -1.2280084| 2.708211|\n", + "| -0.5070446| -1.0043093|-0.78254056|0.0122275995| 2.8465424|-0.060435444| 0.8980464| -1.2080427|2.0327075|\n", + "| -0.18690155| 1.2205169|0.015323491| 0.12183313|-0.41015765| 0.04452552| 1.010412| -1.3228445|1.9909104|\n", + "| -1.2551856| 1.6178073| -0.3341509|-0.060125165| -0.7554314| -0.08777025| 1.0291398| -1.3477987|1.2702764|\n", + "| 4.9607058| -1.9578062| 1.4854684| -0.03948475| 2.1833694|0.0029250523| 1.024457| -1.1581304| 5.975229|\n", + "| 0.73652315| -1.6399739| 0.7913185| -0.05238397| 1.67738| 0.01944797| 1.0993668| -1.1331724|1.9309721|\n", + "| -0.505834| 0.18756187|-0.47093546| -0.24297306|-0.60619545| -0.10791535| 0.977639| -1.2879055|1.7610806|\n", + "| -0.88477343|-0.050812364| -0.6318951| -0.15244243| -0.5258376| -0.15618815| 0.9823201| -1.2879055| 1.655031|\n", + "| -0.42840376| 0.9821427| -0.2266495| -0.36083496| -0.6883194| -0.08552282| 0.5328614| -0.12493005|1.1175063|\n", + "| 0.9369153| -1.4810578| 0.6722208|-0.121177554| 0.3996021| 0.01291408| 1.1040496| -1.1082181|2.1779811|\n", + "| -0.80702734| -0.92485124|-0.26602685| -0.1560743| 1.4398388| -0.09314839|0.55627036| -0.09498342|0.9102398|\n", + "+------------+------------+-----------+------------+-----------+------------+----------+------------+---------+\n", "only showing top 20 rows\n", "\n" ] @@ -1903,17 +1776,10 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 65, "id": "8084bdef", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - }, { "name": "stderr", "output_type": "stream", @@ -1927,20 +1793,18 @@ "[True]" ] }, - "execution_count": 64, + "execution_count": 65, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", - "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" + "server_manager.stop_servers()" ] }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 66, "id": "0138a029-87c5-497f-ac5c-3eed0e11b0f6", "metadata": {}, "outputs": [], @@ -1974,7 +1838,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.11.11" } }, "nbformat": 4, diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/image_classification_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/image_classification_torch.ipynb index 4d05305b..3a85020a 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/image_classification_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytorch/image_classification_torch.ipynb @@ -53,7 +53,7 @@ { "data": { "text/plain": [ - "'2.4.1+cu121'" + "'2.5.1+cu124'" ] }, "execution_count": 3, @@ -78,89 +78,7 @@ "execution_count": 4, "id": "1c942a46", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz\n", - "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz to datasets/data/FashionMNIST/raw/train-images-idx3-ubyte.gz\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 26421880/26421880 [00:02<00:00, 12502879.39it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Extracting datasets/data/FashionMNIST/raw/train-images-idx3-ubyte.gz to datasets/data/FashionMNIST/raw\n", - "\n", - "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz\n", - "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz to datasets/data/FashionMNIST/raw/train-labels-idx1-ubyte.gz\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 29515/29515 [00:00<00:00, 195601.60it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Extracting datasets/data/FashionMNIST/raw/train-labels-idx1-ubyte.gz to datasets/data/FashionMNIST/raw\n", - "\n", - "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz\n", - "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz to datasets/data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 4422102/4422102 [00:01<00:00, 3727194.18it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Extracting datasets/data/FashionMNIST/raw/t10k-images-idx3-ubyte.gz to datasets/data/FashionMNIST/raw\n", - "\n", - "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz\n", - "Downloading http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz to datasets/data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████| 5148/5148 [00:00<00:00, 8599074.87it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Extracting datasets/data/FashionMNIST/raw/t10k-labels-idx1-ubyte.gz to datasets/data/FashionMNIST/raw\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], + "outputs": [], "source": [ "# Download training data from open datasets.\n", "training_data = datasets.FashionMNIST(\n", @@ -375,78 +293,78 @@ "name": "stdout", "output_type": "stream", "text": [ - "loss: 2.303572 [ 64/60000]\n", - "loss: 2.283972 [ 6464/60000]\n", - "loss: 2.266706 [12864/60000]\n", - "loss: 2.268926 [19264/60000]\n", - "loss: 2.241029 [25664/60000]\n", - "loss: 2.226260 [32064/60000]\n", - "loss: 2.230959 [38464/60000]\n", - "loss: 2.197126 [44864/60000]\n", - "loss: 2.198262 [51264/60000]\n", - "loss: 2.169759 [57664/60000]\n", + "loss: 2.298206 [ 64/60000]\n", + "loss: 2.283203 [ 6464/60000]\n", + "loss: 2.262282 [12864/60000]\n", + "loss: 2.259791 [19264/60000]\n", + "loss: 2.240928 [25664/60000]\n", + "loss: 2.218922 [32064/60000]\n", + "loss: 2.225280 [38464/60000]\n", + "loss: 2.193091 [44864/60000]\n", + "loss: 2.194699 [51264/60000]\n", + "loss: 2.157922 [57664/60000]\n", "Test Error: \n", - " Accuracy: 48.6%, Avg loss: 2.155098 \n", + " Accuracy: 38.6%, Avg loss: 2.149652 \n", "\n", "Epoch 2\n", "-------------------------------\n", - "loss: 2.164836 [ 64/60000]\n", - "loss: 2.146720 [ 6464/60000]\n", - "loss: 2.086537 [12864/60000]\n", - "loss: 2.111843 [19264/60000]\n", - "loss: 2.043706 [25664/60000]\n", - "loss: 1.994988 [32064/60000]\n", - "loss: 2.016762 [38464/60000]\n", - "loss: 1.935367 [44864/60000]\n", - "loss: 1.949560 [51264/60000]\n", - "loss: 1.869589 [57664/60000]\n", + "loss: 2.164765 [ 64/60000]\n", + "loss: 2.153999 [ 6464/60000]\n", + "loss: 2.094229 [12864/60000]\n", + "loss: 2.107332 [19264/60000]\n", + "loss: 2.060189 [25664/60000]\n", + "loss: 2.009164 [32064/60000]\n", + "loss: 2.033063 [38464/60000]\n", + "loss: 1.954014 [44864/60000]\n", + "loss: 1.968186 [51264/60000]\n", + "loss: 1.892358 [57664/60000]\n", "Test Error: \n", - " Accuracy: 51.8%, Avg loss: 1.868578 \n", + " Accuracy: 54.1%, Avg loss: 1.883826 \n", "\n", "Epoch 3\n", "-------------------------------\n", - "loss: 1.898998 [ 64/60000]\n", - "loss: 1.865363 [ 6464/60000]\n", - "loss: 1.748555 [12864/60000]\n", - "loss: 1.797765 [19264/60000]\n", - "loss: 1.677429 [25664/60000]\n", - "loss: 1.638624 [32064/60000]\n", - "loss: 1.653928 [38464/60000]\n", - "loss: 1.564719 [44864/60000]\n", - "loss: 1.596518 [51264/60000]\n", - "loss: 1.483102 [57664/60000]\n", + "loss: 1.922989 [ 64/60000]\n", + "loss: 1.895849 [ 6464/60000]\n", + "loss: 1.767882 [12864/60000]\n", + "loss: 1.804950 [19264/60000]\n", + "loss: 1.702711 [25664/60000]\n", + "loss: 1.664090 [32064/60000]\n", + "loss: 1.682484 [38464/60000]\n", + "loss: 1.577310 [44864/60000]\n", + "loss: 1.613093 [51264/60000]\n", + "loss: 1.510797 [57664/60000]\n", "Test Error: \n", - " Accuracy: 61.1%, Avg loss: 1.506650 \n", + " Accuracy: 59.5%, Avg loss: 1.517127 \n", "\n", "Epoch 4\n", "-------------------------------\n", - "loss: 1.572357 [ 64/60000]\n", - "loss: 1.540674 [ 6464/60000]\n", - "loss: 1.396118 [12864/60000]\n", - "loss: 1.464891 [19264/60000]\n", - "loss: 1.345048 [25664/60000]\n", - "loss: 1.346302 [32064/60000]\n", - "loss: 1.351714 [38464/60000]\n", - "loss: 1.288438 [44864/60000]\n", - "loss: 1.324335 [51264/60000]\n", - "loss: 1.220271 [57664/60000]\n", + "loss: 1.588409 [ 64/60000]\n", + "loss: 1.558777 [ 6464/60000]\n", + "loss: 1.393466 [12864/60000]\n", + "loss: 1.465835 [19264/60000]\n", + "loss: 1.350062 [25664/60000]\n", + "loss: 1.359687 [32064/60000]\n", + "loss: 1.370576 [38464/60000]\n", + "loss: 1.287119 [44864/60000]\n", + "loss: 1.330430 [51264/60000]\n", + "loss: 1.238912 [57664/60000]\n", "Test Error: \n", - " Accuracy: 63.2%, Avg loss: 1.246688 \n", + " Accuracy: 62.4%, Avg loss: 1.254357 \n", "\n", "Epoch 5\n", "-------------------------------\n", - "loss: 1.320306 [ 64/60000]\n", - "loss: 1.310512 [ 6464/60000]\n", - "loss: 1.143846 [12864/60000]\n", - "loss: 1.247822 [19264/60000]\n", - "loss: 1.120661 [25664/60000]\n", - "loss: 1.147298 [32064/60000]\n", - "loss: 1.163092 [38464/60000]\n", - "loss: 1.110569 [44864/60000]\n", - "loss: 1.149649 [51264/60000]\n", - "loss: 1.061627 [57664/60000]\n", + "loss: 1.333722 [ 64/60000]\n", + "loss: 1.322049 [ 6464/60000]\n", + "loss: 1.143545 [12864/60000]\n", + "loss: 1.250494 [19264/60000]\n", + "loss: 1.123120 [25664/60000]\n", + "loss: 1.166146 [32064/60000]\n", + "loss: 1.181268 [38464/60000]\n", + "loss: 1.112326 [44864/60000]\n", + "loss: 1.155791 [51264/60000]\n", + "loss: 1.079376 [57664/60000]\n", "Test Error: \n", - " Accuracy: 64.8%, Avg loss: 1.082042 \n", + " Accuracy: 64.0%, Avg loss: 1.092456 \n", "\n", "Done!\n" ] @@ -500,7 +418,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "6d9b3a45-7618-43e4-8bd3-8bb317a484d3", "metadata": {}, "outputs": [ @@ -645,15 +563,7 @@ "execution_count": 19, "id": "362b266b", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:torch_tensorrt.dynamo.conversion.aten_ops_converters:Unable to import quantization op. Please install modelopt library (https://github.com/NVIDIA/TensorRT-Model-Optimizer?tab=readme-ov-file#installation) to add support for compiling quantized models\n" - ] - } - ], + "outputs": [], "source": [ "import torch_tensorrt as trt\n", "import time" @@ -702,41 +612,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO:torch_tensorrt.dynamo.utils:Using Default Torch-TRT Runtime (as requested by user)\n", - "INFO:torch_tensorrt.dynamo.utils:Device not specified, using Torch default current device - cuda:0. If this is incorrect, please specify an input device, via the device keyword.\n", - "INFO:torch_tensorrt.dynamo.utils:Compilation Settings: CompilationSettings(enabled_precisions={}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, refit=False, engine_capability=, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/timing_cache-1738007742.9371898.bin')\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:torch_tensorrt.dynamo._compiler:Node _param_constant1 of op type get_attr does not have metadata. This could sometimes lead to undefined behavior.\n", - "WARNING:torch_tensorrt.dynamo._compiler:Some nodes do not have metadata (shape and dtype information). This could lead to problems sometimes if the graph has PyTorch and TensorRT segments.\n", - "INFO:torch_tensorrt.dynamo._compiler:Partitioning the graph via the fast partitioner\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init CUDA: CPU +2, GPU +0, now: CPU 457, GPU 2889 (MiB)\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init builder kernel library: CPU +1634, GPU +288, now: CPU 2238, GPU 3177 (MiB)\n", - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/torch_tensorrt/dynamo/conversion/impl/activation/base.py:40: DeprecationWarning: Use Deprecated in TensorRT 10.1. Superseded by explicit quantization. instead.\n", - " if input_val.dynamic_range is not None and dyn_range_fn is not None:\n", - "\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT INetwork construction elapsed time: 0:00:00.005228\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Global timing cache in use. Profiling results in this builder pass will be stored.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Detected 1 inputs and 1 output network tensors.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Host Persistent Memory: 21984\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Device Persistent Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Scratch Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Started assigning block shifts. This will take 4 steps to complete.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Algorithm ShiftNTopDown took 0.127126ms to assign 2 blocks to 4 nodes requiring 4096 bytes.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Activation Memory: 4096\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Weights Memory: 2678824\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Engine generation completed in 1.59804 seconds.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage of TRT CPU/GPU memory allocators: CPU 1 MiB, GPU 5 MiB\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage during Engine building and serialization: CPU: 3951 MiB\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:Build TRT engine elapsed time: 0:00:01.601778\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT Engine uses: 2832188 bytes of Memory\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 26 bytes of code generator cache.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 43 timing cache entries\n" + "WARNING:torch_tensorrt.dynamo._compiler:Node linear_default of op type call_function does not have metadata. This could sometimes lead to undefined behavior.\n", + "WARNING:torch_tensorrt.dynamo._compiler:Some nodes do not have metadata (shape and dtype information). This could lead to problems sometimes if the graph has PyTorch and TensorRT segments.\n" ] }, { @@ -771,39 +648,7 @@ "execution_count": 23, "id": "3e7e7689", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:torch_tensorrt.dynamo._compiler:Compilation Settings: CompilationSettings(enabled_precisions={}, debug=False, workspace_size=0, min_block_size=5, torch_executed_ops=set(), pass_through_build_failures=False, max_aux_streams=None, version_compatible=False, optimization_level=None, use_python_runtime=False, truncate_double=False, use_fast_partitioner=True, enable_experimental_decompositions=False, device=Device(type=DeviceType.GPU, gpu_id=0), require_full_compilation=False, disable_tf32=False, assume_dynamic_shape_support=False, sparse_weights=False, refit=False, engine_capability=, num_avg_timing_iters=1, dla_sram_size=1048576, dla_local_dram_size=1073741824, dla_global_dram_size=536870912, dryrun=False, hardware_compatible=False, timing_cache_path='/tmp/timing_cache-1738007742.9371898.bin')\n", - "\n", - "INFO:torch_tensorrt.dynamo._compiler:Partitioning the graph via the fast partitioner\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init CUDA: CPU +0, GPU +0, now: CPU 759, GPU 2891 (MiB)\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageChange] Init builder kernel library: CPU +1633, GPU +286, now: CPU 2392, GPU 3177 (MiB)\n", - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/torch_tensorrt/dynamo/conversion/impl/activation/base.py:40: DeprecationWarning: Use Deprecated in TensorRT 10.1. Superseded by explicit quantization. instead.\n", - " if input_val.dynamic_range is not None and dyn_range_fn is not None:\n", - "\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT INetwork construction elapsed time: 0:00:00.004842\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Global timing cache in use. Profiling results in this builder pass will be stored.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Detected 1 inputs and 1 output network tensors.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Host Persistent Memory: 22944\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Device Persistent Memory: 0\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Scratch Memory: 25088\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Started assigning block shifts. This will take 7 steps to complete.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[BlockAssignment] Algorithm ShiftNTopDown took 0.253198ms to assign 4 blocks to 7 nodes requiring 263168 bytes.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Activation Memory: 262144\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Total Weights Memory: 2678824\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Engine generation completed in 1.20766 seconds.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage of TRT CPU/GPU memory allocators: CPU 1 MiB, GPU 5 MiB\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:[MemUsageStats] Peak memory usage during Engine building and serialization: CPU: 3978 MiB\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:Build TRT engine elapsed time: 0:00:01.210199\n", - "INFO:torch_tensorrt.dynamo.conversion._TRTInterpreter:TRT Engine uses: 2885916 bytes of Memory\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 26 bytes of code generator cache.\n", - "INFO:torch_tensorrt [TensorRT Conversion Context]:Serialized 76 timing cache entries\n" - ] - } - ], + "outputs": [], "source": [ "example_inputs = (torch.randn((10, 784), dtype=torch.float).to(\"cuda\"),)\n", "\n", @@ -935,17 +780,7 @@ "execution_count": 28, "id": "42c5feba", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/pyspark/broadcast.py:38: DeprecationWarning: typing.io is deprecated, import directly from typing instead. typing.io will be removed in Python 3.12.\n", - " from typing.io import BinaryIO # type: ignore[import]\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "from pyspark.sql.functions import col, struct, pandas_udf, array\n", "from pyspark.ml.functions import predict_batch_udf\n", @@ -1008,12 +843,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:55:48 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "25/01/27 19:55:48 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "25/02/04 13:50:47 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/02/04 13:50:47 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "25/01/27 19:55:48 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", - "25/01/27 19:55:49 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + "25/02/04 13:50:48 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" ] } ], @@ -1100,7 +934,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "id": "7760bdbe", "metadata": {}, "outputs": [ @@ -1594,21 +1428,12 @@ "id": "4863d5ff", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:py.warnings:/home/rishic/anaconda3/envs/spark-dl-torch/lib/python3.11/site-packages/pyspark/sql/pandas/serializers.py:229: DeprecationWarning: is_categorical_dtype is deprecated and will be removed in a future version. Use isinstance(dtype, pd.CategoricalDtype) instead\n", - " elif is_categorical_dtype(s.dtype):\n", - "\n" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 151 ms, sys: 36.7 ms, total: 187 ms\n", - "Wall time: 1.49 s\n" + "CPU times: user 185 ms, sys: 28.9 ms, total: 214 ms\n", + "Wall time: 1.5 s\n" ] }, { @@ -1640,8 +1465,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 199 ms, sys: 28 ms, total: 227 ms\n", - "Wall time: 995 ms\n" + "CPU times: user 66.9 ms, sys: 11.2 ms, total: 78.1 ms\n", + "Wall time: 875 ms\n" ] }, { @@ -1673,16 +1498,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:55:52 WARN TaskSetManager: Stage 0 contains a task of very large size (4030 KiB). The maximum recommended task size is 1000 KiB.\n", - "[Stage 0:> (0 + 8) / 8]\r" + "25/02/04 13:50:51 WARN TaskSetManager: Stage 0 contains a task of very large size (4030 KiB). The maximum recommended task size is 1000 KiB.\n", + "[Stage 0:=======> (1 + 7) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 0 ns, sys: 4.56 ms, total: 4.56 ms\n", - "Wall time: 1.74 s\n" + "CPU times: user 2.09 ms, sys: 1.6 ms, total: 3.69 ms\n", + "Wall time: 1.71 s\n" ] }, { @@ -1713,16 +1538,22 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 19:55:53 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n", - "25/01/27 19:55:54 WARN TaskSetManager: Stage 3 contains a task of very large size (7847 KiB). The maximum recommended task size is 1000 KiB.\n" + "25/02/04 13:50:53 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/02/04 13:50:53 WARN TaskSetManager: Stage 3 contains a task of very large size (7847 KiB). The maximum recommended task size is 1000 KiB.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.32 ms, sys: 287 μs, total: 2.61 ms\n", - "Wall time: 864 ms\n" + "CPU times: user 2.94 ms, sys: 61 μs, total: 3 ms\n", + "Wall time: 943 ms\n" ] } ], @@ -1883,8 +1714,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 543 ms, sys: 1.13 s, total: 1.67 s\n", - "Wall time: 22.7 s\n" + "CPU times: user 167 ms, sys: 76.2 ms, total: 243 ms\n", + "Wall time: 10.9 s\n" ] } ], @@ -1904,8 +1735,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 211 ms, sys: 65.9 ms, total: 277 ms\n", - "Wall time: 715 ms\n" + "CPU times: user 234 ms, sys: 64.1 ms, total: 298 ms\n", + "Wall time: 685 ms\n" ] } ], @@ -1924,8 +1755,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 192 ms, sys: 54.9 ms, total: 247 ms\n", - "Wall time: 628 ms\n" + "CPU times: user 403 ms, sys: 60.1 ms, total: 463 ms\n", + "Wall time: 809 ms\n" ] } ], @@ -1984,7 +1815,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAknklEQVR4nO3de3SV9Z3v8c9OSDa3ZMcQcpNAAyi0cumUSsqoFEsOkM64QDkdb3MOuDow0uCqUqsnPVZq27PS4hrrqUNxrbNaqKvihXNERsdiBSWMCnRAGMZeUsAoYSChYJMNCUl2sn/nD8bMREH4/kzyS8L7tdZei+z9fHh+efIknzzZO99EnHNOAAD0spTQCwAAXJooIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBDAq9gA9LJpM6evSoMjIyFIlEQi8HAGDknNOpU6dUWFiolJTzX+f0uQI6evSoioqKQi8DAPAJ1dbWatSoUed9vM8VUEZGhiTpWn1Zg5QWeDXodr11VcuEKSCYdiX0ul7q/Hp+Pj1WQKtXr9bDDz+suro6TZ06VY899pimT59+wdwHP3YbpDQNilBAA06v/ViVAgKC+fdPvws9jdIjL0J45plntGLFCq1cuVJvvfWWpk6dqrlz5+r48eM9sTsAQD/UIwX0yCOPaMmSJbrjjjv0mc98Ro8//riGDh2qn/3sZz2xOwBAP9TtBdTW1qY9e/aotLT0P3aSkqLS0lLt2LHjI9u3trYqHo93uQEABr5uL6ATJ06oo6NDeXl5Xe7Py8tTXV3dR7avrKxULBbrvPEKOAC4NAT/RdSKigo1NjZ23mpra0MvCQDQC7r9VXA5OTlKTU1VfX19l/vr6+uVn5//ke2j0aii0Wh3LwMA0Md1+xVQenq6pk2bpq1bt3bel0wmtXXrVs2YMaO7dwcA6Kd65PeAVqxYoUWLFunzn/+8pk+frkcffVRNTU264447emJ3AIB+qEcK6Oabb9Yf//hHPfjgg6qrq9NnP/tZbd68+SMvTAAAXLoizvWtmSXxeFyxWEyzNJ9JCBiwUscXmzPH/s7+XGnu/N+bM8An1e4S2qZNamxsVGZm5nm3C/4qOADApYkCAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQfTINGygv6r5gf1vVt0/f6M5U9Vw/gGN5zM+7Yw5k7/fnpGkX2yYbc4Ufe9Nr33h0sUVEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIJgGnYfFhlk//C4jg77jpyzZzxF0tLNGZdoM2cGFY8xZyRpy20PmzNf/NXd5syVf7PbnKk3J6TdN1/vkZIe/O5T5sza7/kdc7NIxJ7pxXMcF48rIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIIuJc35rSF4/HFYvFNEvzNSiSFno53cdjgGIkNdWc8RpG6qtvnTpd/GHtNK/c2KI/mjODSg977asvO/NysTlzw+X7zZktkzLMGfR97S6hbdqkxsZGZWZmnnc7roAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIIhBoRdwyfAY3Ona2+378Rh62peHikpSytRPmzNbvvS/vfZV+ssV5syV8hhGmmIfNBtJsX9svc4hSWk/zDZnbl73L+bMhsXfNGcuW7fDnEHfxBUQACAICggAEES3F9B3vvMdRSKRLreJEyd2924AAP1cjzwHdNVVV2nLli3/sZNBPNUEAOiqR5ph0KBBys/P74n/GgAwQPTIc0AHDhxQYWGhxo4dq9tvv12HD5//VUKtra2Kx+NdbgCAga/bC6ikpETr1q3T5s2btWbNGtXU1Oi6667TqVOnzrl9ZWWlYrFY562oqKi7lwQA6IO6vYDKysr0la98RVOmTNHcuXP10ksvqaGhQc8+++w5t6+oqFBjY2Pnrba2truXBADog3r81QFZWVm68sordfDgwXM+Ho1GFY1Ge3oZAIA+psd/D+j06dM6dOiQCgoKenpXAIB+pNsL6N5771VVVZXeffddvfnmm7rxxhuVmpqqW2+9tbt3BQDox7r9R3BHjhzRrbfeqpMnT2rkyJG69tprtXPnTo0cObK7dwUA6Me6vYCefvrp7v4vYeEzWNRjMKYkKdlhjsRv/YI5M2b5H8yZx09eZ85IUuGrvTSdyiXNkUh0qH03nsNI675gf172nUSmOfPMQw+bMyv/9svmTP0Mfr2jL2IWHAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAE0eN/kK7XRCL2jM/gzt7kMyTUY8ilz1BRX1d/Y485U92YZ840t6ebM5I0/NmdXjmrSKrnANheknbKnnmz6Qpz5tE/fcqcuWvUFnPmgduWmDOSlLne43zora9FPvvx3VcP4QoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQQycadh9ncfkWp+JyS7Re5Oth20fac60O/uY5dQU+4TvdzeNNWckqUB1Xjkr1+HxcWpLdP9CziPvsTfNmW9VVJsz1x69ypxZeWC+OXPz/9xszkjSyy8UmTPJUx6jxH34TrXuQ385gCsgAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAhi4Awj9RiWF0lL99tVos0jZF+f1348HL3vz71y38x91px58t++YM4kZR+eWPCIfZhmr+rD54OvXzWnmTP/fcxOc2bV3jnmTGqR3zDNjF/av0Y0Xuu1q94T8bjucD0z5JgrIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIIuKcx1TEHhSPxxWLxTRL8zUoYh9uOJAcX24fEtqaZd/Pr5ausockrWv4vDkzKv19c+Z7v7zJnIn9wT7AVJKu/5td5szG33zWnBn0b1FzJqXN432K+H16++yrdUTSnBk54YQ5E4u2mDNn2v2+lqwc/w/mzLINS82Z4v+xw5zpy9pdQtu0SY2NjcrMzDzvdlwBAQCCoIAAAEGYC2j79u264YYbVFhYqEgkoueff77L4845PfjggyooKNCQIUNUWlqqAwcOdNd6AQADhLmAmpqaNHXqVK1evfqcj69atUo//vGP9fjjj2vXrl0aNmyY5s6dq5YW+89tAQADl/kvopaVlamsrOycjznn9Oijj+qBBx7Q/PnzJUlPPPGE8vLy9Pzzz+uWW275ZKsFAAwY3focUE1Njerq6lRaWtp5XywWU0lJiXbsOPerPFpbWxWPx7vcAAADX7cWUF1dnSQpLy+vy/15eXmdj31YZWWlYrFY562oqKg7lwQA6KOCvwquoqJCjY2Nnbfa2trQSwIA9IJuLaD8/HxJUn19fZf76+vrOx/7sGg0qszMzC43AMDA160FVFxcrPz8fG3durXzvng8rl27dmnGjBnduSsAQD9nfhXc6dOndfDgwc63a2pqtG/fPmVnZ2v06NG6++679f3vf19XXHGFiouL9e1vf1uFhYVasGBBd64bANDPmQto9+7duv766zvfXrFihSRp0aJFWrdune677z41NTVp6dKlamho0LXXXqvNmzdr8ODB3bdqAEC/d0kPI31nld+PBf/uxp+bMyv++a/MmfT0dnPmB1OfM2c2N0wxZyQpRfZT53eNeRfe6EPe23u5OdNxWcKckaTBtenmTOY79uOQ0m7PdKTbB4Qmzd9inuVS7Zlkmn19kaT9OJye2WzOfLboiDkjSUdPx8yZa/LeMWfeet/+6t8j72eZM5I0+iv/6pWzYBgpAKBPo4AAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAjPWbkDw29v/3uv3MKDf2HO5Pyj/c9RnF54ypz52dHrzJnGNr8/lXFH0RvmzIm2YeZMzeCkOaMO+2RmSWq7zL6vxFf+ZM6MijWaMyOjp82ZaKp9orokZQ2yT5xOeIzQbuqImjMzM6vt+0na9yNJbw4ab86kyn4ODRvUZs68NH2NOSNJt95+rzkTe3Kn174uhCsgAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAhiwAwjbblhujmTFtnnt69v5ZszOf/rPXPm5fHPmTMPn7Afh6Ep9kGIkrTy9QXmTErcfsq5LI+Bmh7zS8/uK2HOxA9cZs4cbBhhztTaZ54qtdXZQ546ovYBsM5jZuzr6Z8zZ25f/Ip9R5Kuy/qDOfO5wYfNmZfTrjJn/uKf7zRnJOlvKn5lzrz8ZKbXvi6EKyAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACGLADCOt/3zvvStZP6w1Z/4y51/MmZ822AcU/lXWP5sz3639S3NGki7/Zao5kxjqMX1SaeZEJOk3hNOl2NfXkW7fTzLNvj6ftbVm+RxvSR6xiMfMWJ/9DP83+6TZx//pevuOJP1h/hpz5jdt9ndqUWy/OfOPmZPNGUn669i/mjO/+rOlpu0jHa3Sv2y64HZcAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEANmGGna6d7b172XbzZnNvxpujmTmx43Z7556L+aM81/f7k5I0lncu3fv/gMx0xtM0eUTPUbwpnSSwM1fQZ3OvvsV6+1SVL7EL+clc9xODXaft4VVNn3I0lfmzbTnCkc3GDOVJ/OM2cWXr7XnJGkESn2D+6Zy4eZtm9PpEoXMX+ZKyAAQBAUEAAgCHMBbd++XTfccIMKCwsViUT0/PPPd3l88eLFikQiXW7z5s3rrvUCAAYIcwE1NTVp6tSpWr169Xm3mTdvno4dO9Z5e+qppz7RIgEAA4/5RQhlZWUqKyv72G2i0ajy8/O9FwUAGPh65Dmgbdu2KTc3VxMmTNCyZct08uTJ827b2tqqeDze5QYAGPi6vYDmzZunJ554Qlu3btUPf/hDVVVVqaysTB0dHefcvrKyUrFYrPNWVFTU3UsCAPRB3f57QLfcckvnvydPnqwpU6Zo3Lhx2rZtm2bPnv2R7SsqKrRixYrOt+PxOCUEAJeAHn8Z9tixY5WTk6ODBw+e8/FoNKrMzMwuNwDAwNfjBXTkyBGdPHlSBQUFPb0rAEA/Yv4R3OnTp7tczdTU1Gjfvn3Kzs5Wdna2HnroIS1cuFD5+fk6dOiQ7rvvPo0fP15z587t1oUDAPo3cwHt3r1b119/fefbHzx/s2jRIq1Zs0b79+/Xz3/+czU0NKiwsFBz5szR9773PUWj0e5bNQCg3zMX0KxZs+ScO+/jL7/88idakK9Bzb23rwNt9t9xejjfPjjw/522Px/W+LP/Ys4kY34TKxPD7LnIuV8M+bE60u0Z3yGcH3Nqn39XvTRY1GtAqOdxGHTGnknxGBrrcz74HLuOqN+B+PX6qebMhhUPmzP/lD7OnLl6yLvmjCTFk0lzZlj1CdP27R2tF7Uds+AAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQRLf/Se5QhpywT3j1NS7tuDlzpN0+Xvi+l+40ZzJG2L+nSAwzRyRJ6XF7xnl8y9Mx2J6Rx1RrSWof6rErj+nMPuvzmbrtKzHcvsCkx1eT1Db7lOqUixu03EX8U37TsDMO24/Dtw7PN2f+77gt5sy2Mx4nq6TC1FPmTMeBd2zbu8RFbccVEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEMWCGkWYeOm3OJFyH177GprWYMzN3LDNn8t+wD0L800RzxHvIZUuOPdMRtb9P0fc9Bkl6fmvlMyy1faj9fWrPsJ97KcMvbsDjf5Zs9ZmUKkWPppkzaaftHyevY+cxnDa11W8Y6Zlce+6d9VeYM7+7/x/MGWm4R0a6LGWIOZM6Ybxpe9fRKh248HZcAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEANmGGlKc5s5kxbxG9T4r22Z5szwX9kHB/5poscAxaQ90jHEPhBSktqy7DuLnrAf8/RT9vWlLzhuzkjSycZh5kxHm/3TKP1w1JzJ3m7/fjHi96HVmWz7uddc6DFY1GMYqXzmivrNIlUiw76+ISn2j9P8nXeaM7+c8RNzRpLaZT/3IgnbxOJI8uK25woIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIIYMMNI1d5hjjQmz3jt6m93fs2cSRtln4bYmmN/n1Ja7fuJJPwmNabnN5szY38UN2fa3ztiztRHS8wZSRq/9aQ9dLTWnskdYY68d1OuOdM8xjZE8gORofbhvu6Mx3DfpMf52m7PdKT6TWV1g+y55lH2TMZO+7DiuulDzRlJGpdmv+5of+dd2/YucVHbcQUEAAiCAgIABGEqoMrKSl199dXKyMhQbm6uFixYoOrq6i7btLS0qLy8XCNGjNDw4cO1cOFC1dfXd+uiAQD9n6mAqqqqVF5erp07d+qVV15RIpHQnDlz1NTU1LnNPffcoxdeeEEbNmxQVVWVjh49qptuuqnbFw4A6N9ML0LYvHlzl7fXrVun3Nxc7dmzRzNnzlRjY6N++tOfav369frSl74kSVq7dq0+/elPa+fOnfrCF77QfSsHAPRrn+g5oMbGRklSdna2JGnPnj1KJBIqLS3t3GbixIkaPXq0duzYcc7/o7W1VfF4vMsNADDweRdQMpnU3XffrWuuuUaTJk2SJNXV1Sk9PV1ZWVldts3Ly1NdXd05/5/KykrFYrHOW1FRke+SAAD9iHcBlZeX6+2339bTTz/9iRZQUVGhxsbGzlttrcfvVAAA+h2vX0Rdvny5XnzxRW3fvl2jRo3qvD8/P19tbW1qaGjochVUX1+v/Pz8c/5f0WhU0WjUZxkAgH7MdAXknNPy5cu1ceNGvfrqqyouLu7y+LRp05SWlqatW7d23lddXa3Dhw9rxowZ3bNiAMCAYLoCKi8v1/r167Vp0yZlZGR0Pq8Ti8U0ZMgQxWIxffWrX9WKFSuUnZ2tzMxM3XXXXZoxYwavgAMAdGEqoDVr1kiSZs2a1eX+tWvXavHixZKkH/3oR0pJSdHChQvV2tqquXPn6ic/+Um3LBYAMHCYCsi5Cw/ZGzx4sFavXq3Vq1d7L8pLqv31FC82jbrwRufgkh4Zj3mfqc0eQwOHewwwjfi9FiXZan8KsfnKkeZMes175szlzx4yZySp/i/GmjMnr8kwZ7JGnDZnWk/bh+dG3k83ZyRJDWn2ffnMtPU59XwyfrNIFUnYd+aG2wfANhfa9/PXW/7WnJGkmr/8P+ZM6ohs0/Yu2Sa9f+HtmAUHAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAILz+Impf1PG7A+bMsJRWr309cc1PzZm/Tiw1ZyLNqeZMaixhziQ7/CYmuxb76RNf3mjOpNx1pTnT3Gqf5ixJzp2yh94fYo40vptlzkTsg86lNM8x0PZTz2vitEv1CHlMo4/4jKOX5AbbD3pqg/3zomOY/Z1Kr++9L98tf1Z84Y3+k/b2Fum1C2/HFRAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABDFghpH6yEpp9sqlekxd/NF1T5szhYP+ZM784uSfmzMvvjHNnJEkddgHPL5/JMucSW22f5/kevFbq4jHQE2XZh8+6fxmxnqJtHsM7/SZ9+kzK9VjPy7FcyirxzmejHruyygl4TdgtTnZZs60Zdmqoj1xcdtzBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVzSw0i3nLrKKzdl6GFz5nKPwaIT0trNmU8NPmnO/LdZ/2TOSNL631xtDx0eYo6k2A+DEjH7sE9JingMn4wk7ZmUM36DJK1c7+xGkhTxmMHpUuwL9Bk067O2s/vyead8ziH7bga12DOSVNPeYc4MPpEwbd/efnHbcwUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFc2sNIj07wysVGN5sz49L+aM78In6lOZMWsQ8aLMvYb85I0pTP15ozI0pOmzN5qfbMxvifmTOS9NJR+4Da9qT9+7jmtjRzJumxn2iabYjkB4Z4DML1OQ6DUuxTOH3mip5uiXqkpNgQ+8TPwuGN5kxbR6o5k/SZyirpjTPjzJljMwabtu9olXQRM465AgIABEEBAQCCMBVQZWWlrr76amVkZCg3N1cLFixQdXV1l21mzZqlSCTS5XbnnXd266IBAP2fqYCqqqpUXl6unTt36pVXXlEikdCcOXPU1NTUZbslS5bo2LFjnbdVq1Z166IBAP2f6UUImzdv7vL2unXrlJubqz179mjmzJmd9w8dOlT5+fnds0IAwID0iZ4Damw8+2qP7OzsLvc/+eSTysnJ0aRJk1RRUaHm5vO/aqy1tVXxeLzLDQAw8Hm/DDuZTOruu+/WNddco0mTJnXef9ttt2nMmDEqLCzU/v37df/996u6ulrPPffcOf+fyspKPfTQQ77LAAD0U94FVF5errfffluvv/56l/uXLl3a+e/JkyeroKBAs2fP1qFDhzRu3Edff15RUaEVK1Z0vh2Px1VUVOS7LABAP+FVQMuXL9eLL76o7du3a9SoUR+7bUlJiSTp4MGD5yygaDSqaNTvl8QAAP2XqYCcc7rrrru0ceNGbdu2TcXFxRfM7Nu3T5JUUFDgtUAAwMBkKqDy8nKtX79emzZtUkZGhurq6iRJsVhMQ4YM0aFDh7R+/Xp9+ctf1ogRI7R//37dc889mjlzpqZMmdIj7wAAoH8yFdCaNWsknf1l0/9s7dq1Wrx4sdLT07VlyxY9+uijampqUlFRkRYuXKgHHnig2xYMABgYzD+C+zhFRUWqqqr6RAsCAFwaLulp2EM8JwUvy/qNOXMwETFnyrPs06b92CfxnmX/na3mZJs5MzRlqDnz6ZzqC290Dssu22vOXJZqX5+PN1rsk6Mbkn5ry0+1f2wHe0xi75D986LF2c/XkSmt5owkFacNN2f2tNrP8fFp9mO3oyXLnJGk9X8sMWdGVb5p2r7dJXTgIrZjGCkAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABHFpDyP9ut9fYi2b8HVzJtpgH3yaTOud7w86on77OXK9PRfJsw+FzHhziDlT+Muj5owkuVT7+9Rx2TBzJvVUizmjY8fNEZdot+9HUmSofYhpZLjH4NMLTNg/p3b74E5fzVfZ/5Cmz+ft8D2HzZn2Y3XmzFn2QbM9hSsgAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQRJ+bBef+fTZUuxKSx5go07467HPJJKk9YZ/jldruMQsu0kuz4FL89pNs8ZgF12w/5h1tEXOmPen3sXUe35N1tKfa9+Nz7rk2e8R5zoJL2r80RJL24+A1Cy7Ze7Pg2tvtn+tJj3OoPWn/2LY7+9eU3tKus2tzF/j4RtyFtuhlR44cUVFRUehlAAA+odraWo0aNeq8j/e5Akomkzp69KgyMjIUiXT9zjcej6uoqEi1tbXKzMwMtMLwOA5ncRzO4jicxXE4qy8cB+ecTp06pcLCQqV8zE9Y+tyP4FJSUj62MSUpMzPzkj7BPsBxOIvjcBbH4SyOw1mhj0MsFrvgNrwIAQAQBAUEAAiiXxVQNBrVypUrFY36/SXTgYLjcBbH4SyOw1kch7P603Hocy9CAABcGvrVFRAAYOCggAAAQVBAAIAgKCAAQBD9poBWr16tT33qUxo8eLBKSkr061//OvSSet13vvMdRSKRLreJEyeGXlaP2759u2644QYVFhYqEono+eef7/K4c04PPvigCgoKNGTIEJWWlurAgQNhFtuDLnQcFi9e/JHzY968eWEW20MqKyt19dVXKyMjQ7m5uVqwYIGqq6u7bNPS0qLy8nKNGDFCw4cP18KFC1VfXx9oxT3jYo7DrFmzPnI+3HnnnYFWfG79ooCeeeYZrVixQitXrtRbb72lqVOnau7cuTp+/HjopfW6q666SseOHeu8vf7666GX1OOampo0depUrV69+pyPr1q1Sj/+8Y/1+OOPa9euXRo2bJjmzp2rlhb7IMm+7ELHQZLmzZvX5fx46qmnenGFPa+qqkrl5eXauXOnXnnlFSUSCc2ZM0dNTU2d29xzzz164YUXtGHDBlVVVeno0aO66aabAq66+13McZCkJUuWdDkfVq1aFWjF5+H6genTp7vy8vLOtzs6OlxhYaGrrKwMuKret3LlSjd16tTQywhKktu4cWPn28lk0uXn57uHH364876GhgYXjUbdU089FWCFvePDx8E55xYtWuTmz58fZD2hHD9+3ElyVVVVzrmzH/u0tDS3YcOGzm1+97vfOUlux44doZbZ4z58HJxz7otf/KL7+te/Hm5RF6HPXwG1tbVpz549Ki0t7bwvJSVFpaWl2rFjR8CVhXHgwAEVFhZq7Nixuv3223X48OHQSwqqpqZGdXV1Xc6PWCymkpKSS/L82LZtm3JzczVhwgQtW7ZMJ0+eDL2kHtXY2ChJys7OliTt2bNHiUSiy/kwceJEjR49ekCfDx8+Dh948sknlZOTo0mTJqmiokLNzc0hlndefW4Y6YedOHFCHR0dysvL63J/Xl6efv/73wdaVRglJSVat26dJkyYoGPHjumhhx7Sddddp7ffflsZGRmhlxdEXV2dJJ3z/PjgsUvFvHnzdNNNN6m4uFiHDh3St771LZWVlWnHjh1KTfX4Wz19XDKZ1N13361rrrlGkyZNknT2fEhPT1dWVlaXbQfy+XCu4yBJt912m8aMGaPCwkLt379f999/v6qrq/Xcc88FXG1Xfb6A8B/Kyso6/z1lyhSVlJRozJgxevbZZ/XVr3414MrQF9xyyy2d/548ebKmTJmicePGadu2bZo9e3bAlfWM8vJyvf3225fE86Af53zHYenSpZ3/njx5sgoKCjR79mwdOnRI48aN6+1lnlOf/xFcTk6OUlNTP/Iqlvr6euXn5wdaVd+QlZWlK6+8UgcPHgy9lGA+OAc4Pz5q7NixysnJGZDnx/Lly/Xiiy/qtdde6/LnW/Lz89XW1qaGhoYu2w/U8+F8x+FcSkpKJKlPnQ99voDS09M1bdo0bd26tfO+ZDKprVu3asaMGQFXFt7p06d16NAhFRQUhF5KMMXFxcrPz+9yfsTjce3ateuSPz+OHDmikydPDqjzwzmn5cuXa+PGjXr11VdVXFzc5fFp06YpLS2ty/lQXV2tw4cPD6jz4ULH4Vz27dsnSX3rfAj9KoiL8fTTT7toNOrWrVvnfvvb37qlS5e6rKwsV1dXF3ppveob3/iG27Ztm6upqXFvvPGGKy0tdTk5Oe748eOhl9ajTp065fbu3ev27t3rJLlHHnnE7d2717333nvOOed+8IMfuKysLLdp0ya3f/9+N3/+fFdcXOzOnDkTeOXd6+OOw6lTp9y9997rduzY4WpqatyWLVvc5z73OXfFFVe4lpaW0EvvNsuWLXOxWMxt27bNHTt2rPPW3Nzcuc2dd97pRo8e7V599VW3e/duN2PGDDdjxoyAq+5+FzoOBw8edN/97nfd7t27XU1Njdu0aZMbO3asmzlzZuCVd9UvCsg55x577DE3evRol56e7qZPn+527twZekm97uabb3YFBQUuPT3dXX755e7mm292Bw8eDL2sHvfaa685SR+5LVq0yDl39qXY3/72t11eXp6LRqNu9uzZrrq6Ouyie8DHHYfm5mY3Z84cN3LkSJeWlubGjBnjlixZMuC+STvX+y/JrV27tnObM2fOuK997Wvusssuc0OHDnU33nijO3bsWLhF94ALHYfDhw+7mTNnuuzsbBeNRt348ePdN7/5TdfY2Bh24R/Cn2MAAATR558DAgAMTBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAI4v8DIE1CeiobCX8AAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAJJ5JREFUeJzt3Xt0lfWd7/HPTkg2t2THEHKTQAMotHLplErKqBRLDpDOuEA5HW9zDrg6MNLgqlKrJz1Watuz0uIa66lDca2zWqir4oVzREbHYgUljAp0QBjGXlLAKGEgoWCTDQlJdrJ/5w/GzERB+P5M8kvC+7XWXovs/Xx4fnnyJJ882TvfRJxzTgAA9LKU0AsAAFyaKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQQwKvYAPSyaTOnr0qDIyMhSJREIvBwBg5JzTqVOnVFhYqJSU81/n9LkCOnr0qIqKikIvAwDwCdXW1mrUqFHnfbzPFVBGRoYk6Vp9WYOUFng16Ha9dVXLhCkgmHYl9Lpe6vx6fj49VkCrV6/Www8/rLq6Ok2dOlWPPfaYpk+ffsHcBz92G6Q0DYpQQANOr/1YlQICgvn3T78LPY3SIy9CeOaZZ7RixQqtXLlSb731lqZOnaq5c+fq+PHjPbE7AEA/1CMF9Mgjj2jJkiW644479JnPfEaPP/64hg4dqp/97Gc9sTsAQD/U7QXU1tamPXv2qLS09D92kpKi0tJS7dix4yPbt7a2Kh6Pd7kBAAa+bi+gEydOqKOjQ3l5eV3uz8vLU11d3Ue2r6ysVCwW67zxCjgAuDQE/0XUiooKNTY2dt5qa2tDLwkA0Au6/VVwOTk5Sk1NVX19fZf76+vrlZ+f/5Hto9GootFody8DANDHdfsVUHp6uqZNm6atW7d23pdMJrV161bNmDGju3cHAOineuT3gFasWKFFixbp85//vKZPn65HH31UTU1NuuOOO3pidwCAfqhHCujmm2/WH//4Rz344IOqq6vTZz/7WW3evPkjL0wAAFy6Is71rZkl8XhcsVhMszSfSQgYsFLHF5szx/7O/lxp7vzfmzPAJ9XuEtqmTWpsbFRmZuZ5twv+KjgAwKWJAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEH0yDRsoL+q+YH9b1bdP3+jOVPVcP4BjeczPu2MOZO/356RpF9smG3OFH3vTa994dLFFRAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCYBp2HxYZZP/wuI4O+46cs2c8RdLSzRmXaDNnBhWPMWckacttD5szX/zV3ebMlX+z25ypNyek3Tdf75GSHvzuU+bM2u/5HXOzSMSe6cVzHBePKyAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACCLiXN+a0hePxxWLxTRL8zUokhZ6Od3HY4BiJDXVnPEaRuqrb506Xfxh7TSv3NiiP5ozg0oPe+2rLzvzcrE5c8Pl+82ZLZMyzBn0fe0uoW3apMbGRmVmZp53O66AAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACCIQaEXcMnwGNzp2tvt+/EYetqXh4pKUsrUT5szW770v732VfrLFebMlfIYRppiHzQbSbF/bL3OIUlpP8w2Z25e9y/mzIbF3zRnLlu3w5xB38QVEAAgCAoIABBEtxfQd77zHUUikS63iRMndvduAAD9XI88B3TVVVdpy5Yt/7GTQTzVBADoqkeaYdCgQcrPz++J/xoAMED0yHNABw4cUGFhocaOHavbb79dhw+f/1VCra2tisfjXW4AgIGv2wuopKRE69at0+bNm7VmzRrV1NTouuuu06lTp865fWVlpWKxWOetqKiou5cEAOiDur2AysrK9JWvfEVTpkzR3Llz9dJLL6mhoUHPPvvsObevqKhQY2Nj5622tra7lwQA6IN6/NUBWVlZuvLKK3Xw4MFzPh6NRhWNRnt6GQCAPqbHfw/o9OnTOnTokAoKCnp6VwCAfqTbC+jee+9VVVWV3n33Xb355pu68cYblZqaqltvvbW7dwUA6Me6/UdwR44c0a233qqTJ09q5MiRuvbaa7Vz506NHDmyu3cFAOjHur2Ann766e7+L2HhM1jUYzCmJCnZYY7Eb/2COTNm+R/MmcdPXmfOSFLhq700ncolzZFIdKh9N57DSOu+YH9e9p1EpjnzzEMPmzMr//bL5kz9DH69oy9iFhwAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABNHjf5Cu10Qi9ozP4M7e5DMk1GPIpc9QUV9Xf2OPOVPdmGfONLenmzOSNPzZnV45q0iq5wDYXpJ2yp55s+kKc+bRP33KnLlr1BZz5oHblpgzkpS53uN86K2vRT778d1XD+EKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEEMnGnYfZ3H5Fqficku0XuTrYdtH2nOtDv7mOXUFPuE73c3jTVnJKlAdV45K9fh8XFqS3T/Qs4j77E3zZlvVVSbM9cevcqcWXlgvjlz8//cbM5I0ssvFJkzyVMeo8R9+E617kN/OYArIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIYuAMI/UYlhdJS/fbVaLNI2Rfn9d+PBy978+9ct/MfdacefLfvmDOJGUfnljwiH2YZq/qw+eDr181p5kz/33MTnNm1d455kxqkd8wzYxf2r9GNF7rtaveE/G47nA9M+SYKyAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACCLinMdUxB4Uj8cVi8U0S/M1KGIfbjiQHF9uHxLammXfz6+WrrKHJK1r+Lw5Myr9fXPme7+8yZyJ/cE+wFSSrv+bXebMxt981pwZ9G9RcyalzeN9ivh9evvsq3VE0pwZOeGEOROLtpgzZ9r9vpasHP8P5syyDUvNmeL/scOc6cvaXULbtEmNjY3KzMw873ZcAQEAgqCAAABBmAto+/btuuGGG1RYWKhIJKLnn3++y+POOT344IMqKCjQkCFDVFpaqgMHDnTXegEAA4S5gJqamjR16lStXr36nI+vWrVKP/7xj/X4449r165dGjZsmObOnauWFvvPbQEAA5f5L6KWlZWprKzsnI855/Too4/qgQce0Pz58yVJTzzxhPLy8vT888/rlltu+WSrBQAMGN36HFBNTY3q6upUWlraeV8sFlNJSYl27Dj3qzxaW1sVj8e73AAAA1+3FlBdXZ0kKS8vr8v9eXl5nY99WGVlpWKxWOetqKioO5cEAOijgr8KrqKiQo2NjZ232tra0EsCAPSCbi2g/Px8SVJ9fX2X++vr6zsf+7BoNKrMzMwuNwDAwNetBVRcXKz8/Hxt3bq18754PK5du3ZpxowZ3bkrAEA/Z34V3OnTp3Xw4MHOt2tqarRv3z5lZ2dr9OjRuvvuu/X9739fV1xxhYqLi/Xtb39bhYWFWrBgQXeuGwDQz5kLaPfu3br++us7316xYoUkadGiRVq3bp3uu+8+NTU1aenSpWpoaNC1116rzZs3a/Dgwd23agBAv3dJDyN9Z5XfjwX/7safmzMr/vmvzJn09HZz5gdTnzNnNjdMMWckKUX2U+d3jXkX3uhD3tt7uTnTcVnCnJGkwbXp5kzmO/bjkNJuz3Sk2weEJs3fYp7lUu2ZZJp9fZGk/Ticntlszny26Ig5I0lHT8fMmWvy3jFn3nrf/urfI+9nmTOSNPor/+qVs2AYKQCgT6OAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAIz1m5A8Nvb/97r9zCg39hzuT8o/3PUZxeeMqc+dnR68yZxja/P5VxR9Eb5syJtmHmTM3gpDmjDvtkZklqu8y+r8RX/mTOjIo1mjMjo6fNmWiqfaK6JGUNsk+cTniM0G7qiJozMzOr7ftJ2vcjSW8OGm/OpMp+Dg0b1GbOvDR9jTkjSbfefq85E3typ9e+LoQrIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIYsAMI225Ybo5kxbZ57evb+WbMzn/6z1z5uXxz5kzD5+wH4ehKfZBiJK08vUF5kxK3H7KuSyPgZoe80vP7ithzsQPXGbOHGwYYc7U2meeKrXV2UOeOqL2AbDOY2bs6+mfM2duX/yKfUeSrsv6gznzucGHzZmX064yZ/7in+80ZyTpbyp+Zc68/GSm174uhCsgAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAhiwAwjrf98770rWT+sNWf+MudfzJmfNtgHFP5V1j+bM9+t/UtzRpIu/2WqOZMY6jF9UmnmRCTpN4TTpdjX15Fu308yzb4+n7W1Zvkcb0kesYjHzFif/Qz/N/uk2cf/6Xr7jiT9Yf4ac+Y3bfZ3alFsvznzj5mTzRlJ+uvYv5ozv/qzpabtIx2t0r9suuB2XAEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBADZhhp2une29e9l282Zzb8abo5k5seN2e+eei/mjPNf3+5OSNJZ3Lt37/4DMdMbTNHlEz1G8KZ0ksDNX0Gdzr77FevtUlS+xC/nJXPcTg12n7eFVTZ9yNJX5s205wpHNxgzlSfzjNnFl6+15yRpBEp9g/umcuHmbZvT6RKFzF/mSsgAEAQFBAAIAhzAW3fvl033HCDCgsLFYlE9Pzzz3d5fPHixYpEIl1u8+bN6671AgAGCHMBNTU1aerUqVq9evV5t5k3b56OHTvWeXvqqac+0SIBAAOP+UUIZWVlKisr+9htotGo8vPzvRcFABj4euQ5oG3btik3N1cTJkzQsmXLdPLkyfNu29raqng83uUGABj4ur2A5s2bpyeeeEJbt27VD3/4Q1VVVamsrEwdHR3n3L6yslKxWKzzVlRU1N1LAgD0Qd3+e0C33HJL578nT56sKVOmaNy4cdq2bZtmz579ke0rKiq0YsWKzrfj8TglBACXgB5/GfbYsWOVk5OjgwcPnvPxaDSqzMzMLjcAwMDX4wV05MgRnTx5UgUFBT29KwBAP2L+Edzp06e7XM3U1NRo3759ys7OVnZ2th566CEtXLhQ+fn5OnTokO677z6NHz9ec+fO7daFAwD6N3MB7d69W9dff33n2x88f7No0SKtWbNG+/fv189//nM1NDSosLBQc+bM0fe+9z1Fo9HuWzUAoN8zF9CsWbPknDvv4y+//PInWpCvQc29t68DbfbfcXo43z448P+dtj8f1viz/2LOJGN+EysTw+y5yLlfDPmxOtLtGd8hnB9zap9/V700WNRrQKjncRh0xp5J8Rga63M++By7jqjfgfj1+qnmzIYVD5sz/5Q+zpy5esi75owkxZNJc2ZY9QnT9u0drRe1HbPgAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEES3/0nuUIacsE949TUu7bg5c6TdPl74vpfuNGcyRti/p0gMM0ckSelxe8Z5fMvTMdiekcdUa0lqH+qxK4/pzD7r85m67Ssx3L7ApMdXk9Q2+5TqlIsbtNxF/FN+07AzDtuPw7cOzzdn/u+4LebMtjMeJ6ukwtRT5kzHgXds27vERW3HFRAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABDFghpFmHjptziRch9e+xqa1mDMzdywzZ/LfsA9C/NNEc8R7yGVLjj3TEbW/T9H3PQZJen5r5TMstX2o/X1qz7CfeynDL27A43+WbPWZlCpFj6aZM2mn7R8nr2PnMZw2tdVvGOmZXHvunfVXmDO/u/8fzBlpuEdGuixliDmTOmG8aXvX0SoduPB2XAEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBADZhhpSnObOZMW8RvU+K9tmebM8F/ZBwf+aaLHAMWkPdIxxD4QUpLasuw7i56wH/P0U/b1pS84bs5I0snGYeZMR5v90yj9cNScyd5u/34x4veh1Zls+7nXXOgxWNRjGKl85or6zSJVIsO+viEp9o/T/J13mjO/nPETc0aS2mU/9yIJ28TiSPLitucKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCGDDDSNXeYY40Js947epvd37NnEkbZZ+G2Jpjf59SWu37iST8JjWm5zebM2N/FDdn2t87Ys7UR0vMGUkav/WkPXS01p7JHWGOvHdTrjnTPMY2RPIDkaH24b7ujMdw36TH+dpuz3Sk+k1ldYPsueZR9kzGTvuw4rrpQ80ZSRqXZr/uaH/nXdv2LnFR23EFBAAIggICAARhKqDKykpdffXVysjIUG5urhYsWKDq6uou27S0tKi8vFwjRozQ8OHDtXDhQtXX13frogEA/Z+pgKqqqlReXq6dO3fqlVdeUSKR0Jw5c9TU1NS5zT333KMXXnhBGzZsUFVVlY4ePaqbbrqp2xcOAOjfTC9C2Lx5c5e3161bp9zcXO3Zs0czZ85UY2OjfvrTn2r9+vX60pe+JElau3atPv3pT2vnzp36whe+0H0rBwD0a5/oOaDGxkZJUnZ2tiRpz549SiQSKi0t7dxm4sSJGj16tHbs2HHO/6O1tVXxeLzLDQAw8HkXUDKZ1N13361rrrlGkyZNkiTV1dUpPT1dWVlZXbbNy8tTXV3dOf+fyspKxWKxzltRUZHvkgAA/Yh3AZWXl+vtt9/W008//YkWUFFRocbGxs5bba3H71QAAPodr19EXb58uV588UVt375do0aN6rw/Pz9fbW1tamho6HIVVF9fr/z8/HP+X9FoVNFo1GcZAIB+zHQF5JzT8uXLtXHjRr366qsqLi7u8vi0adOUlpamrVu3dt5XXV2tw4cPa8aMGd2zYgDAgGC6AiovL9f69eu1adMmZWRkdD6vE4vFNGTIEMViMX31q1/VihUrlJ2drczMTN11112aMWMGr4ADAHRhKqA1a9ZIkmbNmtXl/rVr12rx4sWSpB/96EdKSUnRwoUL1draqrlz5+onP/lJtywWADBwmArIuQsP2Rs8eLBWr16t1atXey/KS6r99RQvNo268Ebn4JIeGY95n6nNHkMDh3sMMI34vRYl2Wp/CrH5ypHmTHrNe+bM5c8eMmckqf4vxpozJ6/JMGeyRpw2Z1pP24fnRt5PN2ckSQ1p9n35zLT1OfV8Mn6zSBVJ2HfmhtsHwDYX2vfz11v+1pyRpJq//D/mTOqIbNP2LtkmvX/h7ZgFBwAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCC8/iJqX9TxuwPmzLCUVq99PXHNT82Zv04sNWcizanmTGosYc4kO/wmJrsW++kTX95ozqTcdaU509xqn+YsSc6dsofeH2KONL6bZc5E7IPOpTTPMdD2U89r4rRL9Qh5TKOP+Iyjl+QG2w96aoP986JjmP2dSq/vvS/fLX9WfOGN/pP29hbptQtvxxUQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAAQxYIaR+shKafbKpXpMXfzRdU+bM4WD/mTO/OLkn5szL74xzZyRJHXYBzy+fyTLnElttn+f5HrxW6uIx0BNl2YfPun8ZsZ6ibR7DO/0mffpMyvVYz8uxXMoq8c5nox67ssoJeE3YLU52WbOtGXZqqI9cXHbcwUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFc0sNIt5y6yis3Zehhc+Zyj8GiE9LazZlPDT5pzvy3Wf9kzkjS+t9cbQ8dHmKOpNgPgxIx+7BPSYp4DJ+MJO2ZlDN+gyStXO/sRpIU8ZjB6VLsC/QZNOuztrP78nmnfM4h+24GtdgzklTT3mHODD6RMG3f3n5x23MFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBXNrDSI9O8MrFRjebM+PS/mjO/CJ+pTmTFrEPGizL2G/OSNKUz9eaMyNKTpszean2zMb4n5kzkvTSUfuA2vak/fu45rY0cybpsZ9omm2I5AeGeAzC9TkOg1LsUzh95oqebol6pKTYEPvEz8LhjeZMW0eqOZP0mcoq6Y0z48yZYzMGm7bvaJV0ETOOuQICAARBAQEAgjAVUGVlpa6++mplZGQoNzdXCxYsUHV1dZdtZs2apUgk0uV25513duuiAQD9n6mAqqqqVF5erp07d+qVV15RIpHQnDlz1NTU1GW7JUuW6NixY523VatWdeuiAQD9n+lFCJs3b+7y9rp165Sbm6s9e/Zo5syZnfcPHTpU+fn53bNCAMCA9ImeA2psPPtqj+zs7C73P/nkk8rJydGkSZNUUVGh5ubzv2qstbVV8Xi8yw0AMPB5vww7mUzq7rvv1jXXXKNJkyZ13n/bbbdpzJgxKiws1P79+3X//ferurpazz333Dn/n8rKSj300EO+ywAA9FPeBVReXq63335br7/+epf7ly5d2vnvyZMnq6CgQLNnz9ahQ4c0btxHX39eUVGhFStWdL4dj8dVVFTkuywAQD/hVUDLly/Xiy++qO3bt2vUqFEfu21JSYkk6eDBg+csoGg0qmjU75fEAAD9l6mAnHO66667tHHjRm3btk3FxcUXzOzbt0+SVFBQ4LVAAMDAZCqg8vJyrV+/Xps2bVJGRobq6uokSbFYTEOGDNGhQ4e0fv16ffnLX9aIESO0f/9+3XPPPZo5c6amTJnSI+8AAKB/MhXQmjVrJJ39ZdP/bO3atVq8eLHS09O1ZcsWPfroo2pqalJRUZEWLlyoBx54oNsWDAAYGMw/gvs4RUVFqqqq+kQLAgBcGi7padhDPCcFL8v6jTlzMBExZ8qz7NOm/dgn8Z5l/52t5mSbOTM0Zag58+mc6gtvdA7LLttrzlyWal+fjzda7JOjG5J+a8tPtX9sB3tMYu+Q/fOixdnP15EpreaMJBWnDTdn9rTaz/HxafZjt6Mly5yRpPV/LDFnRlW+adq+3SV04CK2YxgpACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARxaQ8j/brfX2Itm/B1cybaYB98mkzrne8POqJ++zlyvT0XybMPhcx4c4g5U/jLo+aMJLlU+/vUcdkwcyb1VIs5o2PHzRGXaLfvR1JkqH2IaWS4x+DTC0zYP6d2++BOX81X2f+Qps/n7fA9h82Z9mN15sxZ9kGzPYUrIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEESfmwXn/n02VLsSkseYKNO+OuxzySSpPWGf45Xa7jELLtJLs+BS/PaTbPGYBddsP+YdbRFzpj3p97F1Ht+TdbSn2vfjc+65NnvEec6CS9q/NESS9uPgNQsu2Xuz4Nrb7Z/rSY9zqD1p/9i2O/vXlN7SrrNrcxf4+EbchbboZUeOHFFRUVHoZQAAPqHa2lqNGjXqvI/3uQJKJpM6evSoMjIyFIl0/c43Ho+rqKhItbW1yszMDLTC8DgOZ3EczuI4nMVxOKsvHAfnnE6dOqXCwkKlfMxPWPrcj+BSUlI+tjElKTMz85I+wT7AcTiL43AWx+EsjsNZoY9DLBa74Da8CAEAEAQFBAAIol8VUDQa1cqVKxWN+v0l04GC43AWx+EsjsNZHIez+tNx6HMvQgAAXBr61RUQAGDgoIAAAEFQQACAICggAEAQ/aaAVq9erU996lMaPHiwSkpK9Otf/zr0knrdd77zHUUikS63iRMnhl5Wj9u+fbtuuOEGFRYWKhKJ6Pnnn+/yuHNODz74oAoKCjRkyBCVlpbqwIEDYRbbgy50HBYvXvyR82PevHlhFttDKisrdfXVVysjI0O5ublasGCBqquru2zT0tKi8vJyjRgxQsOHD9fChQtVX18faMU942KOw6xZsz5yPtx5552BVnxu/aKAnnnmGa1YsUIrV67UW2+9palTp2ru3Lk6fvx46KX1uquuukrHjh3rvL3++uuhl9TjmpqaNHXqVK1evfqcj69atUo//vGP9fjjj2vXrl0aNmyY5s6dq5YW+yDJvuxCx0GS5s2b1+X8eOqpp3pxhT2vqqpK5eXl2rlzp1555RUlEgnNmTNHTU1Nndvcc889euGFF7RhwwZVVVXp6NGjuummmwKuuvtdzHGQpCVLlnQ5H1atWhVoxefh+oHp06e78vLyzrc7OjpcYWGhq6ysDLiq3rdy5Uo3derU0MsISpLbuHFj59vJZNLl5+e7hx9+uPO+hoYGF41G3VNPPRVghb3jw8fBOecWLVrk5s+fH2Q9oRw/ftxJclVVVc65sx/7tLQ0t2HDhs5tfve73zlJbseOHaGW2eM+fBycc+6LX/yi+/rXvx5uURehz18BtbW1ac+ePSotLe28LyUlRaWlpdqxY0fAlYVx4MABFRYWauzYsbr99tt1+PDh0EsKqqamRnV1dV3Oj1gsppKSkkvy/Ni2bZtyc3M1YcIELVu2TCdPngy9pB7V2NgoScrOzpYk7dmzR4lEosv5MHHiRI0ePXpAnw8fPg4fePLJJ5WTk6NJkyapoqJCzc3NIZZ3Xn1uGOmHnThxQh0dHcrLy+tyf15enn7/+98HWlUYJSUlWrdunSZMmKBjx47poYce0nXXXae3335bGRkZoZcXRF1dnSSd8/z44LFLxbx583TTTTepuLhYhw4d0re+9S2VlZVpx44dSk31+Fs9fVwymdTdd9+ta665RpMmTZJ09nxIT09XVlZWl20H8vlwruMgSbfddpvGjBmjwsJC7d+/X/fff7+qq6v13HPPBVxtV32+gPAfysrKOv89ZcoUlZSUaMyYMXr22Wf11a9+NeDK0Bfccsstnf+ePHmypkyZonHjxmnbtm2aPXt2wJX1jPLycr399tuXxPOgH+d8x2Hp0qWd/548ebIKCgo0e/ZsHTp0SOPGjevtZZ5Tn/8RXE5OjlJTUz/yKpb6+nrl5+cHWlXfkJWVpSuvvFIHDx4MvZRgPjgHOD8+auzYscrJyRmQ58fy5cv14osv6rXXXuvy51vy8/PV1tamhoaGLtsP1PPhfMfhXEpKSiSpT50Pfb6A0tPTNW3aNG3durXzvmQyqa1bt2rGjBkBVxbe6dOndejQIRUUFIReSjDFxcXKz8/vcn7E43Ht2rXrkj8/jhw5opMnTw6o88M5p+XLl2vjxo169dVXVVxc3OXxadOmKS0trcv5UF1drcOHDw+o8+FCx+Fc9u3bJ0l963wI/SqIi/H000+7aDTq1q1b537729+6pUuXuqysLFdXVxd6ab3qG9/4htu2bZurqalxb7zxhistLXU5OTnu+PHjoZfWo06dOuX27t3r9u7d6yS5Rx55xO3du9e99957zjnnfvCDH7isrCy3adMmt3//fjd//nxXXFzszpw5E3jl3evjjsOpU6fcvffe63bs2OFqamrcli1b3Oc+9zl3xRVXuJaWltBL7zbLli1zsVjMbdu2zR07dqzz1tzc3LnNnXfe6UaPHu1effVVt3v3bjdjxgw3Y8aMgKvufhc6DgcPHnTf/e533e7du11NTY3btGmTGzt2rJs5c2bglXfVLwrIOecee+wxN3r0aJeenu6mT5/udu7cGXpJve7mm292BQUFLj093V1++eXu5ptvdgcPHgy9rB732muvOUkfuS1atMg5d/al2N/+9rddXl6ei0ajbvbs2a66ujrsonvAxx2H5uZmN2fOHDdy5EiXlpbmxowZ45YsWTLgvkk71/svya1du7ZzmzNnzrivfe1r7rLLLnNDhw51N954ozt27Fi4RfeACx2Hw4cPu5kzZ7rs7GwXjUbd+PHj3Te/+U3X2NgYduEfwp9jAAAE0eefAwIADEwUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACOL/AyBNQnoqGwl/AAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -2009,7 +1840,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[-1.3527451753616333, -3.264960765838623, 0.6525070071220398, -2.173452138900757, 0.7591032385826111, 1.0686758756637573, 0.4610092043876648, 0.4507816433906555, 3.1264138221740723, 1.1761378049850464]\n", + "[-1.0776339769363403, -3.4281859397888184, 1.0321333408355713, -2.1151161193847656, 0.7665405869483948, 0.7089913487434387, 0.6775667071342468, 0.3138602077960968, 2.9969606399536133, 0.7927607297897339]\n", "predicted label: Bag\n" ] } @@ -2066,8 +1897,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 231 ms, sys: 75.8 ms, total: 307 ms\n", - "Wall time: 3.22 s\n" + "CPU times: user 225 ms, sys: 91.1 ms, total: 316 ms\n", + "Wall time: 3.16 s\n" ] } ], @@ -2082,12 +1913,19 @@ "id": "0b3fb48b-f871-41f2-ac57-346899a6fe48", "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 292 ms, sys: 65.2 ms, total: 357 ms\n", - "Wall time: 1.48 s\n" + "CPU times: user 283 ms, sys: 67.8 ms, total: 351 ms\n", + "Wall time: 1.47 s\n" ] } ], @@ -2106,8 +1944,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 235 ms, sys: 75.2 ms, total: 310 ms\n", - "Wall time: 1.2 s\n" + "CPU times: user 543 ms, sys: 65.1 ms, total: 608 ms\n", + "Wall time: 1.36 s\n" ] } ], @@ -2177,7 +2015,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAelElEQVR4nO3dfXCU9b338c9mk2wChI0h5KkEGlChlYeeUkm5VYolA6RzPKJMx6c/wHFgtMEpUquTjorazqTFGevoUDx/tFDvW3yaERg9vekomjC2gQ4oN4fTNkJOLHhCgtLmgYQ8kP2dPzhu74UA/V1s8t2E92vmmsnuXt9cX669yGev7LXfhJxzTgAADLM06wYAAFcmAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAm0q0bOFcsFlNzc7NycnIUCoWs2wEAeHLOqbOzUyUlJUpLu/B5TsoFUHNzs0pLS63bAABcpmPHjmnSpEkXfDzlAignJ0eSdKO+o3RlGHeDi0mf/CXvmrL/fcK7Zvd/TfOuyco4410jSWcGhue30gNueM7uY7Fg/56M8IB3TUd7tnfN3KlHvWva/7nbu8b19nrXILgz6tcH+k385/mFDFkAbdy4Uc8884xaWlo0Z84cvfDCC5o3b94l6774tVu6MpQeIoBSWXpaxLsmc5z/cxoe47+dcEbYu0aS3DAFkIYpgEIBAygcIIDS+rK8azLGZnrXpIf6vWtcKOZdg8vwPxNGL/U2ypD8b3vttde0bt06rV+/Xh9++KHmzJmjJUuW6MQJ/1e/AIDRaUgC6Nlnn9WqVat077336qtf/apefPFFjRkzRr/61a+GYnMAgBEo6QHU19en/fv3q6Ki4u8bSUtTRUWF6uvrz1u/t7dXHR0dCQsAYPRLegB9/vnnGhgYUGFhYcL9hYWFamlpOW/9mpoaRaPR+MIVcABwZTD/IGp1dbXa29vjy7Fjx6xbAgAMg6RfBZefn69wOKzW1taE+1tbW1VUVHTe+pFIRJGI/1VOAICRLelnQJmZmZo7d6527doVvy8Wi2nXrl2aP39+sjcHABihhuRzQOvWrdOKFSv0jW98Q/PmzdNzzz2nrq4u3XvvvUOxOQDACDQkAXTHHXfos88+0xNPPKGWlhZ97Wtf086dO8+7MAEAcOUaskkIa9as0Zo1a4bq2yMF/PUG/1E8bxa/6V3zTGand01Z5DPvGkkKy/8T82kBPmU/Ns1/NMyA8/+NeTjgBID/1z3Fu2bP38q8a/51yr951/zL4u9712S99QfvGgw986vgAABXJgIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACaGbBgpRr/2qf6vX/b1hr1rmntzvWvS5LxrJCmmkHdNTyzDuyaa3u1dkxU6410TZFCqJDV2T/SuOdaW613z2+7z/0jlpbRd7f9jy38rGA6cAQEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATDANG4H1FA1410zPOO1dUxJp867Jz+j0rpGkE/3jvWsyQv77oT/m/1+v20W8a3LCPd41klSc1e5d840i/wnkszKPe9f05XiXIEVxBgQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEw0gRWGZBt3dNj/MfWDkuwEDNfhf2rpGkMWl93jWdA1neNe0D2d41n/eO8665Kfdj7xop2D4Ph2IBavyPh1jEvwapiTMgAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJhhGisDGZvd61zQPRLxrYs7/dVK/d8VZGaGBYanpjfn/1ysb87l3TdvAGO+aoE6d8X9um8/keNf0FQd9dpFqOAMCAJgggAAAJpIeQE8++aRCoVDCMmPGjGRvBgAwwg3Je0DXXXed3n333b9vJJ23mgAAiYYkGdLT01VUVDQU3xoAMEoMyXtAhw8fVklJiaZOnap77rlHR48eveC6vb296ujoSFgAAKNf0gOovLxcW7Zs0c6dO7Vp0yY1NTXppptuUmdn56Dr19TUKBqNxpfS0tJktwQASEFJD6DKykp997vf1ezZs7VkyRL95je/UVtbm15//fVB16+urlZ7e3t8OXbsWLJbAgCkoCG/OiA3N1fXXnutjhw5MujjkUhEkYj/B9gAACPbkH8O6NSpU2psbFRxcfFQbwoAMIIkPYAefvhh1dXV6ZNPPtHvf/973XbbbQqHw7rrrruSvSkAwAiW9F/Bffrpp7rrrrt08uRJTZw4UTfeeKP27NmjiRMnJntTAIARLOkB9Oqrryb7WyJF5UT6vGsyFfOuSQv512SFgg2s7Hf+/yWmRk5411yd1eJd09x/lXdNd4Dhr5KUlea//3pjGd41HbEs75rMsf7HHVITs+AAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYGPI/SIfRKxRy3jVdzn9gZfuZMd41Su/2r1GwIaY54dPeNT/++J+9a743tda75uPuIu8aScoNsP+6BjIDbctXJBJs0CxSD2dAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATTMNGYD1n/A+fgQCveZp7o941QU3MavWuydCAd030O0e8a2Y0Hfeu2dN5tXeNJLUFmEDe2R/xrgkyfTwW43XzaMEzCQAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwATDSBHYqR7/4ZNjQ33eNTHn/zrpk9MTvGsk6Z/GfOJdExum13GfDeR415RlfxZoW3v/VuZdc6Lbv7/MkP8g156eDO8apCbOgAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJhgGCkC6+7yH0aaEYoNQSfnG3ChQHUF4U7vmv9z8n8F2FKvd8XuzhneNf8S/dC7RpL+rXmWd03PGf8fJzlpPd41A6f5sTVacAYEADBBAAEATHgH0O7du3XLLbeopKREoVBI27dvT3jcOacnnnhCxcXFys7OVkVFhQ4fPpysfgEAo4R3AHV1dWnOnDnauHHjoI9v2LBBzz//vF588UXt3btXY8eO1ZIlS9TT4/+7XgDA6OX9bl5lZaUqKysHfcw5p+eee06PPfaYbr31VknSSy+9pMLCQm3fvl133nnn5XULABg1kvoeUFNTk1paWlRRURG/LxqNqry8XPX19YPW9Pb2qqOjI2EBAIx+SQ2glpYWSVJhYWHC/YWFhfHHzlVTU6NoNBpfSktLk9kSACBFmV8FV11drfb29vhy7Ngx65YAAMMgqQFUVFQkSWptbU24v7W1Nf7YuSKRiMaPH5+wAABGv6QGUFlZmYqKirRr1674fR0dHdq7d6/mz5+fzE0BAEY476vgTp06pSNHjsRvNzU16cCBA8rLy9PkyZO1du1a/eQnP9E111yjsrIyPf744yopKdGyZcuS2TcAYITzDqB9+/bp5ptvjt9et26dJGnFihXasmWLHnnkEXV1dWn16tVqa2vTjTfeqJ07dyorKyt5XQMARjzvAFq4cKGccxd8PBQK6emnn9bTTz99WY0h9cW6/IdCZmh4hpEGNTOz37tm58df9a6Zpo+8a+pPlHnX3J/3gXeNJPXH/H87n5V+xr8mNOBdE+oOe9cgNZlfBQcAuDIRQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEz4jzMG/kdal/9U4rFpwzMN+0ws2MTkcWn+fzYkoyE70LZ8HWvO864pvC4z0La6e/3r8sZ2e9eMCTANO3ya182jBc8kAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwwjRWDhnpB3TVbIv+b0QIZ3TX7klHdNUNH/HJ4Bq9mNEe+a8GL//R1UWsh512QEaC90xr8GqYkzIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYYRorAwr3+kyT7nP/Aykia//TJIIMxgxrf1DMs2xnbPHz/pjGRPu+asen+NWHvimBDcJGaOAMCAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABggmGkCCz99PBsJyb/4ZPRgM31uwHvmvSP/8u7xn8r0vhP/Id9BjUl52/eNX0x/9GiGSH/5zY8fLsBQ4wzIACACQIIAGDCO4B2796tW265RSUlJQqFQtq+fXvC4ytXrlQoFEpYli5dmqx+AQCjhHcAdXV1ac6cOdq4ceMF11m6dKmOHz8eX1555ZXLahIAMPp4X4RQWVmpysrKi64TiURUVFQUuCkAwOg3JO8B1dbWqqCgQNOnT9cDDzygkydPXnDd3t5edXR0JCwAgNEv6QG0dOlSvfTSS9q1a5d+9rOfqa6uTpWVlRoYGPzC05qaGkWj0fhSWlqa7JYAACko6Z8DuvPOO+Nfz5o1S7Nnz9a0adNUW1urRYsWnbd+dXW11q1bF7/d0dFBCAHAFWDIL8OeOnWq8vPzdeTIkUEfj0QiGj9+fMICABj9hjyAPv30U508eVLFxcVDvSkAwAji/Su4U6dOJZzNNDU16cCBA8rLy1NeXp6eeuopLV++XEVFRWpsbNQjjzyiq6++WkuWLElq4wCAkc07gPbt26ebb745fvuL929WrFihTZs26eDBg/r1r3+ttrY2lZSUaPHixfrxj3+sSCSSvK4BACOedwAtXLhQzrkLPv7b3/72shrCyBEOMO8zJ81/YGXvgP+1MuPCPd41QQ189tmwbCfrkwt/nCHZ8jK7vWv+2jdmCDo5Xyg2LJvBMGAWHADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADARNL/JDeuHOmnLzwV/UIy5D8NO4iw/HtLdQPHmodtWxMyT3nXBJmGnRHyfw0cOuNdghTFGRAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATDCNFYJmd/gM/w6GQd81Xc45716SFYt41kpQRGp5hqUG4/r5h29ZV6V3eNdPHtXrXZIX8fwRldI2+QbNXKs6AAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmGAYKQJL7/Uf+JkW4DVPNHzauyaS1u9dI0n9biBQXarqjgXbDzlpPd414fThGRLKMNLRgzMgAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJhhGipSXETrjXTM2rTfQtmLyH7CayvoVbHBnkP3X78KBtuUro5thpKMFZ0AAABMEEADAhFcA1dTU6Prrr1dOTo4KCgq0bNkyNTQ0JKzT09OjqqoqTZgwQePGjdPy5cvV2tqa1KYBACOfVwDV1dWpqqpKe/bs0TvvvKP+/n4tXrxYXV1d8XUeeughvfXWW3rjjTdUV1en5uZm3X777UlvHAAwsnldhLBz586E21u2bFFBQYH279+vBQsWqL29Xb/85S+1detWffvb35Ykbd68WV/5yle0Z88effOb30xe5wCAEe2y3gNqb2+XJOXl5UmS9u/fr/7+flVUVMTXmTFjhiZPnqz6+vpBv0dvb686OjoSFgDA6Bc4gGKxmNauXasbbrhBM2fOlCS1tLQoMzNTubm5CesWFhaqpaVl0O9TU1OjaDQaX0pLS4O2BAAYQQIHUFVVlQ4dOqRXX331shqorq5We3t7fDl27NhlfT8AwMgQ6IOoa9as0dtvv63du3dr0qRJ8fuLiorU19entra2hLOg1tZWFRUVDfq9IpGIIpFIkDYAACOY1xmQc05r1qzRtm3b9N5776msrCzh8blz5yojI0O7du2K39fQ0KCjR49q/vz5yekYADAqeJ0BVVVVaevWrdqxY4dycnLi7+tEo1FlZ2crGo3qvvvu07p165SXl6fx48frwQcf1Pz587kCDgCQwCuANm3aJElauHBhwv2bN2/WypUrJUk///nPlZaWpuXLl6u3t1dLlizRL37xi6Q0CwAYPbwCyLlLDwHMysrSxo0btXHjxsBNYWRI7x6ewZ0DAa6VCTLAVJJ6XLC6VNUZCza4My3k/9xmhAb8txPguc1sH13P0ZWMWXAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABOB/iIqIEnh7tSdShxWsCnQfx3wn+icyj4byA5UF3T/DYfMk6e9a4Znbjt8cQYEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABMNIkfJizv91UkYo2KDUf+8rCFSXqloGooHqstL6vGvCsaxA2/KV9lmbdw3DSFMTZ0AAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMMIwUgYUGnHdNfW94CDpJnv8cZcNIG3qKA9X9U/Yn3jVpAUZ+/t/uHO8a19XlXYPUxBkQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwwjRWCxLP/BohPSTnvXFGa0eddMTv+bd40ktZzJDVSXqv7jVLBhpJU5/+5d0+0i3jW54W7vGqXzY2u04AwIAGCCAAIAmPAKoJqaGl1//fXKyclRQUGBli1bpoaGhoR1Fi5cqFAolLDcf//9SW0aADDyeQVQXV2dqqqqtGfPHr3zzjvq7+/X4sWL1XXOH4hatWqVjh8/Hl82bNiQ1KYBACOf17t5O3fuTLi9ZcsWFRQUaP/+/VqwYEH8/jFjxqioqCg5HQIARqXLeg+ovb1dkpSXl5dw/8svv6z8/HzNnDlT1dXV6u6+8JUuvb296ujoSFgAAKNf4OsZY7GY1q5dqxtuuEEzZ86M33/33XdrypQpKikp0cGDB/Xoo4+qoaFBb7755qDfp6amRk899VTQNgAAI1TgAKqqqtKhQ4f0wQcfJNy/evXq+NezZs1ScXGxFi1apMbGRk2bNu2871NdXa1169bFb3d0dKi0tDRoWwCAESJQAK1Zs0Zvv/22du/erUmTJl103fLycknSkSNHBg2gSCSiSMT/A2wAgJHNK4Ccc3rwwQe1bds21dbWqqys7JI1Bw4ckCQVFwf7RDYAYHTyCqCqqipt3bpVO3bsUE5OjlpaWiRJ0WhU2dnZamxs1NatW/Wd73xHEyZM0MGDB/XQQw9pwYIFmj179pD8AwAAI5NXAG3atEnS2Q+b/v82b96slStXKjMzU++++66ee+45dXV1qbS0VMuXL9djjz2WtIYBAKOD96/gLqa0tFR1dXWX1RAA4MrAWFkEFopd/AXJYK7LzPau+dXJ8y9euZTmrKu8ayTp5aZ53jV5+jjQtoZDdrg/UN2vTt7oXZMRGvCu+eHEDy690rku8UIYIwfDSAEAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgIuUuNuB5mHR0dikajWqhblR7KsG4HAODpjOtXrXaovb1d48ePv+B6nAEBAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwES6dQPn+mI03Rn1Syk1pQ4A8I84o35Jf/95fiEpF0CdnZ2SpA/0G+NOAACXo7OzU9Fo9IKPp9w07FgspubmZuXk5CgUCiU81tHRodLSUh07duyiE1ZHO/bDWeyHs9gPZ7EfzkqF/eCcU2dnp0pKSpSWduF3elLuDCgtLU2TJk266Drjx4+/og+wL7AfzmI/nMV+OIv9cJb1frjYmc8XuAgBAGCCAAIAmBhRARSJRLR+/XpFIhHrVkyxH85iP5zFfjiL/XDWSNoPKXcRAgDgyjCizoAAAKMHAQQAMEEAAQBMEEAAABMjJoA2btyoL3/5y8rKylJ5ebn+8Ic/WLc07J588kmFQqGEZcaMGdZtDbndu3frlltuUUlJiUKhkLZv357wuHNOTzzxhIqLi5Wdna2KigodPnzYptkhdKn9sHLlyvOOj6VLl9o0O0Rqamp0/fXXKycnRwUFBVq2bJkaGhoS1unp6VFVVZUmTJigcePGafny5WptbTXqeGj8I/th4cKF5x0P999/v1HHgxsRAfTaa69p3bp1Wr9+vT788EPNmTNHS5Ys0YkTJ6xbG3bXXXedjh8/Hl8++OAD65aGXFdXl+bMmaONGzcO+viGDRv0/PPP68UXX9TevXs1duxYLVmyRD09PcPc6dC61H6QpKVLlyYcH6+88sowdjj06urqVFVVpT179uidd95Rf3+/Fi9erK6urvg6Dz30kN566y298cYbqqurU3Nzs26//XbDrpPvH9kPkrRq1aqE42HDhg1GHV+AGwHmzZvnqqqq4rcHBgZcSUmJq6mpMexq+K1fv97NmTPHug1Tkty2bdvit2OxmCsqKnLPPPNM/L62tjYXiUTcK6+8YtDh8Dh3Pzjn3IoVK9ytt95q0o+VEydOOEmurq7OOXf2uc/IyHBvvPFGfJ0//elPTpKrr6+3anPInbsfnHPuW9/6lvv+979v19Q/IOXPgPr6+rR//35VVFTE70tLS1NFRYXq6+sNO7Nx+PBhlZSUaOrUqbrnnnt09OhR65ZMNTU1qaWlJeH4iEajKi8vvyKPj9raWhUUFGj69Ol64IEHdPLkSeuWhlR7e7skKS8vT5K0f/9+9ff3JxwPM2bM0OTJk0f18XDufvjCyy+/rPz8fM2cOVPV1dXq7u62aO+CUm4Y6bk+//xzDQwMqLCwMOH+wsJC/fnPfzbqykZ5ebm2bNmi6dOn6/jx43rqqad000036dChQ8rJybFuz0RLS4skDXp8fPHYlWLp0qW6/fbbVVZWpsbGRv3oRz9SZWWl6uvrFQ6HrdtLulgsprVr1+qGG27QzJkzJZ09HjIzM5Wbm5uw7mg+HgbbD5J09913a8qUKSopKdHBgwf16KOPqqGhQW+++aZht4lSPoDwd5WVlfGvZ8+erfLyck2ZMkWvv/667rvvPsPOkAruvPPO+NezZs3S7NmzNW3aNNXW1mrRokWGnQ2NqqoqHTp06Ip4H/RiLrQfVq9eHf961qxZKi4u1qJFi9TY2Khp06YNd5uDSvlfweXn5yscDp93FUtra6uKioqMukoNubm5uvbaa3XkyBHrVsx8cQxwfJxv6tSpys/PH5XHx5o1a/T222/r/fffT/jzLUVFRerr61NbW1vC+qP1eLjQfhhMeXm5JKXU8ZDyAZSZmam5c+dq165d8ftisZh27dql+fPnG3Zm79SpU2psbFRxcbF1K2bKyspUVFSUcHx0dHRo7969V/zx8emnn+rkyZOj6vhwzmnNmjXatm2b3nvvPZWVlSU8PnfuXGVkZCQcDw0NDTp69OioOh4utR8Gc+DAAUlKrePB+iqIf8Srr77qIpGI27Jli/vjH//oVq9e7XJzc11LS4t1a8PqBz/4gautrXVNTU3ud7/7nauoqHD5+fnuxIkT1q0Nqc7OTvfRRx+5jz76yElyzz77rPvoo4/cX/7yF+eccz/96U9dbm6u27Fjhzt48KC79dZbXVlZmTt9+rRx58l1sf3Q2dnpHn74YVdfX++amprcu+++677+9a+7a665xvX09Fi3njQPPPCAi0ajrra21h0/fjy+dHd3x9e5//773eTJk917773n9u3b5+bPn+/mz59v2HXyXWo/HDlyxD399NNu3759rqmpye3YscNNnTrVLViwwLjzRCMigJxz7oUXXnCTJ092mZmZbt68eW7Pnj3WLQ27O+64wxUXF7vMzEz3pS99yd1xxx3uyJEj1m0Nuffff99JOm9ZsWKFc+7spdiPP/64KywsdJFIxC1atMg1NDTYNj0ELrYfuru73eLFi93EiRNdRkaGmzJlilu1atWoe5E22L9fktu8eXN8ndOnT7vvfe977qqrrnJjxoxxt912mzt+/Lhd00PgUvvh6NGjbsGCBS4vL89FIhF39dVXux/+8Ieuvb3dtvFz8OcYAAAmUv49IADA6EQAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMDEfwNH5VgkZZXaeQAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAHpRJREFUeJzt3X1wlPW99/HPZpNsAoSNIeSpBBpQoZWHnlJJuVWKJQOkczyiTMenP8BxYLTBKVKrk46K2s6kxRnr6FA8f7RQ71t8mhEYPb3pKJowtoEOKDeH0zZCTix4QoLS5oGEPJD9nT84bu+FAP1dbPLdhPdr5prJ7l7fXF+uvchnr+y134Scc04AAAyzNOsGAABXJgIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJtKtGzhXLBZTc3OzcnJyFAqFrNsBAHhyzqmzs1MlJSVKS7vweU7KBVBzc7NKS0ut2wAAXKZjx45p0qRJF3w85QIoJydHknSjvqN0ZRh3g4tJn/wl75qy/33Cu2b3f03zrsnKOONdI0lnBobnt9IDbnjO7mOxYP+ejPCAd01He7Z3zdypR71r2v+527vG9fZ61yC4M+rXB/pN/Of5hQxZAG3cuFHPPPOMWlpaNGfOHL3wwguaN2/eJeu++LVbujKUHiKAUll6WsS7JnOc/3MaHuO/nXBG2LtGktwwBZCGKYBCAQMoHCCA0vqyvGsyxmZ616SH+r1rXCjmXYPL8D8TRi/1NsqQ/G977bXXtG7dOq1fv14ffvih5syZoyVLlujECf9XvwCA0WlIAujZZ5/VqlWrdO+99+qrX/2qXnzxRY0ZM0a/+tWvhmJzAIARKOkB1NfXp/3796uiouLvG0lLU0VFherr689bv7e3Vx0dHQkLAGD0S3oAff755xoYGFBhYWHC/YWFhWppaTlv/ZqaGkWj0fjCFXAAcGUw/yBqdXW12tvb48uxY8esWwIADIOkXwWXn5+vcDis1tbWhPtbW1tVVFR03vqRSESRiP9VTgCAkS3pZ0CZmZmaO3eudu3aFb8vFotp165dmj9/frI3BwAYoYbkc0Dr1q3TihUr9I1vfEPz5s3Tc889p66uLt17771DsTkAwAg0JAF0xx136LPPPtMTTzyhlpYWfe1rX9POnTvPuzABAHDlGrJJCGvWrNGaNWuG6tsjBfz1Bv9RPG8Wv+ld80xmp3dNWeQz7xpJCsv/E/NpAT5lPzbNfzTMgPP/jXk44ASA/9c9xbtmz9/KvGv+dcq/edf8y+Lve9dkvfUH7xoMPfOr4AAAVyYCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmhmwYKUa/9qn+r1/29Ya9a5p7c71r0uS8ayQpppB3TU8sw7smmt7tXZMVOuNdE2RQqiQ1dk/0rjnWlutd89vu8/9I5aW0Xe3/Y8t/KxgOnAEBAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwwDRuB9RQNeNdMzzjtXVMSafOuyc/o9K6RpBP9471rMkL++6E/5v9fr9tFvGtywj3eNZJUnNXuXfONIv8J5LMyj3vX9OV4lyBFcQYEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABMNIEVhmQbd3TY/zH1g5LsBAzX4X9q6RpDFpfd41nQNZ3jXtA9neNZ/3jvOuuSn3Y+8aKdg+D4diAWr8j4dYxL8GqYkzIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYYRorAxmb3etc0D0S8a2LO/3VSv3fFWRmhgWGp6Y35/9crG/O5d03bwBjvmqBOnfF/bpvP5HjX9BUHfXaRajgDAgCYIIAAACaSHkBPPvmkQqFQwjJjxoxkbwYAMMINyXtA1113nd59992/bySdt5oAAImGJBnS09NVVFQ0FN8aADBKDMl7QIcPH1ZJSYmmTp2qe+65R0ePHr3gur29vero6EhYAACjX9IDqLy8XFu2bNHOnTu1adMmNTU16aabblJnZ+eg69fU1CgajcaX0tLSZLcEAEhBSQ+gyspKffe739Xs2bO1ZMkS/eY3v1FbW5tef/31Qdevrq5We3t7fDl27FiyWwIApKAhvzogNzdX1157rY4cOTLo45FIRJGI/wfYAAAj25B/DujUqVNqbGxUcXHxUG8KADCCJD2AHn74YdXV1emTTz7R73//e912220Kh8O66667kr0pAMAIlvRfwX366ae66667dPLkSU2cOFE33nij9uzZo4kTJyZ7UwCAESzpAfTqq68m+1siReVE+rxrMhXzrkkL+ddkhYINrOx3/v8lpkZOeNdcndXiXdPcf5V3TXeA4a+SlJXmv/96YxneNR2xLO+azLH+xx1SE7PgAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmBjyP0iH0SsUct41Xc5/YGX7mTHeNUrv9q9RsCGmOeHT3jU//vifvWu+N7XWu+bj7iLvGknKDbD/ugYyA23LVyQSbNAsUg9nQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAE0zDRmA9Z/wPn4EAr3mae6PeNUFNzGr1rsnQgHdN9DtHvGtmNB33rtnTebV3jSS1BZhA3tkf8a4JMn08FuN182jBMwkAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEw0gR2Kke/+GTY0N93jUx5/866ZPTE7xrJOmfxnziXRMbptdxnw3keNeUZX8WaFt7/1bmXXOi27+/zJD/INeengzvGqQmzoAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYYBgpAuvu8h9GmhGKDUEn5xtwoUB1BeFO75r/c/J/BdhSr3fF7s4Z3jX/Ev3Qu0aS/q15lndNzxn/Hyc5aT3eNQOn+bE1WnAGBAAwQQABAEx4B9Du3bt1yy23qKSkRKFQSNu3b0943DmnJ554QsXFxcrOzlZFRYUOHz6crH4BAKOEdwB1dXVpzpw52rhx46CPb9iwQc8//7xefPFF7d27V2PHjtWSJUvU0+P/u14AwOjl/W5eZWWlKisrB33MOafnnntOjz32mG699VZJ0ksvvaTCwkJt375dd9555+V1CwAYNZL6HlBTU5NaWlpUUVERvy8ajaq8vFz19fWD1vT29qqjoyNhAQCMfkkNoJaWFklSYWFhwv2FhYXxx85VU1OjaDQaX0pLS5PZEgAgRZlfBVddXa329vb4cuzYMeuWAADDIKkBVFRUJElqbW1NuL+1tTX+2LkikYjGjx+fsAAARr+kBlBZWZmKioq0a9eu+H0dHR3au3ev5s+fn8xNAQBGOO+r4E6dOqUjR47Ebzc1NenAgQPKy8vT5MmTtXbtWv3kJz/RNddco7KyMj3++OMqKSnRsmXLktk3AGCE8w6gffv26eabb47fXrdunSRpxYoV2rJlix555BF1dXVp9erVamtr04033qidO3cqKysreV0DAEY87wBauHChnHMXfDwUCunpp5/W008/fVmNIfXFuvyHQmZoeIaRBjUzs9+7ZufHX/WumaaPvGvqT5R519yf94F3jST1x/x/O5+Vfsa/JjTgXRPqDnvXIDWZXwUHALgyEUAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBM+I8zBv5HWpf/VOKxacMzDftMLNjE5HFp/n82JKMhO9C2fB1rzvOuKbwuM9C2unv96/LGdnvXjAkwDTt8mtfNowXPJADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMMI0Vg4Z6Qd01WyL/m9ECGd01+5JR3TVDR/xyeAavZjRHvmvBi//0dVFrIeddkBGgvdMa/BqmJMyAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmGEaKwMK9/pMk+5z/wMpImv/0ySCDMYMa39QzLNsZ2zx8/6YxkT7vmrHp/jVh74pgQ3CRmjgDAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIJhpAgs/fTwbCcm/+GT0YDN9bsB75r0j//Lu8Z/K9L4T/yHfQY1Jedv3jV9Mf/Rohkh/+c2PHy7AUOMMyAAgAkCCABgwjuAdu/erVtuuUUlJSUKhULavn17wuMrV65UKBRKWJYuXZqsfgEAo4R3AHV1dWnOnDnauHHjBddZunSpjh8/Hl9eeeWVy2oSADD6eF+EUFlZqcrKyouuE4lEVFRUFLgpAMDoNyTvAdXW1qqgoEDTp0/XAw88oJMnT15w3d7eXnV0dCQsAIDRL+kBtHTpUr300kvatWuXfvazn6murk6VlZUaGBj8wtOamhpFo9H4UlpamuyWAAApKOmfA7rzzjvjX8+aNUuzZ8/WtGnTVFtbq0WLFp23fnV1tdatWxe/3dHRQQgBwBVgyC/Dnjp1qvLz83XkyJFBH49EIho/fnzCAgAY/YY8gD799FOdPHlSxcXFQ70pAMAI4v0ruFOnTiWczTQ1NenAgQPKy8tTXl6ennrqKS1fvlxFRUVqbGzUI488oquvvlpLlixJauMAgJHNO4D27dunm2++OX77i/dvVqxYoU2bNungwYP69a9/rba2NpWUlGjx4sX68Y9/rEgkkryuAQAjnncALVy4UM65Cz7+29/+9rIawsgRDjDvMyfNf2Bl74D/tTLjwj3eNUENfPbZsGwn65MLf5wh2fIyu71r/to3Zgg6OV8oNiybwTBgFhwAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwETS/yQ3rhzppy88Ff1CMuQ/DTuIsPx7S3UDx5qHbVsTMk951wSZhp0R8n8NHDrjXYIUxRkQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwwjRWCZnf4DP8OhkHfNV3OOe9ekhWLeNZKUERqeYalBuP6+YdvWVeld3jXTx7V612SF/H8EZXSNvkGzVyrOgAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJhgGCkCS+/1H/iZFuA1TzR82rsmktbvXSNJ/W4gUF2q6o4F2w85aT3eNeH04RkSyjDS0YMzIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYYRoqUlxE6410zNq030LZi8h+wmsr6FWxwZ5D91+/CgbblK6ObYaSjBWdAAAATBBAAwIRXANXU1Oj6669XTk6OCgoKtGzZMjU0NCSs09PTo6qqKk2YMEHjxo3T8uXL1dramtSmAQAjn1cA1dXVqaqqSnv27NE777yj/v5+LV68WF1dXfF1HnroIb311lt64403VFdXp+bmZt1+++1JbxwAMLJ5XYSwc+fOhNtbtmxRQUGB9u/frwULFqi9vV2//OUvtXXrVn3729+WJG3evFlf+cpXtGfPHn3zm99MXucAgBHtst4Dam9vlyTl5eVJkvbv36/+/n5VVFTE15kxY4YmT56s+vr6Qb9Hb2+vOjo6EhYAwOgXOIBisZjWrl2rG264QTNnzpQktbS0KDMzU7m5uQnrFhYWqqWlZdDvU1NTo2g0Gl9KS0uDtgQAGEECB1BVVZUOHTqkV1999bIaqK6uVnt7e3w5duzYZX0/AMDIEOiDqGvWrNHbb7+t3bt3a9KkSfH7i4qK1NfXp7a2toSzoNbWVhUVFQ36vSKRiCKRSJA2AAAjmNcZkHNOa9as0bZt2/Tee++prKws4fG5c+cqIyNDu3btit/X0NCgo0ePav78+cnpGAAwKnidAVVVVWnr1q3asWOHcnJy4u/rRKNRZWdnKxqN6r777tO6deuUl5en8ePH68EHH9T8+fO5Ag4AkMArgDZt2iRJWrhwYcL9mzdv1sqVKyVJP//5z5WWlqbly5ert7dXS5Ys0S9+8YukNAsAGD28Asi5Sw8BzMrK0saNG7Vx48bATWFkSO8ensGdAwGulQkywFSSelywulTVGQs2uDMt5P/cZoQG/LcT4LnNbB9dz9GVjFlwAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATgf4iKiBJ4e7UnUocVrAp0H8d8J/onMo+G8gOVBd0/w2HzJOnvWuGZ247fHEGBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwATDSJHyYs7/dVJGKNig1H/vKwhUl6paBqKB6rLS+rxrwrGsQNvylfZZm3cNw0hTE2dAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATDCMFIGFBpx3TX1veAg6SZ7/HGXDSBt6igPV/VP2J941aQFGfv7f7hzvGtfV5V2D1MQZEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMMI0VgsSz/waIT0k571xRmtHnXTE7/m3eNJLWcyQ1Ul6r+41SwYaSVOf/uXdPtIt41ueFu7xql82NrtOAMCABgggACAJjwCqCamhpdf/31ysnJUUFBgZYtW6aGhoaEdRYuXKhQKJSw3H///UltGgAw8nkFUF1dnaqqqrRnzx6988476u/v1+LFi9V1zh+IWrVqlY4fPx5fNmzYkNSmAQAjn9e7eTt37ky4vWXLFhUUFGj//v1asGBB/P4xY8aoqKgoOR0CAEaly3oPqL29XZKUl5eXcP/LL7+s/Px8zZw5U9XV1eruvvCVLr29vero6EhYAACjX+DrGWOxmNauXasbbrhBM2fOjN9/9913a8qUKSopKdHBgwf16KOPqqGhQW+++eag36empkZPPfVU0DYAACNU4ACqqqrSoUOH9MEHHyTcv3r16vjXs2bNUnFxsRYtWqTGxkZNmzbtvO9TXV2tdevWxW93dHSotLQ0aFsAgBEiUACtWbNGb7/9tnbv3q1JkyZddN3y8nJJ0pEjRwYNoEgkokjE/wNsAICRzSuAnHN68MEHtW3bNtXW1qqsrOySNQcOHJAkFRcH+0Q2AGB08gqgqqoqbd26VTt27FBOTo5aWlokSdFoVNnZ2WpsbNTWrVv1ne98RxMmTNDBgwf10EMPacGCBZo9e/aQ/AMAACOTVwBt2rRJ0tkPm/7/Nm/erJUrVyozM1PvvvuunnvuOXV1dam0tFTLly/XY489lrSGAQCjg/ev4C6mtLRUdXV1l9UQAODKwFhZBBaKXfwFyWCuy8z2rvnVyfMvXrmU5qyrvGsk6eWmed41efo40LaGQ3a4P1Ddr07e6F2TERrwrvnhxA8uvdK5LvFCGCMHw0gBACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYCLlLjbgeZh0dHYpGo1qoW5UeyrBuBwDg6YzrV612qL29XePHj7/gepwBAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMBEunUD5/piNN0Z9UspNaUOAPCPOKN+SX//eX4hKRdAnZ2dkqQP9BvjTgAAl6Ozs1PRaPSCj6fcNOxYLKbm5mbl5OQoFAolPNbR0aHS0lIdO3bsohNWRzv2w1nsh7PYD2exH85Khf3gnFNnZ6dKSkqUlnbhd3pS7gwoLS1NkyZNuug648ePv6IPsC+wH85iP5zFfjiL/XCW9X642JnPF7gIAQBgggACAJgYUQEUiUS0fv16RSIR61ZMsR/OYj+cxX44i/1w1kjaDyl3EQIA4Mowos6AAACjBwEEADBBAAEATBBAAAATIyaANm7cqC9/+cvKyspSeXm5/vCHP1i3NOyefPJJhUKhhGXGjBnWbQ253bt365ZbblFJSYlCoZC2b9+e8LhzTk888YSKi4uVnZ2tiooKHT582KbZIXSp/bBy5crzjo+lS5faNDtEampqdP311ysnJ0cFBQVatmyZGhoaEtbp6elRVVWVJkyYoHHjxmn58uVqbW016nho/CP7YeHChecdD/fff79Rx4MbEQH02muvad26dVq/fr0+/PBDzZkzR0uWLNGJEyesWxt21113nY4fPx5fPvjgA+uWhlxXV5fmzJmjjRs3Dvr4hg0b9Pzzz+vFF1/U3r17NXbsWC1ZskQ9PT3D3OnQutR+kKSlS5cmHB+vvPLKMHY49Orq6lRVVaU9e/bonXfeUX9/vxYvXqyurq74Og899JDeeustvfHGG6qrq1Nzc7Nuv/12w66T7x/ZD5K0atWqhONhw4YNRh1fgBsB5s2b56qqquK3BwYGXElJiaupqTHsavitX7/ezZkzx7oNU5Lctm3b4rdjsZgrKipyzzzzTPy+trY2F4lE3CuvvGLQ4fA4dz8459yKFSvcrbfeatKPlRMnTjhJrq6uzjl39rnPyMhwb7zxRnydP/3pT06Sq6+vt2pzyJ27H5xz7lvf+pb7/ve/b9fUPyDlz4D6+vq0f/9+VVRUxO9LS0tTRUWF6uvrDTuzcfjwYZWUlGjq1Km65557dPToUeuWTDU1NamlpSXh+IhGoyovL78ij4/a2loVFBRo+vTpeuCBB3Ty5EnrloZUe3u7JCkvL0+StH//fvX39yccDzNmzNDkyZNH9fFw7n74wssvv6z8/HzNnDlT1dXV6u7utmjvglJuGOm5Pv/8cw0MDKiwsDDh/sLCQv35z3826spGeXm5tmzZounTp+v48eN66qmndNNNN+nQoUPKycmxbs9ES0uLJA16fHzx2JVi6dKluv3221VWVqbGxkb96Ec/UmVlperr6xUOh63bS7pYLKa1a9fqhhtu0MyZMyWdPR4yMzOVm5ubsO5oPh4G2w+SdPfdd2vKlCkqKSnRwYMH9eijj6qhoUFvvvmmYbeJUj6A8HeVlZXxr2fPnq3y8nJNmTJFr7/+uu677z7DzpAK7rzzzvjXs2bN0uzZszVt2jTV1tZq0aJFhp0NjaqqKh06dOiKeB/0Yi60H1avXh3/etasWSouLtaiRYvU2NioadOmDXebg0r5X8Hl5+crHA6fdxVLa2urioqKjLpKDbm5ubr22mt15MgR61bMfHEMcHycb+rUqcrPzx+Vx8eaNWv09ttv6/3330/48y1FRUXq6+tTW1tbwvqj9Xi40H4YTHl5uSSl1PGQ8gGUmZmpuXPnateuXfH7YrGYdu3apfnz5xt2Zu/UqVNqbGxUcXGxdStmysrKVFRUlHB8dHR0aO/evVf88fHpp5/q5MmTo+r4cM5pzZo12rZtm9577z2VlZUlPD537lxlZGQkHA8NDQ06evToqDoeLrUfBnPgwAFJSq3jwfoqiH/Eq6++6iKRiNuyZYv74x//6FavXu1yc3NdS0uLdWvD6gc/+IGrra11TU1N7ne/+52rqKhw+fn57sSJE9atDanOzk730UcfuY8++shJcs8++6z76KOP3F/+8hfnnHM//elPXW5urtuxY4c7ePCgu/XWW11ZWZk7ffq0cefJdbH90NnZ6R5++GFXX1/vmpqa3Lvvvuu+/vWvu2uuucb19PRYt540DzzwgItGo662ttYdP348vnR3d8fXuf/++93kyZPde++95/bt2+fmz5/v5s+fb9h18l1qPxw5csQ9/fTTbt++fa6pqcnt2LHDTZ061S1YsMC480QjIoCcc+6FF15wkydPdpmZmW7evHluz5491i0NuzvuuMMVFxe7zMxM96Uvfcndcccd7siRI9ZtDbn333/fSTpvWbFihXPu7KXYjz/+uCssLHSRSMQtWrTINTQ02DY9BC62H7q7u93ixYvdxIkTXUZGhpsyZYpbtWrVqHuRNti/X5LbvHlzfJ3Tp0+7733ve+6qq65yY8aMcbfddps7fvy4XdND4FL74ejRo27BggUuLy/PRSIRd/XVV7sf/vCHrr293bbxc/DnGAAAJlL+PSAAwOhEAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADAxH8DR+VYJGWV2nkAAAAASUVORK5CYII=", "text/plain": [ "
" ] @@ -2202,8 +2040,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "[ 2.4792047 3.8394718 0.3413597 3.1128588 1.103995 -3.024009\n", - " 1.1684668 -3.6187618 -1.6347903 -3.1772547]\n", + "[ 2.44647 3.8623989 0.14587203 3.2146688 1.0799949 -2.5363288\n", + " 0.86715794 -3.8287208 -2.02238 -2.9016623 ]\n", "predicted label: Trouser\n" ] } @@ -2257,7 +2095,7 @@ "id": "d8abea75", "metadata": {}, "source": [ - "Import the utility functions from pytriton_utils.py:" + "Import the helper class from pytriton_utils.py:" ] }, { @@ -2269,12 +2107,7 @@ "source": [ "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "from pytriton_utils import (\n", - " use_stage_level_scheduling,\n", - " find_ports,\n", - " start_triton,\n", - " stop_triton\n", - ")" + "from pytriton_utils import TritonServerManager" ] }, { @@ -2349,6 +2182,7 @@ " )\n", "\n", " def _stop_triton(signum, frame):\n", + " # The server manager sends SIGTERM to stop the server; this function ensures graceful cleanup.\n", " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", " triton.stop()\n", "\n", @@ -2388,82 +2222,37 @@ }, { "cell_type": "markdown", - "id": "ec34f9eb", "metadata": {}, "source": [ - "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU." + "The `TritonClusterManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n", + "- Find available ports for HTTP/gRPC/metrics\n", + "- Deploy a server on each node via stage-level scheduling\n", + "- Gracefully shutdown servers across nodes" ] }, { "cell_type": "code", "execution_count": 68, - "id": "06349836", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - } - ], - "source": [ - "sc = spark.sparkContext\n", - "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" - ] - }, - { - "cell_type": "markdown", - "id": "508f9972", + "id": "f72c53d6", "metadata": {}, - "source": [ - "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." - ] - }, - { - "cell_type": "code", - "execution_count": 69, - "id": "33cd12f9", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using ports [7000, 7001, 7002]\n" - ] - } - ], + "outputs": [], "source": [ "model_name = \"ImageClassifier\"\n", - "ports = find_ports()\n", - "assert len(ports) == 3\n", - "print(f\"Using ports {ports}\")" + "server_manager = TritonServerManager(num_nodes=num_nodes, model_name=model_name, model_path=model_path)" ] }, { "cell_type": "code", - "execution_count": 70, - "id": "6a0e8778", + "execution_count": null, + "id": "65d3f7be", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 16:> (0 + 1) / 1]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Triton Server PIDs:\n", - " {\n", - " \"cb4ae00-lcedt\": 2968686\n", - "}\n" + "2025-02-07 11:03:44,809 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-07 11:03:44,810 - INFO - Starting 1 servers.\n" ] }, { @@ -2472,14 +2261,20 @@ "text": [ " \r" ] + }, + { + "data": { + "text/plain": [ + "{'cb4ae00-lcedt': (2020631, [7000, 7001, 7002])}" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", - " ports=ports,\n", - " model_name=model_name,\n", - " model_path=model_path)).collectAsMap()\n", - "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n", + "server_manager.start_servers(triton_server)" ] }, { @@ -2490,26 +2285,44 @@ "#### Define client function" ] }, + { + "cell_type": "markdown", + "id": "86c1545a", + "metadata": {}, + "source": [ + "Get the hostname -> url mapping from the server manager:" + ] + }, { "cell_type": "code", - "execution_count": 71, - "id": "549cda8b", + "execution_count": null, + "id": "c4c2833f", "metadata": {}, "outputs": [], "source": [ - "url = f\"http://localhost:{ports[0]}\"" + "host_to_http_url = server_manager.host_to_http_url # or server_manager.host_to_grpc_url" + ] + }, + { + "cell_type": "markdown", + "id": "c6771c93", + "metadata": {}, + "source": [ + "Define the Triton inference function, which returns a predict function for batch inference through the server:" ] }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 71, "id": "cec9a48c", "metadata": {}, "outputs": [], "source": [ - "def triton_fn(url, model_name):\n", + "def triton_fn(model_name, host_to_url):\n", + " import socket\n", " from pytriton.client import ModelClient\n", "\n", + " url = host_to_url[socket.gethostname()]\n", " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", "\n", " def infer_batch(inputs):\n", @@ -2520,6 +2333,19 @@ " return infer_batch" ] }, + { + "cell_type": "code", + "execution_count": 73, + "id": "0262fd4a-9845-44b9-8c75-1c105e7deeca", + "metadata": {}, + "outputs": [], + "source": [ + "mnist = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_http_url),\n", + " input_tensor_shapes=[[784]],\n", + " return_type=ArrayType(FloatType()),\n", + " batch_size=50)" + ] + }, { "cell_type": "markdown", "id": "30a4362d-7514-4b84-b238-f704a97e1e72", @@ -2530,7 +2356,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 72, "id": "ab94d4d1-dac6-4474-9eb0-59478aa98f7d", "metadata": {}, "outputs": [ @@ -2540,7 +2366,7 @@ "StructType([StructField('data', ArrayType(FloatType(), True), True)])" ] }, - "execution_count": 73, + "execution_count": 72, "metadata": {}, "output_type": "execute_result" } @@ -2553,19 +2379,6 @@ { "cell_type": "code", "execution_count": 74, - "id": "0262fd4a-9845-44b9-8c75-1c105e7deeca", - "metadata": {}, - "outputs": [], - "source": [ - "mnist = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", - " input_tensor_shapes=[[784]],\n", - " return_type=ArrayType(FloatType()),\n", - " batch_size=50)" - ] - }, - { - "cell_type": "code", - "execution_count": 75, "id": "fc5f6baa-052e-4b89-94b6-4821cf01952a", "metadata": {}, "outputs": [ @@ -2580,8 +2393,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 159 ms, sys: 37.3 ms, total: 196 ms\n", - "Wall time: 1.53 s\n" + "CPU times: user 157 ms, sys: 47.6 ms, total: 205 ms\n", + "Wall time: 2.49 s\n" ] } ], @@ -2592,16 +2405,23 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 75, "id": "a85dea35-e41d-482d-8a8f-52d3c108f038", "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 176 ms, sys: 68.6 ms, total: 245 ms\n", - "Wall time: 952 ms\n" + "CPU times: user 183 ms, sys: 60.3 ms, total: 243 ms\n", + "Wall time: 1.49 s\n" ] } ], @@ -2612,7 +2432,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 76, "id": "bc3f0dbe-c52b-41d6-8097-8cebaa5ee5a8", "metadata": {}, "outputs": [ @@ -2627,8 +2447,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 189 ms, sys: 43 ms, total: 232 ms\n", - "Wall time: 1.17 s\n" + "CPU times: user 383 ms, sys: 43.9 ms, total: 427 ms\n", + "Wall time: 1.6 s\n" ] } ], @@ -2639,7 +2459,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 77, "id": "99fb5e8d", "metadata": {}, "outputs": [ @@ -2652,7 +2472,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAknklEQVR4nO3de3SV9Z3v8c9OSDa3ZMcQcpNAAyi0cumUSsqoFEsOkM64QDkdb3MOuDow0uCqUqsnPVZq27PS4hrrqUNxrbNaqKvihXNERsdiBSWMCnRAGMZeUsAoYSChYJMNCUl2sn/nD8bMREH4/kzyS8L7tdZei+z9fHh+efIknzzZO99EnHNOAAD0spTQCwAAXJooIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBDAq9gA9LJpM6evSoMjIyFIlEQi8HAGDknNOpU6dUWFiolJTzX+f0uQI6evSoioqKQi8DAPAJ1dbWatSoUed9vM8VUEZGhiTpWn1Zg5QWeDXodr11VcuEKSCYdiX0ul7q/Hp+Pj1WQKtXr9bDDz+suro6TZ06VY899pimT59+wdwHP3YbpDQNilBAA06v/ViVAgKC+fdPvws9jdIjL0J45plntGLFCq1cuVJvvfWWpk6dqrlz5+r48eM9sTsAQD/UIwX0yCOPaMmSJbrjjjv0mc98Ro8//riGDh2qn/3sZz2xOwBAP9TtBdTW1qY9e/aotLT0P3aSkqLS0lLt2LHjI9u3trYqHo93uQEABr5uL6ATJ06oo6NDeXl5Xe7Py8tTXV3dR7avrKxULBbrvPEKOAC4NAT/RdSKigo1NjZ23mpra0MvCQDQC7r9VXA5OTlKTU1VfX19l/vr6+uVn5//ke2j0aii0Wh3LwMA0Md1+xVQenq6pk2bpq1bt3bel0wmtXXrVs2YMaO7dwcA6Kd65PeAVqxYoUWLFunzn/+8pk+frkcffVRNTU264447emJ3AIB+qEcK6Oabb9Yf//hHPfjgg6qrq9NnP/tZbd68+SMvTAAAXLoizvWtmSXxeFyxWEyzNJ9JCBiwUscXmzPH/s7+XGnu/N+bM8An1e4S2qZNamxsVGZm5nm3C/4qOADApYkCAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQfTINGygv6r5gf1vVt0/f6M5U9Vw/gGN5zM+7Yw5k7/fnpGkX2yYbc4Ufe9Nr33h0sUVEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIJgGnYfFhlk//C4jg77jpyzZzxF0tLNGZdoM2cGFY8xZyRpy20PmzNf/NXd5syVf7PbnKk3J6TdN1/vkZIe/O5T5sza7/kdc7NIxJ7pxXMcF48rIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIIuJc35rSF4/HFYvFNEvzNSiSFno53cdjgGIkNdWc8RpG6qtvnTpd/GHtNK/c2KI/mjODSg977asvO/NysTlzw+X7zZktkzLMGfR97S6hbdqkxsZGZWZmnnc7roAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIIhBoRdwyfAY3Ona2+378Rh62peHikpSytRPmzNbvvS/vfZV+ssV5syV8hhGmmIfNBtJsX9svc4hSWk/zDZnbl73L+bMhsXfNGcuW7fDnEHfxBUQACAICggAEES3F9B3vvMdRSKRLreJEyd2924AAP1cjzwHdNVVV2nLli3/sZNBPNUEAOiqR5ph0KBBys/P74n/GgAwQPTIc0AHDhxQYWGhxo4dq9tvv12HD5//VUKtra2Kx+NdbgCAga/bC6ikpETr1q3T5s2btWbNGtXU1Oi6667TqVOnzrl9ZWWlYrFY562oqKi7lwQA6IO6vYDKysr0la98RVOmTNHcuXP10ksvqaGhQc8+++w5t6+oqFBjY2Pnrba2truXBADog3r81QFZWVm68sordfDgwXM+Ho1GFY1Ge3oZAIA+psd/D+j06dM6dOiQCgoKenpXAIB+pNsL6N5771VVVZXeffddvfnmm7rxxhuVmpqqW2+9tbt3BQDox7r9R3BHjhzRrbfeqpMnT2rkyJG69tprtXPnTo0cObK7dwUA6Me6vYCefvrp7v4vYeEzWNRjMKYkKdlhjsRv/YI5M2b5H8yZx09eZ85IUuGrvTSdyiXNkUh0qH03nsNI675gf172nUSmOfPMQw+bMyv/9svmTP0Mfr2jL2IWHAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAE0eN/kK7XRCL2jM/gzt7kMyTUY8ilz1BRX1d/Y485U92YZ840t6ebM5I0/NmdXjmrSKrnANheknbKnnmz6Qpz5tE/fcqcuWvUFnPmgduWmDOSlLne43zora9FPvvx3VcP4QoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQQycadh9ncfkWp+JyS7Re5Oth20fac60O/uY5dQU+4TvdzeNNWckqUB1Xjkr1+HxcWpLdP9CziPvsTfNmW9VVJsz1x69ypxZeWC+OXPz/9xszkjSyy8UmTPJUx6jxH34TrXuQ385gCsgAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAhi4Awj9RiWF0lL99tVos0jZF+f1348HL3vz71y38x91px58t++YM4kZR+eWPCIfZhmr+rD54OvXzWnmTP/fcxOc2bV3jnmTGqR3zDNjF/av0Y0Xuu1q94T8bjucD0z5JgrIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIIuKcx1TEHhSPxxWLxTRL8zUoYh9uOJAcX24fEtqaZd/Pr5ausockrWv4vDkzKv19c+Z7v7zJnIn9wT7AVJKu/5td5szG33zWnBn0b1FzJqXN432K+H16++yrdUTSnBk54YQ5E4u2mDNn2v2+lqwc/w/mzLINS82Z4v+xw5zpy9pdQtu0SY2NjcrMzDzvdlwBAQCCoIAAAEGYC2j79u264YYbVFhYqEgkoueff77L4845PfjggyooKNCQIUNUWlqqAwcOdNd6AQADhLmAmpqaNHXqVK1evfqcj69atUo//vGP9fjjj2vXrl0aNmyY5s6dq5YW+89tAQADl/kvopaVlamsrOycjznn9Oijj+qBBx7Q/PnzJUlPPPGE8vLy9Pzzz+uWW275ZKsFAAwY3focUE1Njerq6lRaWtp5XywWU0lJiXbsOPerPFpbWxWPx7vcAAADX7cWUF1dnSQpLy+vy/15eXmdj31YZWWlYrFY562oqKg7lwQA6KOCvwquoqJCjY2Nnbfa2trQSwIA9IJuLaD8/HxJUn19fZf76+vrOx/7sGg0qszMzC43AMDA160FVFxcrPz8fG3durXzvng8rl27dmnGjBnduSsAQD9nfhXc6dOndfDgwc63a2pqtG/fPmVnZ2v06NG6++679f3vf19XXHGFiouL9e1vf1uFhYVasGBBd64bANDPmQto9+7duv766zvfXrFihSRp0aJFWrdune677z41NTVp6dKlamho0LXXXqvNmzdr8ODB3bdqAEC/d0kPI31nld+PBf/uxp+bMyv++a/MmfT0dnPmB1OfM2c2N0wxZyQpRfZT53eNeRfe6EPe23u5OdNxWcKckaTBtenmTOY79uOQ0m7PdKTbB4Qmzd9inuVS7Zlkmn19kaT9OJye2WzOfLboiDkjSUdPx8yZa/LeMWfeet/+6t8j72eZM5I0+iv/6pWzYBgpAKBPo4AAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAjPWbkDw29v/3uv3MKDf2HO5Pyj/c9RnF54ypz52dHrzJnGNr8/lXFH0RvmzIm2YeZMzeCkOaMO+2RmSWq7zL6vxFf+ZM6MijWaMyOjp82ZaKp9orokZQ2yT5xOeIzQbuqImjMzM6vt+0na9yNJbw4ab86kyn4ODRvUZs68NH2NOSNJt95+rzkTe3Kn174uhCsgAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAhiwAwjbblhujmTFtnnt69v5ZszOf/rPXPm5fHPmTMPn7Afh6Ep9kGIkrTy9QXmTErcfsq5LI+Bmh7zS8/uK2HOxA9cZs4cbBhhztTaZ54qtdXZQ546ovYBsM5jZuzr6Z8zZ25f/Ip9R5Kuy/qDOfO5wYfNmZfTrjJn/uKf7zRnJOlvKn5lzrz8ZKbXvi6EKyAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACGLADCOt/3zvvStZP6w1Z/4y51/MmZ822AcU/lXWP5sz3639S3NGki7/Zao5kxjqMX1SaeZEJOk3hNOl2NfXkW7fTzLNvj6ftbVm+RxvSR6xiMfMWJ/9DP83+6TZx//pevuOJP1h/hpz5jdt9ndqUWy/OfOPmZPNGUn669i/mjO/+rOlpu0jHa3Sv2y64HZcAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEANmGGna6d7b172XbzZnNvxpujmTmx43Z7556L+aM81/f7k5I0lncu3fv/gMx0xtM0eUTPUbwpnSSwM1fQZ3OvvsV6+1SVL7EL+clc9xODXaft4VVNn3I0lfmzbTnCkc3GDOVJ/OM2cWXr7XnJGkESn2D+6Zy4eZtm9PpEoXMX+ZKyAAQBAUEAAgCHMBbd++XTfccIMKCwsViUT0/PPPd3l88eLFikQiXW7z5s3rrvUCAAYIcwE1NTVp6tSpWr169Xm3mTdvno4dO9Z5e+qppz7RIgEAA4/5RQhlZWUqKyv72G2i0ajy8/O9FwUAGPh65Dmgbdu2KTc3VxMmTNCyZct08uTJ827b2tqqeDze5QYAGPi6vYDmzZunJ554Qlu3btUPf/hDVVVVqaysTB0dHefcvrKyUrFYrPNWVFTU3UsCAPRB3f57QLfcckvnvydPnqwpU6Zo3Lhx2rZtm2bPnv2R7SsqKrRixYrOt+PxOCUEAJeAHn8Z9tixY5WTk6ODBw+e8/FoNKrMzMwuNwDAwNfjBXTkyBGdPHlSBQUFPb0rAEA/Yv4R3OnTp7tczdTU1Gjfvn3Kzs5Wdna2HnroIS1cuFD5+fk6dOiQ7rvvPo0fP15z587t1oUDAPo3cwHt3r1b119/fefbHzx/s2jRIq1Zs0b79+/Xz3/+czU0NKiwsFBz5szR9773PUWj0e5bNQCg3zMX0KxZs+ScO+/jL7/88idakK9Bzb23rwNt9t9xejjfPjjw/522Px/W+LP/Ys4kY34TKxPD7LnIuV8M+bE60u0Z3yGcH3Nqn39XvTRY1GtAqOdxGHTGnknxGBrrcz74HLuOqN+B+PX6qebMhhUPmzP/lD7OnLl6yLvmjCTFk0lzZlj1CdP27R2tF7Uds+AAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQRLf/Se5QhpywT3j1NS7tuDlzpN0+Xvi+l+40ZzJG2L+nSAwzRyRJ6XF7xnl8y9Mx2J6Rx1RrSWof6rErj+nMPuvzmbrtKzHcvsCkx1eT1Db7lOqUixu03EX8U37TsDMO24/Dtw7PN2f+77gt5sy2Mx4nq6TC1FPmTMeBd2zbu8RFbccVEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEMWCGkWYeOm3OJFyH177GprWYMzN3LDNn8t+wD0L800RzxHvIZUuOPdMRtb9P0fc9Bkl6fmvlMyy1faj9fWrPsJ97KcMvbsDjf5Zs9ZmUKkWPppkzaaftHyevY+cxnDa11W8Y6Zlce+6d9VeYM7+7/x/MGWm4R0a6LGWIOZM6Ybxpe9fRKh248HZcAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEANmGGlKc5s5kxbxG9T4r22Z5szwX9kHB/5poscAxaQ90jHEPhBSktqy7DuLnrAf8/RT9vWlLzhuzkjSycZh5kxHm/3TKP1w1JzJ3m7/fjHi96HVmWz7uddc6DFY1GMYqXzmivrNIlUiw76+ISn2j9P8nXeaM7+c8RNzRpLaZT/3IgnbxOJI8uK25woIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIIYMMNI1d5hjjQmz3jt6m93fs2cSRtln4bYmmN/n1Ja7fuJJPwmNabnN5szY38UN2fa3ztiztRHS8wZSRq/9aQ9dLTWnskdYY68d1OuOdM8xjZE8gORofbhvu6Mx3DfpMf52m7PdKT6TWV1g+y55lH2TMZO+7DiuulDzRlJGpdmv+5of+dd2/YucVHbcQUEAAiCAgIABGEqoMrKSl199dXKyMhQbm6uFixYoOrq6i7btLS0qLy8XCNGjNDw4cO1cOFC1dfXd+uiAQD9n6mAqqqqVF5erp07d+qVV15RIpHQnDlz1NTU1LnNPffcoxdeeEEbNmxQVVWVjh49qptuuqnbFw4A6N9ML0LYvHlzl7fXrVun3Nxc7dmzRzNnzlRjY6N++tOfav369frSl74kSVq7dq0+/elPa+fOnfrCF77QfSsHAPRrn+g5oMbGRklSdna2JGnPnj1KJBIqLS3t3GbixIkaPXq0duzYcc7/o7W1VfF4vMsNADDweRdQMpnU3XffrWuuuUaTJk2SJNXV1Sk9PV1ZWVldts3Ly1NdXd05/5/KykrFYrHOW1FRke+SAAD9iHcBlZeX6+2339bTTz/9iRZQUVGhxsbGzlttrcfvVAAA+h2vX0Rdvny5XnzxRW3fvl2jRo3qvD8/P19tbW1qaGjochVUX1+v/Pz8c/5f0WhU0WjUZxkAgH7MdAXknNPy5cu1ceNGvfrqqyouLu7y+LRp05SWlqatW7d23lddXa3Dhw9rxowZ3bNiAMCAYLoCKi8v1/r167Vp0yZlZGR0Pq8Ti8U0ZMgQxWIxffWrX9WKFSuUnZ2tzMxM3XXXXZoxYwavgAMAdGEqoDVr1kiSZs2a1eX+tWvXavHixZKkH/3oR0pJSdHChQvV2tqquXPn6ic/+Um3LBYAMHCYCsi5Cw/ZGzx4sFavXq3Vq1d7L8pLqv31FC82jbrwRufgkh4Zj3mfqc0eQwOHewwwjfi9FiXZan8KsfnKkeZMes175szlzx4yZySp/i/GmjMnr8kwZ7JGnDZnWk/bh+dG3k83ZyRJDWn2ffnMtPU59XwyfrNIFUnYd+aG2wfANhfa9/PXW/7WnJGkmr/8P+ZM6ohs0/Yu2Sa9f+HtmAUHAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAILz+Impf1PG7A+bMsJRWr309cc1PzZm/Tiw1ZyLNqeZMaixhziQ7/CYmuxb76RNf3mjOpNx1pTnT3Gqf5ixJzp2yh94fYo40vptlzkTsg86lNM8x0PZTz2vitEv1CHlMo4/4jKOX5AbbD3pqg/3zomOY/Z1Kr++9L98tf1Z84Y3+k/b2Fum1C2/HFRAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABDFghpH6yEpp9sqlekxd/NF1T5szhYP+ZM784uSfmzMvvjHNnJEkddgHPL5/JMucSW22f5/kevFbq4jHQE2XZh8+6fxmxnqJtHsM7/SZ9+kzK9VjPy7FcyirxzmejHruyygl4TdgtTnZZs60Zdmqoj1xcdtzBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVzSw0i3nLrKKzdl6GFz5nKPwaIT0trNmU8NPmnO/LdZ/2TOSNL631xtDx0eYo6k2A+DEjH7sE9JingMn4wk7ZmUM36DJK1c7+xGkhTxmMHpUuwL9Bk067O2s/vyead8ziH7bga12DOSVNPeYc4MPpEwbd/efnHbcwUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFc2sNIj07wysVGN5sz49L+aM78In6lOZMWsQ8aLMvYb85I0pTP15ozI0pOmzN5qfbMxvifmTOS9NJR+4Da9qT9+7jmtjRzJumxn2iabYjkB4Z4DML1OQ6DUuxTOH3mip5uiXqkpNgQ+8TPwuGN5kxbR6o5k/SZyirpjTPjzJljMwabtu9olXQRM465AgIABEEBAQCCMBVQZWWlrr76amVkZCg3N1cLFixQdXV1l21mzZqlSCTS5XbnnXd266IBAP2fqYCqqqpUXl6unTt36pVXXlEikdCcOXPU1NTUZbslS5bo2LFjnbdVq1Z166IBAP2f6UUImzdv7vL2unXrlJubqz179mjmzJmd9w8dOlT5+fnds0IAwID0iZ4Damw8+2qP7OzsLvc/+eSTysnJ0aRJk1RRUaHm5vO/aqy1tVXxeLzLDQAw8Hm/DDuZTOruu+/WNddco0mTJnXef9ttt2nMmDEqLCzU/v37df/996u6ulrPPffcOf+fyspKPfTQQ77LAAD0U94FVF5errfffluvv/56l/uXLl3a+e/JkyeroKBAs2fP1qFDhzRu3Edff15RUaEVK1Z0vh2Px1VUVOS7LABAP+FVQMuXL9eLL76o7du3a9SoUR+7bUlJiSTp4MGD5yygaDSqaNTvl8QAAP2XqYCcc7rrrru0ceNGbdu2TcXFxRfM7Nu3T5JUUFDgtUAAwMBkKqDy8nKtX79emzZtUkZGhurq6iRJsVhMQ4YM0aFDh7R+/Xp9+ctf1ogRI7R//37dc889mjlzpqZMmdIj7wAAoH8yFdCaNWsknf1l0/9s7dq1Wrx4sdLT07VlyxY9+uijampqUlFRkRYuXKgHHnig2xYMABgYzD+C+zhFRUWqqqr6RAsCAFwaLulp2EM8JwUvy/qNOXMwETFnyrPs06b92CfxnmX/na3mZJs5MzRlqDnz6ZzqC290Dssu22vOXJZqX5+PN1rsk6Mbkn5ry0+1f2wHe0xi75D986LF2c/XkSmt5owkFacNN2f2tNrP8fFp9mO3oyXLnJGk9X8sMWdGVb5p2r7dJXTgIrZjGCkAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABHFpDyP9ut9fYi2b8HVzJtpgH3yaTOud7w86on77OXK9PRfJsw+FzHhziDlT+Muj5owkuVT7+9Rx2TBzJvVUizmjY8fNEZdot+9HUmSofYhpZLjH4NMLTNg/p3b74E5fzVfZ/5Cmz+ft8D2HzZn2Y3XmzFn2QbM9hSsgAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQRJ+bBef+fTZUuxKSx5go07467HPJJKk9YZ/jldruMQsu0kuz4FL89pNs8ZgF12w/5h1tEXOmPen3sXUe35N1tKfa9+Nz7rk2e8R5zoJL2r80RJL24+A1Cy7Ze7Pg2tvtn+tJj3OoPWn/2LY7+9eU3tKus2tzF/j4RtyFtuhlR44cUVFRUehlAAA+odraWo0aNeq8j/e5Akomkzp69KgyMjIUiXT9zjcej6uoqEi1tbXKzMwMtMLwOA5ncRzO4jicxXE4qy8cB+ecTp06pcLCQqV8zE9Y+tyP4FJSUj62MSUpMzPzkj7BPsBxOIvjcBbH4SyOw1mhj0MsFrvgNrwIAQAQBAUEAAiiXxVQNBrVypUrFY36/SXTgYLjcBbH4SyOw1kch7P603Hocy9CAABcGvrVFRAAYOCggAAAQVBAAIAgKCAAQBD9poBWr16tT33qUxo8eLBKSkr061//OvSSet13vvMdRSKRLreJEyeGXlaP2759u2644QYVFhYqEono+eef7/K4c04PPvigCgoKNGTIEJWWlurAgQNhFtuDLnQcFi9e/JHzY968eWEW20MqKyt19dVXKyMjQ7m5uVqwYIGqq6u7bNPS0qLy8nKNGDFCw4cP18KFC1VfXx9oxT3jYo7DrFmzPnI+3HnnnYFWfG79ooCeeeYZrVixQitXrtRbb72lqVOnau7cuTp+/HjopfW6q666SseOHeu8vf7666GX1OOampo0depUrV69+pyPr1q1Sj/+8Y/1+OOPa9euXRo2bJjmzp2rlhb7IMm+7ELHQZLmzZvX5fx46qmnenGFPa+qqkrl5eXauXOnXnnlFSUSCc2ZM0dNTU2d29xzzz164YUXtGHDBlVVVeno0aO66aabAq66+13McZCkJUuWdDkfVq1aFWjF5+H6genTp7vy8vLOtzs6OlxhYaGrrKwMuKret3LlSjd16tTQywhKktu4cWPn28lk0uXn57uHH364876GhgYXjUbdU089FWCFvePDx8E55xYtWuTmz58fZD2hHD9+3ElyVVVVzrmzH/u0tDS3YcOGzm1+97vfOUlux44doZbZ4z58HJxz7otf/KL7+te/Hm5RF6HPXwG1tbVpz549Ki0t7bwvJSVFpaWl2rFjR8CVhXHgwAEVFhZq7Nixuv3223X48OHQSwqqpqZGdXV1Xc6PWCymkpKSS/L82LZtm3JzczVhwgQtW7ZMJ0+eDL2kHtXY2ChJys7OliTt2bNHiUSiy/kwceJEjR49ekCfDx8+Dh948sknlZOTo0mTJqmiokLNzc0hlndefW4Y6YedOHFCHR0dysvL63J/Xl6efv/73wdaVRglJSVat26dJkyYoGPHjumhhx7Sddddp7ffflsZGRmhlxdEXV2dJJ3z/PjgsUvFvHnzdNNNN6m4uFiHDh3St771LZWVlWnHjh1KTfX4Wz19XDKZ1N13361rrrlGkyZNknT2fEhPT1dWVlaXbQfy+XCu4yBJt912m8aMGaPCwkLt379f999/v6qrq/Xcc88FXG1Xfb6A8B/Kyso6/z1lyhSVlJRozJgxevbZZ/XVr3414MrQF9xyyy2d/548ebKmTJmicePGadu2bZo9e3bAlfWM8vJyvf3225fE86Af53zHYenSpZ3/njx5sgoKCjR79mwdOnRI48aN6+1lnlOf/xFcTk6OUlNTP/Iqlvr6euXn5wdaVd+QlZWlK6+8UgcPHgy9lGA+OAc4Pz5q7NixysnJGZDnx/Lly/Xiiy/qtdde6/LnW/Lz89XW1qaGhoYu2w/U8+F8x+FcSkpKJKlPnQ99voDS09M1bdo0bd26tfO+ZDKprVu3asaMGQFXFt7p06d16NAhFRQUhF5KMMXFxcrPz+9yfsTjce3ateuSPz+OHDmikydPDqjzwzmn5cuXa+PGjXr11VdVXFzc5fFp06YpLS2ty/lQXV2tw4cPD6jz4ULH4Vz27dsnSX3rfAj9KoiL8fTTT7toNOrWrVvnfvvb37qlS5e6rKwsV1dXF3ppveob3/iG27Ztm6upqXFvvPGGKy0tdTk5Oe748eOhl9ajTp065fbu3ev27t3rJLlHHnnE7d2717333nvOOed+8IMfuKysLLdp0ya3f/9+N3/+fFdcXOzOnDkTeOXd6+OOw6lTp9y9997rduzY4WpqatyWLVvc5z73OXfFFVe4lpaW0EvvNsuWLXOxWMxt27bNHTt2rPPW3Nzcuc2dd97pRo8e7V599VW3e/duN2PGDDdjxoyAq+5+FzoOBw8edN/97nfd7t27XU1Njdu0aZMbO3asmzlzZuCVd9UvCsg55x577DE3evRol56e7qZPn+527twZekm97uabb3YFBQUuPT3dXX755e7mm292Bw8eDL2sHvfaa685SR+5LVq0yDl39qXY3/72t11eXp6LRqNu9uzZrrq6Ouyie8DHHYfm5mY3Z84cN3LkSJeWlubGjBnjlixZMuC+STvX+y/JrV27tnObM2fOuK997Wvusssuc0OHDnU33nijO3bsWLhF94ALHYfDhw+7mTNnuuzsbBeNRt348ePdN7/5TdfY2Bh24R/Cn2MAAATR558DAgAMTBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAI4v8DIE1CeiobCX8AAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGdCAYAAABU0qcqAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAJJ5JREFUeJzt3Xt0lfWd7/HPTkg2t2THEHKTQAMotHLplErKqBRLDpDOuEA5HW9zDrg6MNLgqlKrJz1Watuz0uIa66lDca2zWqir4oVzREbHYgUljAp0QBjGXlLAKGEgoWCTDQlJdrJ/5w/GzERB+P5M8kvC+7XWXovs/Xx4fnnyJJ882TvfRJxzTgAA9LKU0AsAAFyaKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQQwKvYAPSyaTOnr0qDIyMhSJREIvBwBg5JzTqVOnVFhYqJSU81/n9LkCOnr0qIqKikIvAwDwCdXW1mrUqFHnfbzPFVBGRoYk6Vp9WYOUFng16Ha9dVXLhCkgmHYl9Lpe6vx6fj49VkCrV6/Www8/rLq6Ok2dOlWPPfaYpk+ffsHcBz92G6Q0DYpQQANOr/1YlQICgvn3T78LPY3SIy9CeOaZZ7RixQqtXLlSb731lqZOnaq5c+fq+PHjPbE7AEA/1CMF9Mgjj2jJkiW644479JnPfEaPP/64hg4dqp/97Gc9sTsAQD/U7QXU1tamPXv2qLS09D92kpKi0tJS7dix4yPbt7a2Kh6Pd7kBAAa+bi+gEydOqKOjQ3l5eV3uz8vLU11d3Ue2r6ysVCwW67zxCjgAuDQE/0XUiooKNTY2dt5qa2tDLwkA0Au6/VVwOTk5Sk1NVX19fZf76+vrlZ+f/5Hto9GootFody8DANDHdfsVUHp6uqZNm6atW7d23pdMJrV161bNmDGju3cHAOineuT3gFasWKFFixbp85//vKZPn65HH31UTU1NuuOOO3pidwCAfqhHCujmm2/WH//4Rz344IOqq6vTZz/7WW3evPkjL0wAAFy6Is71rZkl8XhcsVhMszSfSQgYsFLHF5szx/7O/lxp7vzfmzPAJ9XuEtqmTWpsbFRmZuZ5twv+KjgAwKWJAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEH0yDRsoL+q+YH9b1bdP3+jOVPVcP4BjeczPu2MOZO/356RpF9smG3OFH3vTa994dLFFRAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCYBp2HxYZZP/wuI4O+46cs2c8RdLSzRmXaDNnBhWPMWckacttD5szX/zV3ebMlX+z25ypNyek3Tdf75GSHvzuU+bM2u/5HXOzSMSe6cVzHBePKyAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACCLiXN+a0hePxxWLxTRL8zUokhZ6Od3HY4BiJDXVnPEaRuqrb506Xfxh7TSv3NiiP5ozg0oPe+2rLzvzcrE5c8Pl+82ZLZMyzBn0fe0uoW3apMbGRmVmZp53O66AAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACCIQaEXcMnwGNzp2tvt+/EYetqXh4pKUsrUT5szW770v732VfrLFebMlfIYRppiHzQbSbF/bL3OIUlpP8w2Z25e9y/mzIbF3zRnLlu3w5xB38QVEAAgCAoIABBEtxfQd77zHUUikS63iRMndvduAAD9XI88B3TVVVdpy5Yt/7GTQTzVBADoqkeaYdCgQcrPz++J/xoAMED0yHNABw4cUGFhocaOHavbb79dhw+f/1VCra2tisfjXW4AgIGv2wuopKRE69at0+bNm7VmzRrV1NTouuuu06lTp865fWVlpWKxWOetqKiou5cEAOiDur2AysrK9JWvfEVTpkzR3Llz9dJLL6mhoUHPPvvsObevqKhQY2Nj5622tra7lwQA6IN6/NUBWVlZuvLKK3Xw4MFzPh6NRhWNRnt6GQCAPqbHfw/o9OnTOnTokAoKCnp6VwCAfqTbC+jee+9VVVWV3n33Xb355pu68cYblZqaqltvvbW7dwUA6Me6/UdwR44c0a233qqTJ09q5MiRuvbaa7Vz506NHDmyu3cFAOjHur2Ann766e7+L2HhM1jUYzCmJCnZYY7Eb/2COTNm+R/MmcdPXmfOSFLhq700ncolzZFIdKh9N57DSOu+YH9e9p1EpjnzzEMPmzMr//bL5kz9DH69oy9iFhwAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABNHjf5Cu10Qi9ozP4M7e5DMk1GPIpc9QUV9Xf2OPOVPdmGfONLenmzOSNPzZnV45q0iq5wDYXpJ2yp55s+kKc+bRP33KnLlr1BZz5oHblpgzkpS53uN86K2vRT778d1XD+EKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEEMnGnYfZ3H5Fqficku0XuTrYdtH2nOtDv7mOXUFPuE73c3jTVnJKlAdV45K9fh8XFqS3T/Qs4j77E3zZlvVVSbM9cevcqcWXlgvjlz8//cbM5I0ssvFJkzyVMeo8R9+E617kN/OYArIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIYuAMI/UYlhdJS/fbVaLNI2Rfn9d+PBy978+9ct/MfdacefLfvmDOJGUfnljwiH2YZq/qw+eDr181p5kz/33MTnNm1d455kxqkd8wzYxf2r9GNF7rtaveE/G47nA9M+SYKyAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACCLinMdUxB4Uj8cVi8U0S/M1KGIfbjiQHF9uHxLammXfz6+WrrKHJK1r+Lw5Myr9fXPme7+8yZyJ/cE+wFSSrv+bXebMxt981pwZ9G9RcyalzeN9ivh9evvsq3VE0pwZOeGEOROLtpgzZ9r9vpasHP8P5syyDUvNmeL/scOc6cvaXULbtEmNjY3KzMw873ZcAQEAgqCAAABBmAto+/btuuGGG1RYWKhIJKLnn3++y+POOT344IMqKCjQkCFDVFpaqgMHDnTXegEAA4S5gJqamjR16lStXr36nI+vWrVKP/7xj/X4449r165dGjZsmObOnauWFvvPbQEAA5f5L6KWlZWprKzsnI855/Too4/qgQce0Pz58yVJTzzxhPLy8vT888/rlltu+WSrBQAMGN36HFBNTY3q6upUWlraeV8sFlNJSYl27Dj3qzxaW1sVj8e73AAAA1+3FlBdXZ0kKS8vr8v9eXl5nY99WGVlpWKxWOetqKioO5cEAOijgr8KrqKiQo2NjZ232tra0EsCAPSCbi2g/Px8SVJ9fX2X++vr6zsf+7BoNKrMzMwuNwDAwNetBVRcXKz8/Hxt3bq18754PK5du3ZpxowZ3bkrAEA/Z34V3OnTp3Xw4MHOt2tqarRv3z5lZ2dr9OjRuvvuu/X9739fV1xxhYqLi/Xtb39bhYWFWrBgQXeuGwDQz5kLaPfu3br++us7316xYoUkadGiRVq3bp3uu+8+NTU1aenSpWpoaNC1116rzZs3a/Dgwd23agBAv3dJDyN9Z5XfjwX/7safmzMr/vmvzJn09HZz5gdTnzNnNjdMMWckKUX2U+d3jXkX3uhD3tt7uTnTcVnCnJGkwbXp5kzmO/bjkNJuz3Sk2weEJs3fYp7lUu2ZZJp9fZGk/Ticntlszny26Ig5I0lHT8fMmWvy3jFn3nrf/urfI+9nmTOSNPor/+qVs2AYKQCgT6OAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAIz1m5A8Nvb/97r9zCg39hzuT8o/3PUZxeeMqc+dnR68yZxja/P5VxR9Eb5syJtmHmTM3gpDmjDvtkZklqu8y+r8RX/mTOjIo1mjMjo6fNmWiqfaK6JGUNsk+cTniM0G7qiJozMzOr7ftJ2vcjSW8OGm/OpMp+Dg0b1GbOvDR9jTkjSbfefq85E3typ9e+LoQrIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIYsAMI225Ybo5kxbZ57evb+WbMzn/6z1z5uXxz5kzD5+wH4ehKfZBiJK08vUF5kxK3H7KuSyPgZoe80vP7ithzsQPXGbOHGwYYc7U2meeKrXV2UOeOqL2AbDOY2bs6+mfM2duX/yKfUeSrsv6gznzucGHzZmX064yZ/7in+80ZyTpbyp+Zc68/GSm174uhCsgAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAhiwAwjrf98770rWT+sNWf+MudfzJmfNtgHFP5V1j+bM9+t/UtzRpIu/2WqOZMY6jF9UmnmRCTpN4TTpdjX15Fu308yzb4+n7W1Zvkcb0kesYjHzFif/Qz/N/uk2cf/6Xr7jiT9Yf4ac+Y3bfZ3alFsvznzj5mTzRlJ+uvYv5ozv/qzpabtIx2t0r9suuB2XAEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBADZhhp2une29e9l282Zzb8abo5k5seN2e+eei/mjPNf3+5OSNJZ3Lt37/4DMdMbTNHlEz1G8KZ0ksDNX0Gdzr77FevtUlS+xC/nJXPcTg12n7eFVTZ9yNJX5s205wpHNxgzlSfzjNnFl6+15yRpBEp9g/umcuHmbZvT6RKFzF/mSsgAEAQFBAAIAhzAW3fvl033HCDCgsLFYlE9Pzzz3d5fPHixYpEIl1u8+bN6671AgAGCHMBNTU1aerUqVq9evV5t5k3b56OHTvWeXvqqac+0SIBAAOP+UUIZWVlKisr+9htotGo8vPzvRcFABj4euQ5oG3btik3N1cTJkzQsmXLdPLkyfNu29raqng83uUGABj4ur2A5s2bpyeeeEJbt27VD3/4Q1VVVamsrEwdHR3n3L6yslKxWKzzVlRU1N1LAgD0Qd3+e0C33HJL578nT56sKVOmaNy4cdq2bZtmz579ke0rKiq0YsWKzrfj8TglBACXgB5/GfbYsWOVk5OjgwcPnvPxaDSqzMzMLjcAwMDX4wV05MgRnTx5UgUFBT29KwBAP2L+Edzp06e7XM3U1NRo3759ys7OVnZ2th566CEtXLhQ+fn5OnTokO677z6NHz9ec+fO7daFAwD6N3MB7d69W9dff33n2x88f7No0SKtWbNG+/fv189//nM1NDSosLBQc+bM0fe+9z1Fo9HuWzUAoN8zF9CsWbPknDvv4y+//PInWpCvQc29t68DbfbfcXo43z448P+dtj8f1viz/2LOJGN+EysTw+y5yLlfDPmxOtLtGd8hnB9zap9/V700WNRrQKjncRh0xp5J8Rga63M++By7jqjfgfj1+qnmzIYVD5sz/5Q+zpy5esi75owkxZNJc2ZY9QnT9u0drRe1HbPgAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEES3/0nuUIacsE949TUu7bg5c6TdPl74vpfuNGcyRti/p0gMM0ckSelxe8Z5fMvTMdiekcdUa0lqH+qxK4/pzD7r85m67Ssx3L7ApMdXk9Q2+5TqlIsbtNxF/FN+07AzDtuPw7cOzzdn/u+4LebMtjMeJ6ukwtRT5kzHgXds27vERW3HFRAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABDFghpFmHjptziRch9e+xqa1mDMzdywzZ/LfsA9C/NNEc8R7yGVLjj3TEbW/T9H3PQZJen5r5TMstX2o/X1qz7CfeynDL27A43+WbPWZlCpFj6aZM2mn7R8nr2PnMZw2tdVvGOmZXHvunfVXmDO/u/8fzBlpuEdGuixliDmTOmG8aXvX0SoduPB2XAEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBADZhhpSnObOZMW8RvU+K9tmebM8F/ZBwf+aaLHAMWkPdIxxD4QUpLasuw7i56wH/P0U/b1pS84bs5I0snGYeZMR5v90yj9cNScyd5u/34x4veh1Zls+7nXXOgxWNRjGKl85or6zSJVIsO+viEp9o/T/J13mjO/nPETc0aS2mU/9yIJ28TiSPLitucKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCGDDDSNXeYY40Js947epvd37NnEkbZZ+G2Jpjf59SWu37iST8JjWm5zebM2N/FDdn2t87Ys7UR0vMGUkav/WkPXS01p7JHWGOvHdTrjnTPMY2RPIDkaH24b7ujMdw36TH+dpuz3Sk+k1ldYPsueZR9kzGTvuw4rrpQ80ZSRqXZr/uaH/nXdv2LnFR23EFBAAIggICAARhKqDKykpdffXVysjIUG5urhYsWKDq6uou27S0tKi8vFwjRozQ8OHDtXDhQtXX13frogEA/Z+pgKqqqlReXq6dO3fqlVdeUSKR0Jw5c9TU1NS5zT333KMXXnhBGzZsUFVVlY4ePaqbbrqp2xcOAOjfTC9C2Lx5c5e3161bp9zcXO3Zs0czZ85UY2OjfvrTn2r9+vX60pe+JElau3atPv3pT2vnzp36whe+0H0rBwD0a5/oOaDGxkZJUnZ2tiRpz549SiQSKi0t7dxm4sSJGj16tHbs2HHO/6O1tVXxeLzLDQAw8HkXUDKZ1N13361rrrlGkyZNkiTV1dUpPT1dWVlZXbbNy8tTXV3dOf+fyspKxWKxzltRUZHvkgAA/Yh3AZWXl+vtt9/W008//YkWUFFRocbGxs5bba3H71QAAPodr19EXb58uV588UVt375do0aN6rw/Pz9fbW1tamho6HIVVF9fr/z8/HP+X9FoVNFo1GcZAIB+zHQF5JzT8uXLtXHjRr366qsqLi7u8vi0adOUlpamrVu3dt5XXV2tw4cPa8aMGd2zYgDAgGC6AiovL9f69eu1adMmZWRkdD6vE4vFNGTIEMViMX31q1/VihUrlJ2drczMTN11112aMWMGr4ADAHRhKqA1a9ZIkmbNmtXl/rVr12rx4sWSpB/96EdKSUnRwoUL1draqrlz5+onP/lJtywWADBwmArIuQsP2Rs8eLBWr16t1atXey/KS6r99RQvNo268Ebn4JIeGY95n6nNHkMDh3sMMI34vRYl2Wp/CrH5ypHmTHrNe+bM5c8eMmckqf4vxpozJ6/JMGeyRpw2Z1pP24fnRt5PN2ckSQ1p9n35zLT1OfV8Mn6zSBVJ2HfmhtsHwDYX2vfz11v+1pyRpJq//D/mTOqIbNP2LtkmvX/h7ZgFBwAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCC8/iJqX9TxuwPmzLCUVq99PXHNT82Zv04sNWcizanmTGosYc4kO/wmJrsW++kTX95ozqTcdaU509xqn+YsSc6dsofeH2KONL6bZc5E7IPOpTTPMdD2U89r4rRL9Qh5TKOP+Iyjl+QG2w96aoP986JjmP2dSq/vvS/fLX9WfOGN/pP29hbptQtvxxUQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAAQxYIaR+shKafbKpXpMXfzRdU+bM4WD/mTO/OLkn5szL74xzZyRJHXYBzy+fyTLnElttn+f5HrxW6uIx0BNl2YfPun8ZsZ6ibR7DO/0mffpMyvVYz8uxXMoq8c5nox67ssoJeE3YLU52WbOtGXZqqI9cXHbcwUEAAiCAgIABEEBAQCCoIAAAEFQQACAICggAEAQFBAAIAgKCAAQBAUEAAiCAgIABEEBAQCCoIAAAEFc0sNIt5y6yis3Zehhc+Zyj8GiE9LazZlPDT5pzvy3Wf9kzkjS+t9cbQ8dHmKOpNgPgxIx+7BPSYp4DJ+MJO2ZlDN+gyStXO/sRpIU8ZjB6VLsC/QZNOuztrP78nmnfM4h+24GtdgzklTT3mHODD6RMG3f3n5x23MFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBXNrDSI9O8MrFRjebM+PS/mjO/CJ+pTmTFrEPGizL2G/OSNKUz9eaMyNKTpszean2zMb4n5kzkvTSUfuA2vak/fu45rY0cybpsZ9omm2I5AeGeAzC9TkOg1LsUzh95oqebol6pKTYEPvEz8LhjeZMW0eqOZP0mcoq6Y0z48yZYzMGm7bvaJV0ETOOuQICAARBAQEAgjAVUGVlpa6++mplZGQoNzdXCxYsUHV1dZdtZs2apUgk0uV25513duuiAQD9n6mAqqqqVF5erp07d+qVV15RIpHQnDlz1NTU1GW7JUuW6NixY523VatWdeuiAQD9n+lFCJs3b+7y9rp165Sbm6s9e/Zo5syZnfcPHTpU+fn53bNCAMCA9ImeA2psPPtqj+zs7C73P/nkk8rJydGkSZNUUVGh5ubzv2qstbVV8Xi8yw0AMPB5vww7mUzq7rvv1jXXXKNJkyZ13n/bbbdpzJgxKiws1P79+3X//ferurpazz333Dn/n8rKSj300EO+ywAA9FPeBVReXq63335br7/+epf7ly5d2vnvyZMnq6CgQLNnz9ahQ4c0btxHX39eUVGhFStWdL4dj8dVVFTkuywAQD/hVUDLly/Xiy++qO3bt2vUqFEfu21JSYkk6eDBg+csoGg0qmjU75fEAAD9l6mAnHO66667tHHjRm3btk3FxcUXzOzbt0+SVFBQ4LVAAMDAZCqg8vJyrV+/Xps2bVJGRobq6uokSbFYTEOGDNGhQ4e0fv16ffnLX9aIESO0f/9+3XPPPZo5c6amTJnSI+8AAKB/MhXQmjVrJJ39ZdP/bO3atVq8eLHS09O1ZcsWPfroo2pqalJRUZEWLlyoBx54oNsWDAAYGMw/gvs4RUVFqqqq+kQLAgBcGi7padhDPCcFL8v6jTlzMBExZ8qz7NOm/dgn8Z5l/52t5mSbOTM0Zag58+mc6gtvdA7LLttrzlyWal+fjzda7JOjG5J+a8tPtX9sB3tMYu+Q/fOixdnP15EpreaMJBWnDTdn9rTaz/HxafZjt6Mly5yRpPV/LDFnRlW+adq+3SV04CK2YxgpACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEAQFBAAIggICAARxaQ8j/brfX2Itm/B1cybaYB98mkzrne8POqJ++zlyvT0XybMPhcx4c4g5U/jLo+aMJLlU+/vUcdkwcyb1VIs5o2PHzRGXaLfvR1JkqH2IaWS4x+DTC0zYP6d2++BOX81X2f+Qps/n7fA9h82Z9mN15sxZ9kGzPYUrIABAEBQQACAICggAEAQFBAAIggICAARBAQEAgqCAAABBUEAAgCAoIABAEBQQACAICggAEESfmwXn/n02VLsSkseYKNO+OuxzySSpPWGf45Xa7jELLtJLs+BS/PaTbPGYBddsP+YdbRFzpj3p97F1Ht+TdbSn2vfjc+65NnvEec6CS9q/NESS9uPgNQsu2Xuz4Nrb7Z/rSY9zqD1p/9i2O/vXlN7SrrNrcxf4+EbchbboZUeOHFFRUVHoZQAAPqHa2lqNGjXqvI/3uQJKJpM6evSoMjIyFIl0/c43Ho+rqKhItbW1yszMDLTC8DgOZ3EczuI4nMVxOKsvHAfnnE6dOqXCwkKlfMxPWPrcj+BSUlI+tjElKTMz85I+wT7AcTiL43AWx+EsjsNZoY9DLBa74Da8CAEAEAQFBAAIol8VUDQa1cqVKxWN+v0l04GC43AWx+EsjsNZHIez+tNx6HMvQgAAXBr61RUQAGDgoIAAAEFQQACAICggAEAQ/aaAVq9erU996lMaPHiwSkpK9Otf/zr0knrdd77zHUUikS63iRMnhl5Wj9u+fbtuuOEGFRYWKhKJ6Pnnn+/yuHNODz74oAoKCjRkyBCVlpbqwIEDYRbbgy50HBYvXvyR82PevHlhFttDKisrdfXVVysjI0O5ublasGCBqquru2zT0tKi8vJyjRgxQsOHD9fChQtVX18faMU942KOw6xZsz5yPtx5552BVnxu/aKAnnnmGa1YsUIrV67UW2+9palTp2ru3Lk6fvx46KX1uquuukrHjh3rvL3++uuhl9TjmpqaNHXqVK1evfqcj69atUo//vGP9fjjj2vXrl0aNmyY5s6dq5YW+yDJvuxCx0GS5s2b1+X8eOqpp3pxhT2vqqpK5eXl2rlzp1555RUlEgnNmTNHTU1Nndvcc889euGFF7RhwwZVVVXp6NGjuummmwKuuvtdzHGQpCVLlnQ5H1atWhVoxefh+oHp06e78vLyzrc7OjpcYWGhq6ysDLiq3rdy5Uo3derU0MsISpLbuHFj59vJZNLl5+e7hx9+uPO+hoYGF41G3VNPPRVghb3jw8fBOecWLVrk5s+fH2Q9oRw/ftxJclVVVc65sx/7tLQ0t2HDhs5tfve73zlJbseOHaGW2eM+fBycc+6LX/yi+/rXvx5uURehz18BtbW1ac+ePSotLe28LyUlRaWlpdqxY0fAlYVx4MABFRYWauzYsbr99tt1+PDh0EsKqqamRnV1dV3Oj1gsppKSkkvy/Ni2bZtyc3M1YcIELVu2TCdPngy9pB7V2NgoScrOzpYk7dmzR4lEosv5MHHiRI0ePXpAnw8fPg4fePLJJ5WTk6NJkyapoqJCzc3NIZZ3Xn1uGOmHnThxQh0dHcrLy+tyf15enn7/+98HWlUYJSUlWrdunSZMmKBjx47poYce0nXXXae3335bGRkZoZcXRF1dnSSd8/z44LFLxbx583TTTTepuLhYhw4d0re+9S2VlZVpx44dSk31+Fs9fVwymdTdd9+ta665RpMmTZJ09nxIT09XVlZWl20H8vlwruMgSbfddpvGjBmjwsJC7d+/X/fff7+qq6v13HPPBVxtV32+gPAfysrKOv89ZcoUlZSUaMyYMXr22Wf11a9+NeDK0Bfccsstnf+ePHmypkyZonHjxmnbtm2aPXt2wJX1jPLycr399tuXxPOgH+d8x2Hp0qWd/548ebIKCgo0e/ZsHTp0SOPGjevtZZ5Tn/8RXE5OjlJTUz/yKpb6+nrl5+cHWlXfkJWVpSuvvFIHDx4MvZRgPjgHOD8+auzYscrJyRmQ58fy5cv14osv6rXXXuvy51vy8/PV1tamhoaGLtsP1PPhfMfhXEpKSiSpT50Pfb6A0tPTNW3aNG3durXzvmQyqa1bt2rGjBkBVxbe6dOndejQIRUUFIReSjDFxcXKz8/vcn7E43Ht2rXrkj8/jhw5opMnTw6o88M5p+XLl2vjxo169dVXVVxc3OXxadOmKS0trcv5UF1drcOHDw+o8+FCx+Fc9u3bJ0l963wI/SqIi/H000+7aDTq1q1b537729+6pUuXuqysLFdXVxd6ab3qG9/4htu2bZurqalxb7zxhistLXU5OTnu+PHjoZfWo06dOuX27t3r9u7d6yS5Rx55xO3du9e99957zjnnfvCDH7isrCy3adMmt3//fjd//nxXXFzszpw5E3jl3evjjsOpU6fcvffe63bs2OFqamrcli1b3Oc+9zl3xRVXuJaWltBL7zbLli1zsVjMbdu2zR07dqzz1tzc3LnNnXfe6UaPHu1effVVt3v3bjdjxgw3Y8aMgKvufhc6DgcPHnTf/e533e7du11NTY3btGmTGzt2rJs5c2bglXfVLwrIOecee+wxN3r0aJeenu6mT5/udu7cGXpJve7mm292BQUFLj093V1++eXu5ptvdgcPHgy9rB732muvOUkfuS1atMg5d/al2N/+9rddXl6ei0ajbvbs2a66ujrsonvAxx2H5uZmN2fOHDdy5EiXlpbmxowZ45YsWTLgvkk71/svya1du7ZzmzNnzrivfe1r7rLLLnNDhw51N954ozt27Fi4RfeACx2Hw4cPu5kzZ7rs7GwXjUbd+PHj3Te/+U3X2NgYduEfwp9jAAAE0eefAwIADEwUEAAgCAoIABAEBQQACIICAgAEQQEBAIKggAAAQVBAAIAgKCAAQBAUEAAgCAoIABAEBQQACOL/AyBNQnoqGwl/AAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -2686,22 +2506,16 @@ }, { "cell_type": "code", - "execution_count": 79, - "id": "ab2fe42f-a072-4370-bac2-52fd95363530", + "execution_count": null, + "id": "e02838ba", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "2025-02-04 14:00:18,330 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-04 14:00:28,520 - INFO - Sucessfully stopped 1 servers. \n" ] }, { @@ -2710,20 +2524,17 @@ "[True]" ] }, - "execution_count": 79, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ - "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", - "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" + "server_manager.stop_servers()" ] }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 79, "id": "a0608fff-7cfb-489e-96c9-8e1d92e57562", "metadata": {}, "outputs": [], @@ -2757,7 +2568,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.11.11" } }, "nbformat": 4, diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytriton_utils.py b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytriton_utils.py index 74423cc4..d9ab18d9 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytriton_utils.py +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytriton_utils.py @@ -13,31 +13,63 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import os import inspect -import socket -import psutil +import logging +import os import signal +import socket import time -from pyspark import RDD from multiprocessing import Process +from typing import Dict, List, Optional, Tuple + +import psutil +from pyspark import RDD +from pyspark.sql import SparkSession + from pytriton.client import ModelClient -from typing import Dict, List, Optional +DEFAULT_WAIT_RETRIES = 10 +DEFAULT_WAIT_TIMEOUT = 5 -def start_triton(triton_server_fn: callable, ports: List[int], model_name: str, model_path: Optional[str] = None) -> List[tuple]: - """ - Start a Triton server in a separate process and wait for it to be ready. - Return the (hostname, PID) of the node and process. - """ +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger("TritonManager") + + +def _start_triton_server( + triton_server_fn: callable, + model_name: str, + model_path: Optional[str] = None, + max_retries: int = DEFAULT_WAIT_RETRIES, + wait_timeout: int = DEFAULT_WAIT_TIMEOUT, +) -> List[tuple]: + """Task to start Triton server process on a Spark executor.""" sig = inspect.signature(triton_server_fn) params = sig.parameters + def _find_ports(start_port: int = 7000) -> List[int]: + """Find available ports for Triton's HTTP, gRPC, and metrics services.""" + ports = [] + conns = {conn.laddr.port for conn in psutil.net_connections(kind="inet")} + i = start_port + + while len(ports) < 3: + if i not in conns: + ports.append(i) + i += 1 + + return ports + + ports = _find_ports() + if model_path is not None: - assert len(params) == 2, "To pass a model_path for the server to load, make sure it accepts two arguments: ports and model_path" + assert ( + len(params) == 2 + ), "Server function must accept (ports, model_path) when model_path is provided" args = (ports, model_path) else: - assert len(params) == 1, "To start a Triton server, the function must accept one argument: ports" + assert len(params) == 1, "Server function must accept (ports) argument" args = (ports,) hostname = socket.gethostname() @@ -45,86 +77,207 @@ def start_triton(triton_server_fn: callable, ports: List[int], model_name: str, process.start() client = ModelClient(f"http://localhost:{ports[0]}", model_name) - patience = 10 - while patience > 0: + + for _ in range(max_retries): try: - client.wait_for_model(5) - return [(hostname, process.pid)] + client.wait_for_model(wait_timeout) + client.close() + return [(hostname, (process.pid, ports))] except Exception: - print("Waiting for server to be ready...") - patience -= 1 - - emsg = "Failure: client waited too long for server startup. Check the executor logs for more info." - raise TimeoutError(emsg) - -def find_ports() -> List[int]: - """ - Find three available network ports starting from port 7000 for Triton's HTTP, gRPC, and metrics services. - """ - ports = [] - conns = {conn.laddr.port for conn in psutil.net_connections(kind="inet")} - i = 7000 - while len(ports) < 3: - if i not in conns: - ports.append(i) - i += 1 - - return ports - -def use_stage_level_scheduling(spark, rdd: RDD) -> RDD: - """ - From https://github.com/NVIDIA/spark-rapids-ml/blob/main/python/src/spark_rapids_ml/core.py - Used to ensure each Triton server instance requires a full GPU to create a 1:1 executor-server mapping. - """ + pass - executor_cores = spark.conf.get("spark.executor.cores") - assert executor_cores is not None, "spark.executor.cores is not set" - executor_gpus = spark.conf.get("spark.executor.resource.gpu.amount") - assert executor_gpus is not None and int(executor_gpus) == 1, "spark.executor.resource.gpu.amount must be set and = 1" - - from pyspark.resource.profile import ResourceProfileBuilder - from pyspark.resource.requests import TaskResourceRequests - - # each training task requires cpu cores > total executor cores/2 which can - # ensure each training task be sent to different executor. - # - # Please note that we can't set task_cores to the value which is smaller than total executor cores/2 - # because only task_gpus can't ensure the tasks be sent to different executor even task_gpus=1.0 - # - # If spark-rapids enabled. we don't allow other ETL task running alongside training task to avoid OOM - spark_plugins = spark.conf.get("spark.plugins", " ") - assert spark_plugins is not None - spark_rapids_sql_enabled = spark.conf.get("spark.rapids.sql.enabled", "true") - assert spark_rapids_sql_enabled is not None - - task_cores = ( - int(executor_cores) - if "com.nvidia.spark.SQLPlugin" in spark_plugins - and "true" == spark_rapids_sql_enabled.lower() - else (int(executor_cores) // 2) + 1 + raise TimeoutError( + "Failure: server startup timeout exceeded. Check the executor logs for more info." ) - # task_gpus means how many slots per gpu address the task requires, - # it does mean how many gpus it would like to require, so it can be any value of (0, 0.5] or 1. - task_gpus = 1.0 - treqs = TaskResourceRequests().cpus(task_cores).resource("gpu", task_gpus) - rp = ResourceProfileBuilder().require(treqs).build - print(f"Requesting stage-level resources: (cores={task_cores}, gpu={task_gpus})") - return rdd.withResources(rp) -def stop_triton(pids: Dict[str, int]) -> List[bool]: - """ - Stop Triton server instances by sending a SIGTERM signal. - """ +def _stop_triton_server( + server_pids_ports: Dict[str, Tuple[int, List[int]]], + max_retries: int = DEFAULT_WAIT_RETRIES, + retry_delay: int = DEFAULT_WAIT_TIMEOUT, +) -> List[bool]: + """Task to stop the Triton server on a Spark executor.""" hostname = socket.gethostname() - pid = pids.get(hostname, None) - assert pid is not None, f"Could not find pid for {hostname}" - - for _ in range(5): + pid, _ = server_pids_ports.get(hostname) + assert pid is not None, f"No server PID found for host {hostname}" + + for _ in range(max_retries): try: os.kill(pid, signal.SIGTERM) except OSError: return [True] - time.sleep(5) + time.sleep(retry_delay) + + return [False] # Failed to terminate or timed out + + +class TritonServerManager: + """ + Handle lifecycle of Triton server instances across Spark cluster, e.g. + - Find available ports + - Start server processes across executors via stage-level scheduling + - Gracefully shutdown servers across executors + + Attributes: + spark: Active SparkSession + num_nodes: Number of Triton servers to manage (= # of executors/GPUs) + model_name: Name of the model being served + model_path: Optional path to model files + server_pids_ports: Dictionary of hostname to (server process ID, ports) + + Example usage: + >>> server_manager = TritonServerManager(num_nodes=4, model_name="my_model", model_path="/path/to/my_model") + >>> # Define triton_server(ports, model_path) that contains PyTriton server logic + >>> server_pids_ports = server_manager.start_servers(triton_server) + >>> print(f"Servers started with PIDs/Ports: {server_pids_ports}") + >>> host_to_http_url = server_manager.host_to_http_url + >>> host_to_grpc_url = server_manager.host_to_grpc_url + >>> # Define triton_fn() and predict_batch_udf(triton_fn) and run inference... + >>> success = server_manager.stop_servers() + >>> print(f"Server shutdown success: {success}") + """ + + def __init__( + self, num_nodes: int, model_name: str, model_path: Optional[str] = None + ): + """ + Initialize the Triton server manager. + + Args: + num_nodes: Number of executors (GPUs) in cluster + model_name: Name of the model to serve + model_path: Optional path to model file for server function to load from disk + """ + self.spark = SparkSession.getActiveSession() + self.num_nodes = num_nodes + self.model_name = model_name + self.model_path = model_path + self._server_pids_ports: Dict[str, Tuple[int, List[int]]] = {} + + @property + def host_to_http_url(self) -> Dict[str, str]: + """Map hostname to client HTTP URL for Triton server on that host.""" + if not self._server_pids_ports: + logger.warning("No urls available. Start servers first.") + return None + + return { + host: f"http://{host}:{ports[0]}" + for host, (_, ports) in self._server_pids_ports.items() + } + + @property + def host_to_grpc_url(self) -> Dict[str, str]: + """Map hostname to client gRPC URL for Triton server on that host.""" + if not self._server_pids_ports: + logger.warning("No urls available. Start servers first.") + return None + + return { + host: f"grpc://{host}:{ports[1]}" + for host, (_, ports) in self._server_pids_ports.items() + } + + def _get_node_rdd(self) -> RDD: + """Create and configure RDD with stage-level scheduling for 1 task per node.""" + sc = self.spark.sparkContext + node_rdd = sc.parallelize(list(range(self.num_nodes)), self.num_nodes) + return self._use_stage_level_scheduling(node_rdd) + + def _use_stage_level_scheduling(self, rdd: RDD, task_gpus: float = 1.0) -> RDD: + """ + Use stage-level scheduling to ensure each Triton server instance maps to 1 GPU (executor). + From https://github.com/NVIDIA/spark-rapids-ml/blob/main/python/src/spark_rapids_ml/core.py + """ + executor_cores = self.spark.conf.get("spark.executor.cores") + assert executor_cores is not None, "spark.executor.cores is not set" + executor_gpus = self.spark.conf.get("spark.executor.resource.gpu.amount") + assert ( + executor_gpus is not None and int(executor_gpus) == 1 + ), "spark.executor.resource.gpu.amount must be set and = 1" + + from pyspark.resource.profile import ResourceProfileBuilder + from pyspark.resource.requests import TaskResourceRequests + + spark_plugins = self.spark.conf.get("spark.plugins", " ") + assert spark_plugins is not None + spark_rapids_sql_enabled = self.spark.conf.get( + "spark.rapids.sql.enabled", "true" + ) + assert spark_rapids_sql_enabled is not None + + task_cores = ( + int(executor_cores) + if "com.nvidia.spark.SQLPlugin" in spark_plugins + and "true" == spark_rapids_sql_enabled.lower() + else (int(executor_cores) // 2) + 1 + ) + treqs = TaskResourceRequests().cpus(task_cores).resource("gpu", task_gpus) + rp = ResourceProfileBuilder().require(treqs).build + logger.info( + f"Requesting stage-level resources: (cores={task_cores}, gpu={task_gpus})" + ) + + return rdd.withResources(rp) + + def start_servers( + self, triton_server_fn: callable + ) -> Dict[str, Tuple[int, List[int]]]: + """ + Start Triton servers across the cluster. + + Args: + triton_server_fn: PyTriton server function defining the model and inference logic + + Returns: + Dictionary of hostname -> server PID + """ + node_rdd = self._get_node_rdd() + model_name = self.model_name + model_path = self.model_path + + logger.info(f"Starting {self.num_nodes} servers.") + + self._server_pids_ports = ( + node_rdd.barrier() + .mapPartitions( + lambda _: _start_triton_server( + triton_server_fn=triton_server_fn, + model_name=model_name, + model_path=model_path, + ) + ) + .collectAsMap() + ) + + return self._server_pids_ports + + def stop_servers(self) -> List[bool]: + """ + Stop all Triton servers across the cluster. + + Returns: + List of booleans indicating success/failure of stopping each server + """ + if not self._server_pids_ports: + logger.warning("No servers to stop.") + return + + node_rdd = self._get_node_rdd() + server_pids_ports = self._server_pids_ports + + stop_success = ( + node_rdd.barrier() + .mapPartitions(lambda _: _stop_triton_server(server_pids_ports)) + .collect() + ) + + if all(stop_success): + self._server_pids_ports.clear() + logger.info(f"Sucessfully stopped {self.num_nodes} servers.") + else: + logger.warning( + f"Server termination failed or timed out. Check executor logs." + ) - return [False] + return stop_success diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/image_classification_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/image_classification_tf.ipynb index aec9bc8d..3fcdf3a5 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/image_classification_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/image_classification_tf.ipynb @@ -32,13 +32,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-01-27 12:09:45.634884: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2025-01-27 12:09:45.641535: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2025-01-27 12:09:45.649019: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2025-01-27 12:09:45.651240: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2025-01-27 12:09:45.657098: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "2025-02-04 13:58:23.275397: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2025-02-04 13:58:23.282713: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2025-02-04 13:58:23.290717: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2025-02-04 13:58:23.293187: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2025-02-04 13:58:23.299616: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2025-01-27 12:09:46.013264: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" + "2025-02-04 13:58:23.677341: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" ] } ], @@ -71,9 +71,9 @@ "output_type": "stream", "text": [ "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", - "I0000 00:00:1738008586.411773 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008586.433071 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008586.435784 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" + "I0000 00:00:1738706304.084788 3671509 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706304.107153 3671509 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706304.109954 3671509 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" ] } ], @@ -169,13 +169,19 @@ "text": [ "/home/rishic/anaconda3/envs/spark-dl-tf/lib/python3.11/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.\n", " super().__init__(activity_regularizer=activity_regularizer, **kwargs)\n", - "I0000 00:00:1738008586.592974 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008586.595670 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008586.598309 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008586.702447 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008586.703486 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008586.704390 3021472 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "2025-01-27 12:09:46.705298: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 40715 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" + "I0000 00:00:1738706304.278396 3671509 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "I0000 00:00:1738706304.281131 3671509 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706304.283741 3671509 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706304.403175 3671509 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706304.404296 3671509 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706304.405232 3671509 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2025-02-04 13:58:24.406153: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 40769 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" ] }, { @@ -318,38 +324,38 @@ "output_type": "stream", "text": [ "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", - "I0000 00:00:1738008587.286019 3021614 service.cc:146] XLA service 0x7e8e7c005520 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n", - "I0000 00:00:1738008587.286032 3021614 service.cc:154] StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n", - "2025-01-27 12:09:47.295640: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", - "2025-01-27 12:09:47.333525: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8907\n" + "I0000 00:00:1738706304.982690 3671754 service.cc:146] XLA service 0x7f1464019260 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n", + "I0000 00:00:1738706304.982718 3671754 service.cc:154] StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n", + "2025-02-04 13:58:24.999594: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", + "2025-02-04 13:58:25.043847: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8907\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m25s\u001b[0m 821ms/step - loss: 2.3309 - sparse_categorical_accuracy: 0.1562" + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m26s\u001b[0m 868ms/step - loss: 2.4638 - sparse_categorical_accuracy: 0.0625" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "I0000 00:00:1738008587.874862 3021614 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n" + "I0000 00:00:1738706305.619913 3671754 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - loss: 1.5918 - sparse_categorical_accuracy: 0.5187 " + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 17ms/step - loss: 1.6323 - sparse_categorical_accuracy: 0.4913 " ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025-01-27 12:09:49.034107: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_33', 4 bytes spill stores, 4 bytes spill loads\n", + "2025-02-04 13:58:26.791107: I external/local_xla/xla/stream_executor/cuda/cuda_asm_compiler.cc:393] ptxas warning : Registers are spilled to local memory in function 'gemm_fusion_dot_33', 4 bytes spill stores, 4 bytes spill loads\n", "\n" ] }, @@ -358,50 +364,50 @@ "output_type": "stream", "text": [ "\n", - "Epoch 1: val_sparse_categorical_accuracy improved from -inf to 0.79300, saving model to models/training_1/checkpoint.model.keras\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 49ms/step - loss: 1.5776 - sparse_categorical_accuracy: 0.5239 - val_loss: 0.6896 - val_sparse_categorical_accuracy: 0.7930\n", + "Epoch 1: val_sparse_categorical_accuracy improved from -inf to 0.76100, saving model to models/training_1/checkpoint.model.keras\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 48ms/step - loss: 1.6179 - sparse_categorical_accuracy: 0.4965 - val_loss: 0.7533 - val_sparse_categorical_accuracy: 0.7610\n", "Epoch 2/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 15ms/step - loss: 0.5264 - sparse_categorical_accuracy: 0.9062\n", - "Epoch 2: val_sparse_categorical_accuracy improved from 0.79300 to 0.82800, saving model to models/training_1/checkpoint.model.keras\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.4540 - sparse_categorical_accuracy: 0.8786 - val_loss: 0.5568 - val_sparse_categorical_accuracy: 0.8280\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.3965 - sparse_categorical_accuracy: 0.9062\n", + "Epoch 2: val_sparse_categorical_accuracy improved from 0.76100 to 0.80400, saving model to models/training_1/checkpoint.model.keras\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.4549 - sparse_categorical_accuracy: 0.8773 - val_loss: 0.6002 - val_sparse_categorical_accuracy: 0.8040\n", "Epoch 3/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.2200 - sparse_categorical_accuracy: 0.9375\n", - "Epoch 3: val_sparse_categorical_accuracy improved from 0.82800 to 0.84000, saving model to models/training_1/checkpoint.model.keras\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.2886 - sparse_categorical_accuracy: 0.9285 - val_loss: 0.5069 - val_sparse_categorical_accuracy: 0.8400\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.4427 - sparse_categorical_accuracy: 0.8438\n", + "Epoch 3: val_sparse_categorical_accuracy improved from 0.80400 to 0.85100, saving model to models/training_1/checkpoint.model.keras\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.2924 - sparse_categorical_accuracy: 0.9289 - val_loss: 0.4876 - val_sparse_categorical_accuracy: 0.8510\n", "Epoch 4/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - loss: 0.2431 - sparse_categorical_accuracy: 0.9375\n", - "Epoch 4: val_sparse_categorical_accuracy improved from 0.84000 to 0.84800, saving model to models/training_1/checkpoint.model.keras\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - loss: 0.2302 - sparse_categorical_accuracy: 0.9448 - val_loss: 0.4648 - val_sparse_categorical_accuracy: 0.8480\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.3644 - sparse_categorical_accuracy: 0.9375\n", + "Epoch 4: val_sparse_categorical_accuracy did not improve from 0.85100\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.2790 - sparse_categorical_accuracy: 0.9275 - val_loss: 0.4981 - val_sparse_categorical_accuracy: 0.8430\n", "Epoch 5/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.1749 - sparse_categorical_accuracy: 0.9375\n", - "Epoch 5: val_sparse_categorical_accuracy improved from 0.84800 to 0.87300, saving model to models/training_1/checkpoint.model.keras\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - loss: 0.1602 - sparse_categorical_accuracy: 0.9631 - val_loss: 0.4069 - val_sparse_categorical_accuracy: 0.8730\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - loss: 0.2368 - sparse_categorical_accuracy: 0.9375\n", + "Epoch 5: val_sparse_categorical_accuracy did not improve from 0.85100\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.1794 - sparse_categorical_accuracy: 0.9645 - val_loss: 0.4893 - val_sparse_categorical_accuracy: 0.8450\n", "Epoch 6/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.0735 - sparse_categorical_accuracy: 1.0000\n", - "Epoch 6: val_sparse_categorical_accuracy did not improve from 0.87300\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.1052 - sparse_categorical_accuracy: 0.9867 - val_loss: 0.4261 - val_sparse_categorical_accuracy: 0.8620\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - loss: 0.0830 - sparse_categorical_accuracy: 1.0000\n", + "Epoch 6: val_sparse_categorical_accuracy improved from 0.85100 to 0.85400, saving model to models/training_1/checkpoint.model.keras\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.1430 - sparse_categorical_accuracy: 0.9739 - val_loss: 0.4338 - val_sparse_categorical_accuracy: 0.8540\n", "Epoch 7/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - loss: 0.0753 - sparse_categorical_accuracy: 1.0000\n", - "Epoch 7: val_sparse_categorical_accuracy did not improve from 0.87300\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0881 - sparse_categorical_accuracy: 0.9881 - val_loss: 0.4703 - val_sparse_categorical_accuracy: 0.8580\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.1518 - sparse_categorical_accuracy: 1.0000\n", + "Epoch 7: val_sparse_categorical_accuracy improved from 0.85400 to 0.86200, saving model to models/training_1/checkpoint.model.keras\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0876 - sparse_categorical_accuracy: 0.9909 - val_loss: 0.4194 - val_sparse_categorical_accuracy: 0.8620\n", "Epoch 8/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - loss: 0.0561 - sparse_categorical_accuracy: 1.0000\n", - "Epoch 8: val_sparse_categorical_accuracy did not improve from 0.87300\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0663 - sparse_categorical_accuracy: 0.9902 - val_loss: 0.4121 - val_sparse_categorical_accuracy: 0.8670\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - loss: 0.0209 - sparse_categorical_accuracy: 1.0000\n", + "Epoch 8: val_sparse_categorical_accuracy improved from 0.86200 to 0.86800, saving model to models/training_1/checkpoint.model.keras\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0669 - sparse_categorical_accuracy: 0.9938 - val_loss: 0.4038 - val_sparse_categorical_accuracy: 0.8680\n", "Epoch 9/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.0313 - sparse_categorical_accuracy: 1.0000\n", - "Epoch 9: val_sparse_categorical_accuracy did not improve from 0.87300\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0493 - sparse_categorical_accuracy: 0.9959 - val_loss: 0.4398 - val_sparse_categorical_accuracy: 0.8570\n", + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.0211 - sparse_categorical_accuracy: 1.0000\n", + "Epoch 9: val_sparse_categorical_accuracy improved from 0.86800 to 0.86900, saving model to models/training_1/checkpoint.model.keras\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0429 - sparse_categorical_accuracy: 0.9998 - val_loss: 0.4062 - val_sparse_categorical_accuracy: 0.8690\n", "Epoch 10/10\n", - "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.0590 - sparse_categorical_accuracy: 1.0000\n", - "Epoch 10: val_sparse_categorical_accuracy did not improve from 0.87300\n", - "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0476 - sparse_categorical_accuracy: 0.9981 - val_loss: 0.4099 - val_sparse_categorical_accuracy: 0.8640\n" + "\u001b[1m 1/32\u001b[0m \u001b[37m━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 13ms/step - loss: 0.0283 - sparse_categorical_accuracy: 1.0000\n", + "Epoch 10: val_sparse_categorical_accuracy did not improve from 0.86900\n", + "\u001b[1m32/32\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 1ms/step - loss: 0.0387 - sparse_categorical_accuracy: 0.9992 - val_loss: 0.4069 - val_sparse_categorical_accuracy: 0.8680\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 7, @@ -455,7 +461,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "50eeb6e5", "metadata": {}, "outputs": [ @@ -484,10 +490,10 @@ "Output Type:\n", " TensorSpec(shape=(None, 10), dtype=tf.float32, name=None)\n", "Captures:\n", - " 139159539831760: TensorSpec(shape=(), dtype=tf.resource, name=None)\n", - " 139159206493264: TensorSpec(shape=(), dtype=tf.resource, name=None)\n", - " 139163873256720: TensorSpec(shape=(), dtype=tf.resource, name=None)\n", - " 139159206493456: TensorSpec(shape=(), dtype=tf.resource, name=None)\n" + " 139734758151120: TensorSpec(shape=(), dtype=tf.resource, name=None)\n", + " 139734413261904: TensorSpec(shape=(), dtype=tf.resource, name=None)\n", + " 139739081696528: TensorSpec(shape=(), dtype=tf.resource, name=None)\n", + " 139734413262096: TensorSpec(shape=(), dtype=tf.resource, name=None)\n" ] } ], @@ -514,8 +520,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "32/32 - 0s - 12ms/step - loss: 2.3042 - sparse_categorical_accuracy: 0.1720\n", - "Untrained model, accuracy: 17.20%\n" + "32/32 - 0s - 10ms/step - loss: 2.3876 - sparse_categorical_accuracy: 0.0840\n", + "Untrained model, accuracy: 8.40%\n" ] } ], @@ -530,7 +536,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "22ad1708", "metadata": {}, "outputs": [ @@ -538,8 +544,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "32/32 - 0s - 776us/step - loss: 0.4069 - sparse_categorical_accuracy: 0.8730\n", - "Restored model, accuracy: 87.30%\n" + "32/32 - 0s - 704us/step - loss: 0.4062 - sparse_categorical_accuracy: 0.8690\n", + "Restored model, accuracy: 86.90%\n" ] }, { @@ -613,7 +619,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 13, @@ -707,8 +713,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "32/32 - 0s - 8ms/step - loss: 0.4725 - sparse_categorical_accuracy: 0.8700\n", - "Restored model, accuracy: 87.00%\n" + "32/32 - 0s - 11ms/step - loss: 0.4827 - sparse_categorical_accuracy: 0.8740\n", + "Restored model, accuracy: 87.40%\n" ] } ], @@ -789,12 +795,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 20:09:55 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "25/01/27 20:09:55 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "25/02/04 13:58:33 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/02/04 13:58:33 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "25/01/27 20:09:55 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", - "25/01/27 20:09:55 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + "25/02/04 13:58:33 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" ] } ], @@ -897,7 +902,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 20:09:58 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n", + "25/02/04 13:58:35 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n", "[Stage 0:> (0 + 8) / 8]\r" ] }, @@ -905,8 +910,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.02 ms, sys: 1.31 ms, total: 4.33 ms\n", - "Wall time: 1.92 s\n" + "CPU times: user 3.05 ms, sys: 1.22 ms, total: 4.26 ms\n", + "Wall time: 1.93 s\n" ] }, { @@ -978,8 +983,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 16 μs, sys: 1.01 ms, total: 1.02 ms\n", - "Wall time: 218 ms\n" + "CPU times: user 875 μs, sys: 187 μs, total: 1.06 ms\n", + "Wall time: 196 ms\n" ] } ], @@ -1137,15 +1142,22 @@ "name": "stderr", "output_type": "stream", "text": [ - " \r" + "[Stage 6:===================================================> (7 + 1) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 22.4 ms, sys: 14.1 ms, total: 36.5 ms\n", - "Wall time: 5.89 s\n" + "CPU times: user 24.1 ms, sys: 11 ms, total: 35.2 ms\n", + "Wall time: 5.52 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" ] } ], @@ -1165,8 +1177,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 27.2 ms, sys: 12.8 ms, total: 40 ms\n", - "Wall time: 259 ms\n" + "CPU times: user 21.1 ms, sys: 14.7 ms, total: 35.8 ms\n", + "Wall time: 277 ms\n" ] } ], @@ -1185,8 +1197,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 40.1 ms, sys: 8.81 ms, total: 48.9 ms\n", - "Wall time: 231 ms\n" + "CPU times: user 37.1 ms, sys: 8.46 ms, total: 45.6 ms\n", + "Wall time: 216 ms\n" ] } ], @@ -1247,52 +1259,52 @@ " \n", " 0\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-2.982547, -2.495611, 1.2573445, 12.891551, -...\n", + " [-4.6654954, -2.4895542, -0.5886033, 13.380537...\n", " \n", " \n", " 1\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-0.95476156, -5.622215, 2.241567, -2.0017645,...\n", + " [-2.273215, -7.5127845, 1.1983701, -3.540661, ...\n", " \n", " \n", " 2\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-0.9178914, 0.43411195, 1.9357576, 1.7975042,...\n", + " [-2.28909, 0.8308607, 0.31311005, 1.1683632, -...\n", " \n", " \n", " 3\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [0.21153986, -5.5822043, 11.524151, 2.8064005,...\n", + " [-1.0551968, -6.5028114, 12.420729, 0.45280308...\n", " \n", " \n", " 4\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-3.3954644, 3.7420855, -0.2531985, 0.1679054,...\n", + " [-3.7887802, 3.9983602, -1.5343361, -0.3698440...\n", " \n", " \n", " 5\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-3.1469436, -2.4765797, 3.0782855, 4.3063755,...\n", + " [-4.499274, -1.7618222, 1.1183227, 3.946932, -...\n", " \n", " \n", " 6\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-2.9109762, 4.0204973, 1.3597142, -0.00311854...\n", + " [-2.7540536, 4.8684144, 0.25152916, -0.4730078...\n", " \n", " \n", " 7\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-0.3474851, -0.7892854, -4.275904, 0.51054317...\n", + " [-1.8887109, 0.02717152, -6.0508857, 0.0875094...\n", " \n", " \n", " 8\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [2.0500484, -2.2071126, -0.07787762, -3.207581...\n", + " [0.9541265, -2.113048, -1.7508972, -5.4303794,...\n", " \n", " \n", " 9\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-0.0029632095, -1.7498987, -2.1225448, 2.1881...\n", + " [-1.612412, -0.7655784, -4.473859, 2.0609212, ...\n", " \n", " \n", "\n", @@ -1312,16 +1324,16 @@ "9 [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ... \n", "\n", " preds \n", - "0 [-2.982547, -2.495611, 1.2573445, 12.891551, -... \n", - "1 [-0.95476156, -5.622215, 2.241567, -2.0017645,... \n", - "2 [-0.9178914, 0.43411195, 1.9357576, 1.7975042,... \n", - "3 [0.21153986, -5.5822043, 11.524151, 2.8064005,... \n", - "4 [-3.3954644, 3.7420855, -0.2531985, 0.1679054,... \n", - "5 [-3.1469436, -2.4765797, 3.0782855, 4.3063755,... \n", - "6 [-2.9109762, 4.0204973, 1.3597142, -0.00311854... \n", - "7 [-0.3474851, -0.7892854, -4.275904, 0.51054317... \n", - "8 [2.0500484, -2.2071126, -0.07787762, -3.207581... \n", - "9 [-0.0029632095, -1.7498987, -2.1225448, 2.1881... " + "0 [-4.6654954, -2.4895542, -0.5886033, 13.380537... \n", + "1 [-2.273215, -7.5127845, 1.1983701, -3.540661, ... \n", + "2 [-2.28909, 0.8308607, 0.31311005, 1.1683632, -... \n", + "3 [-1.0551968, -6.5028114, 12.420729, 0.45280308... \n", + "4 [-3.7887802, 3.9983602, -1.5343361, -0.3698440... \n", + "5 [-4.499274, -1.7618222, 1.1183227, 3.946932, -... \n", + "6 [-2.7540536, 4.8684144, 0.25152916, -0.4730078... \n", + "7 [-1.8887109, 0.02717152, -6.0508857, 0.0875094... \n", + "8 [0.9541265, -2.113048, -1.7508972, -5.4303794,... \n", + "9 [-1.612412, -0.7655784, -4.473859, 2.0609212, ... " ] }, "execution_count": 34, @@ -1343,8 +1355,8 @@ { "data": { "text/plain": [ - "array([-2.982547 , -2.495611 , 1.2573445 , 12.891551 , -5.40115 ,\n", - " 2.7227228 , -6.900943 , -1.4402989 , 0.59647846, -3.8927622 ],\n", + "array([-4.6654954, -2.4895542, -0.5886033, 13.380537 , -6.652599 ,\n", + " 2.8400383, -7.9901567, -0.7500452, -2.4487166, -4.349809 ],\n", " dtype=float32)" ] }, @@ -1382,7 +1394,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "id": "eb45ecc9-d376-40c4-ad7b-2bd08ca5aaf6", "metadata": {}, "outputs": [ @@ -1452,7 +1464,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "id": "a40fe207-6246-4b0e-abde-823979878d97", "metadata": {}, "outputs": [ @@ -1482,15 +1494,22 @@ "name": "stderr", "output_type": "stream", "text": [ - " \r" + "[Stage 12:==============> (2 + 6) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 53.2 ms, sys: 31.2 ms, total: 84.4 ms\n", - "Wall time: 6.21 s\n" + "CPU times: user 52.5 ms, sys: 22 ms, total: 74.5 ms\n", + "Wall time: 5.72 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" ] } ], @@ -1509,15 +1528,22 @@ "name": "stderr", "output_type": "stream", "text": [ - " \r" + "[Stage 13:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 54.5 ms, sys: 20.1 ms, total: 74.6 ms\n", - "Wall time: 1.95 s\n" + "CPU times: user 49.4 ms, sys: 31.9 ms, total: 81.2 ms\n", + "Wall time: 1.34 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" ] } ], @@ -1625,7 +1651,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-6.554102, 2.085586, 1.3187729, 0.7378275, -4...\n", + " [-6.9618006, 1.2047814, -0.09570807, 0.0462105...\n", " \n", " \n", " 1\n", @@ -1649,7 +1675,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-3.8565233, 4.518864, -1.2931982, -0.78954405...\n", + " [-5.2882323, 5.902014, -2.0389183, -1.2460864,...\n", " \n", " \n", " 2\n", @@ -1673,7 +1699,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-3.7759259, -2.682865, -1.2959752, -5.4488983...\n", + " [-5.822013, -2.3333628, -2.4322102, -8.040086,...\n", " \n", " \n", " 3\n", @@ -1697,7 +1723,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [0.15489274, -1.4305159, -1.5703316, 1.2637339...\n", + " [-0.57203317, -1.2920653, -2.7234774, 0.914070...\n", " \n", " \n", " 4\n", @@ -1721,7 +1747,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-3.9276226, 4.4247217, 1.0965542, 0.3403727, ...\n", + " [-3.689301, 5.0702505, -0.23930073, -0.7988689...\n", " \n", " \n", " 5\n", @@ -1745,7 +1771,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [8.987603, -3.3934922, 2.4643059, -0.84833026,...\n", + " [8.268821, -2.070008, 1.722378, -1.8471404, -8...\n", " \n", " \n", " 6\n", @@ -1769,7 +1795,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [7.024207, -3.5837536, 1.645798, 0.86984664, -...\n", + " [5.59269, -3.1613479, 0.4734843, -0.7772096, -...\n", " \n", " \n", " 7\n", @@ -1793,7 +1819,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [3.0095522, -4.574903, 0.7152054, -5.211629, 4...\n", + " [1.9852623, -5.166985, 0.86473066, -6.491789, ...\n", " \n", " \n", " 8\n", @@ -1817,7 +1843,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-1.548282, -3.065774, 10.493741, -1.5243877, ...\n", + " [-2.800528, -4.2984514, 10.887824, -3.1346364,...\n", " \n", " \n", " 9\n", @@ -1841,7 +1867,7 @@ " 0.0\n", " 0.0\n", " 0.0\n", - " [-1.5818219, -3.821816, -3.4366767, 7.7891817,...\n", + " [-3.7827752, -4.51145, -5.354035, 9.399383, -6...\n", " \n", " \n", "\n", @@ -1862,16 +1888,16 @@ "9 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 \n", "\n", " 779 780 781 782 783 preds \n", - "0 0.0 0.0 0.0 0.0 0.0 [-6.554102, 2.085586, 1.3187729, 0.7378275, -4... \n", - "1 0.0 0.0 0.0 0.0 0.0 [-3.8565233, 4.518864, -1.2931982, -0.78954405... \n", - "2 0.0 0.0 0.0 0.0 0.0 [-3.7759259, -2.682865, -1.2959752, -5.4488983... \n", - "3 0.0 0.0 0.0 0.0 0.0 [0.15489274, -1.4305159, -1.5703316, 1.2637339... \n", - "4 0.0 0.0 0.0 0.0 0.0 [-3.9276226, 4.4247217, 1.0965542, 0.3403727, ... \n", - "5 0.0 0.0 0.0 0.0 0.0 [8.987603, -3.3934922, 2.4643059, -0.84833026,... \n", - "6 0.0 0.0 0.0 0.0 0.0 [7.024207, -3.5837536, 1.645798, 0.86984664, -... \n", - "7 0.0 0.0 0.0 0.0 0.0 [3.0095522, -4.574903, 0.7152054, -5.211629, 4... \n", - "8 0.0 0.0 0.0 0.0 0.0 [-1.548282, -3.065774, 10.493741, -1.5243877, ... \n", - "9 0.0 0.0 0.0 0.0 0.0 [-1.5818219, -3.821816, -3.4366767, 7.7891817,... \n", + "0 0.0 0.0 0.0 0.0 0.0 [-6.9618006, 1.2047814, -0.09570807, 0.0462105... \n", + "1 0.0 0.0 0.0 0.0 0.0 [-5.2882323, 5.902014, -2.0389183, -1.2460864,... \n", + "2 0.0 0.0 0.0 0.0 0.0 [-5.822013, -2.3333628, -2.4322102, -8.040086,... \n", + "3 0.0 0.0 0.0 0.0 0.0 [-0.57203317, -1.2920653, -2.7234774, 0.914070... \n", + "4 0.0 0.0 0.0 0.0 0.0 [-3.689301, 5.0702505, -0.23930073, -0.7988689... \n", + "5 0.0 0.0 0.0 0.0 0.0 [8.268821, -2.070008, 1.722378, -1.8471404, -8... \n", + "6 0.0 0.0 0.0 0.0 0.0 [5.59269, -3.1613479, 0.4734843, -0.7772096, -... \n", + "7 0.0 0.0 0.0 0.0 0.0 [1.9852623, -5.166985, 0.86473066, -6.491789, ... \n", + "8 0.0 0.0 0.0 0.0 0.0 [-2.800528, -4.2984514, 10.887824, -3.1346364,... \n", + "9 0.0 0.0 0.0 0.0 0.0 [-3.7827752, -4.51145, -5.354035, 9.399383, -6... \n", "\n", "[10 rows x 785 columns]" ] @@ -1906,8 +1932,8 @@ { "data": { "text/plain": [ - "array([-6.554102 , 2.085586 , 1.3187729, 0.7378275, -4.1074104,\n", - " -3.352563 , -4.2061253, 4.6558733, 1.0386009, 0.9432423],\n", + "array([-6.9618006 , 1.2047814 , -0.09570807, 0.04621054, -5.8169513 ,\n", + " -4.148872 , -5.17938 , 6.382909 , -0.11228667, 0.6022302 ],\n", " dtype=float32)" ] }, @@ -1935,7 +1961,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": null, "id": "297811e1-aecb-4afd-9a6a-30c49e8881cc", "metadata": {}, "outputs": [ @@ -1990,7 +2016,7 @@ "id": "d1e63867", "metadata": {}, "source": [ - "Import the utility functions from pytriton_utils.py:" + "Import the helper class from pytriton_utils.py:" ] }, { @@ -2002,12 +2028,7 @@ "source": [ "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "from pytriton_utils import (\n", - " use_stage_level_scheduling,\n", - " find_ports,\n", - " start_triton,\n", - " stop_triton\n", - ")" + "from pytriton_utils import TritonServerManager" ] }, { @@ -2077,6 +2098,7 @@ " )\n", "\n", " def _stop_triton(signum, frame):\n", + " # The server manager sends SIGTERM to stop the server; this function ensures graceful cleanup.\n", " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", " triton.stop()\n", "\n", @@ -2116,82 +2138,38 @@ }, { "cell_type": "markdown", - "id": "73d1e5cb", + "id": "2695d9ab", "metadata": {}, "source": [ - "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + "The `TritonClusterManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n", + "- Find available ports for HTTP/gRPC/metrics\n", + "- Deploy a server on each node via stage-level scheduling\n", + "- Gracefully shutdown servers across nodes" ] }, { "cell_type": "code", - "execution_count": 54, + "execution_count": null, "id": "4deae3b1", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - } - ], - "source": [ - "sc = spark.sparkContext\n", - "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" - ] - }, - { - "cell_type": "markdown", - "id": "7fc49b0b", - "metadata": {}, - "source": [ - "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "id": "3ffd2734", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using ports [7000, 7001, 7002]\n" - ] - } - ], + "outputs": [], "source": [ "model_name = \"ImageClassifier\"\n", - "ports = find_ports()\n", - "assert len(ports) == 3\n", - "print(f\"Using ports {ports}\")" + "server_manager = TritonServerManager(num_nodes=num_nodes, model_name=model_name, model_path=model_path)" ] }, { "cell_type": "code", - "execution_count": 56, - "id": "b6913b2c", + "execution_count": null, + "id": "e56c84f4", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 16:> (0 + 1) / 1]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Triton Server PIDs:\n", - " {\n", - " \"cb4ae00-lcedt\": 3034797\n", - "}\n" + "2025-02-07 11:03:44,809 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-07 11:03:44,810 - INFO - Starting 1 servers.\n" ] }, { @@ -2200,14 +2178,20 @@ "text": [ " \r" ] + }, + { + "data": { + "text/plain": [ + "{'cb4ae00-lcedt': (2020631, [7000, 7001, 7002])}" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", - " ports=ports,\n", - " model_name=model_name,\n", - " model_path=model_path)).collectAsMap()\n", - "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n", + "server_manager.start_servers(triton_server)" ] }, { @@ -2218,26 +2202,44 @@ "#### Define client function" ] }, + { + "cell_type": "markdown", + "id": "e278fde0", + "metadata": {}, + "source": [ + "Get the hostname -> url mapping from the server manager:" + ] + }, { "cell_type": "code", - "execution_count": 57, - "id": "53f58831", + "execution_count": null, + "id": "68a9606e", "metadata": {}, "outputs": [], "source": [ - "url = f\"http://localhost:{ports[0]}\"" + "host_to_http_url = server_manager.host_to_http_url # or server_manager.host_to_grpc_url" + ] + }, + { + "cell_type": "markdown", + "id": "4d70bd6f", + "metadata": {}, + "source": [ + "Define the Triton inference function, which returns a predict function for batch inference through the server:" ] }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 57, "id": "92ba2e26", "metadata": {}, "outputs": [], "source": [ - "def triton_fn(url, model_name):\n", + "def triton_fn(model_name, host_to_url):\n", + " import socket\n", " from pytriton.client import ModelClient\n", "\n", + " url = host_to_url[socket.gethostname()]\n", " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", "\n", " def infer_batch(inputs):\n", @@ -2248,6 +2250,19 @@ " return infer_batch" ] }, + { + "cell_type": "code", + "execution_count": 59, + "id": "6658d2a1-ef7b-4ca1-9fb6-f2ac9050f3e5", + "metadata": {}, + "outputs": [], + "source": [ + "predict = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_http_url),\n", + " input_tensor_shapes=[[784]],\n", + " return_type=ArrayType(FloatType()),\n", + " batch_size=128)" + ] + }, { "cell_type": "markdown", "id": "3842c263", @@ -2258,7 +2273,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 58, "id": "43b93753-1d52-4060-9986-f24c30a67528", "metadata": {}, "outputs": [ @@ -2268,7 +2283,7 @@ "StructType([StructField('data', ArrayType(DoubleType(), True), True)])" ] }, - "execution_count": 59, + "execution_count": 58, "metadata": {}, "output_type": "execute_result" } @@ -2281,19 +2296,6 @@ { "cell_type": "code", "execution_count": 60, - "id": "6658d2a1-ef7b-4ca1-9fb6-f2ac9050f3e5", - "metadata": {}, - "outputs": [], - "source": [ - "predict = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", - " input_tensor_shapes=[[784]],\n", - " return_type=ArrayType(FloatType()),\n", - " batch_size=128)" - ] - }, - { - "cell_type": "code", - "execution_count": 61, "id": "8397aa14-82fd-4351-a477-dc8e8b321fa2", "metadata": {}, "outputs": [ @@ -2301,15 +2303,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 18:> (0 + 8) / 8]\r" + "[Stage 19:> (0 + 8) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 21.3 ms, sys: 5.16 ms, total: 26.5 ms\n", - "Wall time: 1.7 s\n" + "CPU times: user 19.8 ms, sys: 2.89 ms, total: 22.7 ms\n", + "Wall time: 1.67 s\n" ] }, { @@ -2327,7 +2329,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 61, "id": "82698bd9-377a-4415-8971-835487f876cc", "metadata": {}, "outputs": [ @@ -2335,8 +2337,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 17.4 ms, sys: 7.83 ms, total: 25.3 ms\n", - "Wall time: 445 ms\n" + "CPU times: user 19.8 ms, sys: 5.99 ms, total: 25.7 ms\n", + "Wall time: 399 ms\n" ] } ], @@ -2347,7 +2349,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 62, "id": "419ad7bd-fa28-49d3-b98d-db9fba5aeaef", "metadata": {}, "outputs": [ @@ -2355,15 +2357,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 20:==============> (2 + 6) / 8]\r" + "[Stage 21:====================================> (5 + 3) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 8.15 ms, sys: 2.67 ms, total: 10.8 ms\n", - "Wall time: 892 ms\n" + "CPU times: user 9.07 ms, sys: 1.34 ms, total: 10.4 ms\n", + "Wall time: 888 ms\n" ] }, { @@ -2402,52 +2404,52 @@ " \n", " 0\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-2.9825501, -2.4956138, 1.2573452, 12.891572,...\n", + " [-4.6654444, -2.4893682, -0.5888205, 13.380681...\n", " \n", " \n", " 1\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-0.9547625, -5.622221, 2.2415693, -2.0017662,...\n", + " [-2.2732146, -7.5127845, 1.1983705, -3.540661,...\n", " \n", " \n", " 2\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-0.9178927, 0.4341122, 1.9357599, 1.7975069, ...\n", + " [-2.2890894, 0.8308606, 0.31311002, 1.1683631,...\n", " \n", " \n", " 3\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [0.2115398, -5.582211, 11.52417, 2.8064039, -3...\n", + " [-1.055197, -6.502811, 12.420727, 0.4528031, -...\n", " \n", " \n", " 4\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-3.395469, 3.7420897, -0.25319868, 0.16790543...\n", + " [-3.7887795, 3.9983597, -1.5343359, -0.3698441...\n", " \n", " \n", " 5\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-3.1469467, -2.4765825, 3.0782876, 4.3063807,...\n", + " [-4.4992743, -1.7618219, 1.1183226, 3.9469318,...\n", " \n", " \n", " 6\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-2.9109812, 4.020501, 1.359716, -0.003118489,...\n", + " [-2.754053, 4.868414, 0.2515293, -0.47300792, ...\n", " \n", " \n", " 7\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-0.3475021, -0.7891858, -4.275926, 0.51047605...\n", + " [-1.888711, 0.02717158, -6.050885, 0.08750934,...\n", " \n", " \n", " 8\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [2.0500505, -2.2071157, -0.07787818, -3.207583...\n", + " [0.9541264, -2.113048, -1.7508973, -5.4303784,...\n", " \n", " \n", " 9\n", " [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...\n", - " [-0.0029557291, -1.7499021, -2.1225464, 2.1881...\n", + " [-1.612412, -0.7655782, -4.473859, 2.0609212, ...\n", " \n", " \n", "\n", @@ -2467,19 +2469,19 @@ "9 [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ... \n", "\n", " preds \n", - "0 [-2.9825501, -2.4956138, 1.2573452, 12.891572,... \n", - "1 [-0.9547625, -5.622221, 2.2415693, -2.0017662,... \n", - "2 [-0.9178927, 0.4341122, 1.9357599, 1.7975069, ... \n", - "3 [0.2115398, -5.582211, 11.52417, 2.8064039, -3... \n", - "4 [-3.395469, 3.7420897, -0.25319868, 0.16790543... \n", - "5 [-3.1469467, -2.4765825, 3.0782876, 4.3063807,... \n", - "6 [-2.9109812, 4.020501, 1.359716, -0.003118489,... \n", - "7 [-0.3475021, -0.7891858, -4.275926, 0.51047605... \n", - "8 [2.0500505, -2.2071157, -0.07787818, -3.207583... \n", - "9 [-0.0029557291, -1.7499021, -2.1225464, 2.1881... " + "0 [-4.6654444, -2.4893682, -0.5888205, 13.380681... \n", + "1 [-2.2732146, -7.5127845, 1.1983705, -3.540661,... \n", + "2 [-2.2890894, 0.8308606, 0.31311002, 1.1683631,... \n", + "3 [-1.055197, -6.502811, 12.420727, 0.4528031, -... \n", + "4 [-3.7887795, 3.9983597, -1.5343359, -0.3698441... \n", + "5 [-4.4992743, -1.7618219, 1.1183226, 3.9469318,... \n", + "6 [-2.754053, 4.868414, 0.2515293, -0.47300792, ... \n", + "7 [-1.888711, 0.02717158, -6.050885, 0.08750934,... \n", + "8 [0.9541264, -2.113048, -1.7508973, -5.4303784,... \n", + "9 [-1.612412, -0.7655782, -4.473859, 2.0609212, ... " ] }, - "execution_count": 63, + "execution_count": 62, "metadata": {}, "output_type": "execute_result" } @@ -2492,7 +2494,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 63, "id": "79d90a26", "metadata": {}, "outputs": [], @@ -2503,7 +2505,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 64, "id": "4ca495f5", "metadata": {}, "outputs": [], @@ -2517,7 +2519,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 65, "id": "a5d10903", "metadata": {}, "outputs": [ @@ -2551,22 +2553,16 @@ }, { "cell_type": "code", - "execution_count": 67, - "id": "9c9fd967-5cd9-4265-add9-db5c1ccf9893", + "execution_count": null, + "id": "d06de00e", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "2025-02-04 14:00:18,330 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-04 14:00:28,520 - INFO - Sucessfully stopped 1 servers. \n" ] }, { @@ -2575,20 +2571,17 @@ "[True]" ] }, - "execution_count": 67, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ - "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", - "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" + "server_manager.stop_servers()" ] }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 67, "id": "f612dc0b-538f-4ecf-81f7-ef6b58c493ab", "metadata": {}, "outputs": [], diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_preprocessing_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_preprocessing_tf.ipynb index f2d54e9f..5ded9d07 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_preprocessing_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_preprocessing_tf.ipynb @@ -33,13 +33,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-01-27 12:15:59.479063: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2025-01-27 12:15:59.486695: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2025-01-27 12:15:59.495438: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2025-01-27 12:15:59.498089: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2025-01-27 12:15:59.505021: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "2025-02-04 13:59:29.670948: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2025-02-04 13:59:29.679838: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2025-02-04 13:59:29.689914: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2025-02-04 13:59:29.692851: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2025-02-04 13:59:29.700499: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2025-01-27 12:15:59.913410: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" + "2025-02-04 13:59:30.139239: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" ] } ], @@ -65,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "9fa3e1b7-58cd-45f9-9fee-85f25a31c3c6", "metadata": {}, "outputs": [ @@ -81,9 +81,9 @@ "output_type": "stream", "text": [ "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", - "I0000 00:00:1738008960.285379 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008960.309064 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008960.311808 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" + "I0000 00:00:1738706370.524690 3686377 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706370.550329 3686377 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706370.553239 3686377 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" ] } ], @@ -337,7 +337,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "00d403cf-9ae7-4780-9fac-13d920d8b395", "metadata": {}, "outputs": [ @@ -413,7 +413,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "b9ec57c9-080e-4626-9e03-acf309cf3736", "metadata": {}, "outputs": [ @@ -421,13 +421,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "I0000 00:00:1738008960.772797 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008960.775983 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008960.778707 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008960.893233 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008960.894355 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738008960.895274 3039481 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "2025-01-27 12:16:00.896282: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 40284 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" + "I0000 00:00:1738706370.981571 3686377 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706370.984478 3686377 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706370.987280 3686377 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706371.105121 3686377 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706371.106231 3686377 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706371.107182 3686377 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2025-02-04 13:59:31.108098: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 40337 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" ] } ], @@ -446,7 +446,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "dfcbf268-4508-4eb8-abe1-acf1dbb97bd5", "metadata": {}, "outputs": [ @@ -456,19 +456,19 @@ "text": [ "Every feature: ['Type', 'Age', 'Breed1', 'Gender', 'Color1', 'Color2', 'MaturitySize', 'FurLength', 'Vaccinated', 'Sterilized', 'Health', 'Fee', 'PhotoAmt', 'target']\n", "A batch of ages: tf.Tensor(\n", - "[[12]\n", - " [ 2]\n", - " [12]\n", + "[[ 4]\n", " [60]\n", - " [11]], shape=(5, 1), dtype=int64)\n", - "A batch of targets: tf.Tensor([1 0 1 1 1], shape=(5,), dtype=int64)\n" + " [24]\n", + " [ 1]\n", + " [ 2]], shape=(5, 1), dtype=int64)\n", + "A batch of targets: tf.Tensor([1 1 1 1 1], shape=(5,), dtype=int64)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2025-01-27 12:16:00.969048: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + "2025-02-04 13:59:31.170523: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" ] } ], @@ -519,18 +519,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-01-27 12:16:02.497311: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + "2025-02-04 13:59:32.726183: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" ] }, { "data": { "text/plain": [ "" + "array([[-0.19333968],\n", + " [-0.19333968],\n", + " [-0.19333968],\n", + " [-0.51676387],\n", + " [ 1.100357 ]], dtype=float32)>" ] }, "execution_count": 13, @@ -583,10 +583,10 @@ "data": { "text/plain": [ "" ] }, @@ -613,18 +613,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-01-27 12:16:04.075100: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + "2025-02-04 13:59:34.294276: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" ] }, { "data": { "text/plain": [ "" + " [0., 0., 0., 1., 0.],\n", + " [0., 1., 0., 0., 0.]], dtype=float32)>" ] }, "execution_count": 16, @@ -648,7 +648,7 @@ "source": [ "### Preprocess selected features\n", "\n", - "Apply the preprocessing utility functions defined earlier. Add all the feature inputs to a list.\n" + "Apply the preprocessing helper class defined earlier. Add all the feature inputs to a list.\n" ] }, { @@ -711,8 +711,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-01-27 12:16:04.387422: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n", - "2025-01-27 12:16:04.854147: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + "2025-02-04 13:59:34.588989: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n", + "2025-02-04 13:59:35.029267: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" ] } ], @@ -778,31 +778,31 @@ "output_type": "stream", "text": [ "Epoch 1/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.5245 - loss: 0.6416 - val_accuracy: 0.7314 - val_loss: 0.5624\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 16ms/step - accuracy: 0.3658 - loss: 0.7746 - val_accuracy: 0.6854 - val_loss: 0.5841\n", "Epoch 2/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.6565 - loss: 0.5904 - val_accuracy: 0.7392 - val_loss: 0.5425\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 16ms/step - accuracy: 0.6270 - loss: 0.6023 - val_accuracy: 0.7383 - val_loss: 0.5593\n", "Epoch 3/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.6856 - loss: 0.5653 - val_accuracy: 0.7400 - val_loss: 0.5317\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 16ms/step - accuracy: 0.6650 - loss: 0.5781 - val_accuracy: 0.7392 - val_loss: 0.5442\n", "Epoch 4/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.6971 - loss: 0.5572 - val_accuracy: 0.7496 - val_loss: 0.5231\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 17ms/step - accuracy: 0.6609 - loss: 0.5744 - val_accuracy: 0.7418 - val_loss: 0.5329\n", "Epoch 5/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.7148 - loss: 0.5377 - val_accuracy: 0.7400 - val_loss: 0.5203\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.6845 - loss: 0.5555 - val_accuracy: 0.7444 - val_loss: 0.5261\n", "Epoch 6/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.7088 - loss: 0.5373 - val_accuracy: 0.7392 - val_loss: 0.5177\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.6910 - loss: 0.5465 - val_accuracy: 0.7513 - val_loss: 0.5198\n", "Epoch 7/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.7082 - loss: 0.5378 - val_accuracy: 0.7426 - val_loss: 0.5157\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.7018 - loss: 0.5475 - val_accuracy: 0.7556 - val_loss: 0.5145\n", "Epoch 8/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 14ms/step - accuracy: 0.7213 - loss: 0.5289 - val_accuracy: 0.7426 - val_loss: 0.5146\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.7026 - loss: 0.5410 - val_accuracy: 0.7496 - val_loss: 0.5099\n", "Epoch 9/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.7393 - loss: 0.5138 - val_accuracy: 0.7366 - val_loss: 0.5147\n", + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.7145 - loss: 0.5315 - val_accuracy: 0.7530 - val_loss: 0.5066\n", "Epoch 10/10\n", - "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 16ms/step - accuracy: 0.7253 - loss: 0.5173 - val_accuracy: 0.7366 - val_loss: 0.5134\n" + "\u001b[1m37/37\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 15ms/step - accuracy: 0.7099 - loss: 0.5316 - val_accuracy: 0.7539 - val_loss: 0.5038\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 23, @@ -816,7 +816,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "fbccebaa-fc24-4a58-a032-222cef8fdf08", "metadata": {}, "outputs": [ @@ -824,15 +824,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m1/5\u001b[0m \u001b[32m━━━━\u001b[0m\u001b[37m━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 12ms/step - accuracy: 0.7539 - loss: 0.4781" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[1m5/5\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 7ms/step - accuracy: 0.7403 - loss: 0.5034 \n", - "Accuracy 0.737434983253479\n" + "\u001b[1m5/5\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 7ms/step - accuracy: 0.7416 - loss: 0.5196 \n", + "Accuracy 0.7443674206733704\n" ] } ], @@ -873,7 +866,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "f3d2a2d5-fd4d-4320-bacc-fd4571cec709", "metadata": {}, "outputs": [ @@ -882,7 +875,7 @@ "output_type": "stream", "text": [ "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 27ms/step\n", - "This particular pet had a 80.3 percent probability of getting adopted.\n" + "This particular pet had a 83.2 percent probability of getting adopted.\n" ] } ], @@ -978,12 +971,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 20:16:12 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "25/01/27 20:16:12 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "25/02/04 13:59:42 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/02/04 13:59:42 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "25/01/27 20:16:12 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", - "25/01/27 20:16:13 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + "25/02/04 13:59:43 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" ] } ], @@ -1258,16 +1250,22 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 20:16:16 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n", - "[Stage 5:=============================> (4 + 4) / 8]\r" + "25/02/04 13:59:47 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 5:> (0 + 8) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 25.8 ms, sys: 5.79 ms, total: 31.6 ms\n", - "Wall time: 5.04 s\n" + "CPU times: user 19.8 ms, sys: 9.3 ms, total: 29.1 ms\n", + "Wall time: 4.99 s\n" ] }, { @@ -1294,15 +1292,22 @@ "name": "stderr", "output_type": "stream", "text": [ - " \r" + "[Stage 6:> (0 + 8) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 87.7 ms, sys: 11.4 ms, total: 99.1 ms\n", - "Wall time: 1.59 s\n" + "CPU times: user 86.9 ms, sys: 13.7 ms, total: 101 ms\n", + "Wall time: 1.56 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" ] } ], @@ -1329,8 +1334,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 14.9 ms, sys: 9.97 ms, total: 24.9 ms\n", - "Wall time: 1.53 s\n" + "CPU times: user 16.4 ms, sys: 4.46 ms, total: 20.9 ms\n", + "Wall time: 1.52 s\n" ] }, { @@ -1362,26 +1367,26 @@ "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+------------+---+--------+------+-----------+\n", "|Type|Age| Breed1|Gender|Color1| Color2|MaturitySize|FurLength|Vaccinated|Sterilized| Health|Fee|PhotoAmt|target| preds|\n", "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+------------+---+--------+------+-----------+\n", - "| Dog| 3| Mixed Breed| Male| Black|No Color| Small| Medium| Not Sure| Not Sure| Healthy| 0| 2| 0| 0.1276686|\n", - "| Dog| 9| Mixed Breed| Male| Gray|No Color| Medium| Short| Not Sure| No| Healthy| 0| 4| 1| 0.48716992|\n", - "| Cat| 4| Domestic Short Hair| Male| Black| Gray| Medium| Short| Not Sure| Not Sure| Healthy| 0| 4| 1| 0.80358183|\n", - "| Cat| 6| Domestic Short Hair| Male|Yellow| White| Medium| Short| No| No| Healthy| 0| 3| 1| 0.8270739|\n", - "| Cat| 6|Domestic Medium Hair| Male| Gray|No Color| Small| Medium| No| No| Healthy| 0| 4| 1| 0.89303124|\n", - "| Cat| 5|Domestic Medium Hair|Female| Gray|No Color| Medium| Medium| Yes| Not Sure| Healthy| 0| 1| 0| 0.21722752|\n", - "| Dog| 24| Beagle|Female| Black| Golden| Medium| Short| Not Sure| Not Sure|Minor Injury| 0| 1| 1| 0.09258729|\n", - "| Cat| 29| Tabby| Male| Brown| Golden| Medium| Short| No| No| Healthy| 0| 1| 0| 0.74957174|\n", - "| Dog| 9| Mixed Breed|Female| Black| Brown| Medium| Short| Yes| Yes| Healthy| 0| 2| 0|-0.28708756|\n", - "| Dog| 2| Mixed Breed|Female| Cream| White| Medium| Short| No| No| Healthy| 0| 1| 0| 1.632988|\n", - "| Dog| 2| Mixed Breed| Male| Brown| White| Medium| Short| Yes| No| Healthy| 0| 1| 1| 1.512331|\n", - "| Dog| 60| Golden Retriever| Male| Brown| Yellow| Medium| Medium| Yes| Yes| Healthy| 0| 5| 1| 1.2067251|\n", - "| Cat| 9| Siamese| Male| White|No Color| Medium| Short| Yes| No| Healthy| 0| 2| 1| 1.07974|\n", - "| Dog| 19| Doberman Pinscher|Female| Black| Brown| Large| Short| Yes| Yes| Healthy|500| 2| 1| 0.48468828|\n", - "| Cat| 11| Domestic Short Hair| Male| Cream|No Color| Medium| Short| Yes| Yes| Healthy|100| 6| 0| 0.4290621|\n", - "| Dog| 18| Mixed Breed|Female| Brown| White| Small| Short| Yes| No| Healthy| 0| 5| 0| 0.26826438|\n", - "| Dog| 4| Mixed Breed|Female| Brown| White| Medium| Medium| Not Sure| Not Sure| Healthy| 0| 3| 0| 0.2859868|\n", - "| Dog| 96| Golden Retriever| Male|Golden|No Color| Large| Long| Yes| Yes| Healthy| 0| 2| 1| 1.5675049|\n", - "| Dog| 54| Golden Retriever| Male|Golden|No Color| Large| Medium| Yes| No| Healthy|350| 20| 1| 2.913511|\n", - "| Cat| 5|Domestic Medium Hair|Female| Brown| White| Medium| Medium| No| No| Healthy| 0| 5| 1| 1.0410445|\n", + "| Dog| 3| Mixed Breed| Male| Black|No Color| Small| Medium| Not Sure| Not Sure| Healthy| 0| 2| 0| 0.4963937|\n", + "| Dog| 9| Mixed Breed| Male| Gray|No Color| Medium| Short| Not Sure| No| Healthy| 0| 4| 1| 0.6780287|\n", + "| Cat| 4| Domestic Short Hair| Male| Black| Gray| Medium| Short| Not Sure| Not Sure| Healthy| 0| 4| 1| 0.58800673|\n", + "| Cat| 6| Domestic Short Hair| Male|Yellow| White| Medium| Short| No| No| Healthy| 0| 3| 1| 0.7378843|\n", + "| Cat| 6|Domestic Medium Hair| Male| Gray|No Color| Small| Medium| No| No| Healthy| 0| 4| 1| 1.2695599|\n", + "| Cat| 5|Domestic Medium Hair|Female| Gray|No Color| Medium| Medium| Yes| Not Sure| Healthy| 0| 1| 0|0.060457088|\n", + "| Dog| 24| Beagle|Female| Black| Golden| Medium| Short| Not Sure| Not Sure|Minor Injury| 0| 1| 1| 0.28160828|\n", + "| Cat| 29| Tabby| Male| Brown| Golden| Medium| Short| No| No| Healthy| 0| 1| 0| 0.6928505|\n", + "| Dog| 9| Mixed Breed|Female| Black| Brown| Medium| Short| Yes| Yes| Healthy| 0| 2| 0|-0.10125986|\n", + "| Dog| 2| Mixed Breed|Female| Cream| White| Medium| Short| No| No| Healthy| 0| 1| 0| 1.3703903|\n", + "| Dog| 2| Mixed Breed| Male| Brown| White| Medium| Short| Yes| No| Healthy| 0| 1| 1| 1.3243997|\n", + "| Dog| 60| Golden Retriever| Male| Brown| Yellow| Medium| Medium| Yes| Yes| Healthy| 0| 5| 1| 0.9026731|\n", + "| Cat| 9| Siamese| Male| White|No Color| Medium| Short| Yes| No| Healthy| 0| 2| 1| 0.8207382|\n", + "| Dog| 19| Doberman Pinscher|Female| Black| Brown| Large| Short| Yes| Yes| Healthy|500| 2| 1| 0.85343015|\n", + "| Cat| 11| Domestic Short Hair| Male| Cream|No Color| Medium| Short| Yes| Yes| Healthy|100| 6| 0| 0.53920615|\n", + "| Dog| 18| Mixed Breed|Female| Brown| White| Small| Short| Yes| No| Healthy| 0| 5| 0| 0.718272|\n", + "| Dog| 4| Mixed Breed|Female| Brown| White| Medium| Medium| Not Sure| Not Sure| Healthy| 0| 3| 0| 0.16185221|\n", + "| Dog| 96| Golden Retriever| Male|Golden|No Color| Large| Long| Yes| Yes| Healthy| 0| 2| 1| 0.8156965|\n", + "| Dog| 54| Golden Retriever| Male|Golden|No Color| Large| Medium| Yes| No| Healthy|350| 20| 1| 3.5315154|\n", + "| Cat| 5|Domestic Medium Hair|Female| Brown| White| Medium| Medium| No| No| Healthy| 0| 5| 1| 1.1725564|\n", "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+------------+---+--------+------+-----------+\n", "only showing top 20 rows\n", "\n" @@ -1425,7 +1430,7 @@ "id": "ea407357", "metadata": {}, "source": [ - "Import the utility functions from pytriton_utils.py:" + "Import the helper class from pytriton_utils.py:" ] }, { @@ -1437,12 +1442,7 @@ "source": [ "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "from pytriton_utils import (\n", - " use_stage_level_scheduling,\n", - " find_ports,\n", - " start_triton,\n", - " stop_triton\n", - ")" + "from pytriton_utils import TritonServerManager" ] }, { @@ -1544,6 +1544,7 @@ " )\n", "\n", " def _stop_triton(signum, frame):\n", + " # The server manager sends SIGTERM to stop the server; this function ensures graceful cleanup.\n", " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", " triton.stop()\n", "\n", @@ -1583,82 +1584,38 @@ }, { "cell_type": "markdown", - "id": "1d96f480", + "id": "fc93a43a", "metadata": {}, "source": [ - "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + "The `TritonClusterManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n", + "- Find available ports for HTTP/gRPC/metrics\n", + "- Deploy a server on each node via stage-level scheduling\n", + "- Gracefully shutdown servers across nodes" ] }, { "cell_type": "code", - "execution_count": 48, + "execution_count": null, "id": "c9b98208", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - } - ], - "source": [ - "sc = spark.sparkContext\n", - "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" - ] - }, - { - "cell_type": "markdown", - "id": "ee5a2d8b", - "metadata": {}, - "source": [ - "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "id": "918f14b8", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using ports [7000, 7001, 7002]\n" - ] - } - ], + "outputs": [], "source": [ "model_name = \"PetClassifier\"\n", - "ports = find_ports()\n", - "assert len(ports) == 3\n", - "print(f\"Using ports {ports}\")" + "server_manager = TritonServerManager(num_nodes=num_nodes, model_name=model_name, model_path=model_path)" ] }, { "cell_type": "code", - "execution_count": 50, - "id": "dc4ff00f", + "execution_count": null, + "id": "228401f7", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 9:> (0 + 1) / 1]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Triton Server PIDs:\n", - " {\n", - " \"cb4ae00-lcedt\": 3056153\n", - "}\n" + "2025-02-07 11:03:44,809 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-07 11:03:44,810 - INFO - Starting 1 servers.\n" ] }, { @@ -1667,14 +1624,20 @@ "text": [ " \r" ] + }, + { + "data": { + "text/plain": [ + "{'cb4ae00-lcedt': (2020631, [7000, 7001, 7002])}" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", - " ports=ports,\n", - " model_name=model_name,\n", - " model_path=model_path)).collectAsMap()\n", - "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n", + "server_manager.start_servers(triton_server)" ] }, { @@ -1685,27 +1648,45 @@ "#### Define client function" ] }, + { + "cell_type": "markdown", + "id": "5d28b1ca", + "metadata": {}, + "source": [ + "Get the hostname -> url mapping from the server manager:" + ] + }, { "cell_type": "code", - "execution_count": 51, - "id": "3eec95bb", + "execution_count": null, + "id": "d1234a02", "metadata": {}, "outputs": [], "source": [ - "url = f\"http://localhost:{ports[0]}\"" + "host_to_http_url = server_manager.host_to_http_url # or server_manager.host_to_grpc_url" + ] + }, + { + "cell_type": "markdown", + "id": "3c9ef706", + "metadata": {}, + "source": [ + "Define the Triton inference function, which returns a predict function for batch inference through the server:" ] }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 51, "id": "e50b5fc8", "metadata": {}, "outputs": [], "source": [ - "def triton_fn(url, model_name):\n", + "def triton_fn(model_name, host_to_url):\n", + " import socket\n", " import numpy as np\n", " from pytriton.client import ModelClient\n", "\n", + " url = host_to_url[socket.gethostname()]\n", " print(f\"CLIENT: Connecting to {model_name} at {url}\")\n", "\n", " def infer_batch(t, a, b, g, c1, c2, m, f, v, s, h, fee, p):\n", @@ -1735,6 +1716,20 @@ " return infer_batch" ] }, + { + "cell_type": "code", + "execution_count": 54, + "id": "2ffb020e-dc93-456b-bee6-405611eee1e1", + "metadata": {}, + "outputs": [], + "source": [ + "# need to pass the list of columns into the model_udf\n", + "classify = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_http_url),\n", + " input_tensor_shapes=[[1]] * len(columns),\n", + " return_type=FloatType(),\n", + " batch_size=64)" + ] + }, { "cell_type": "markdown", "id": "2edd887f", @@ -1745,7 +1740,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 52, "id": "fe8dc3e6-f1b1-4a24-85f4-0a5ecabef4c5", "metadata": {}, "outputs": [], @@ -1755,7 +1750,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 53, "id": "4cfb3f34-a215-4781-91bf-2bec85e15633", "metadata": {}, "outputs": [], @@ -1776,20 +1771,6 @@ { "cell_type": "code", "execution_count": 55, - "id": "2ffb020e-dc93-456b-bee6-405611eee1e1", - "metadata": {}, - "outputs": [], - "source": [ - "# need to pass the list of columns into the model_udf\n", - "classify = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", - " input_tensor_shapes=[[1]] * len(columns),\n", - " return_type=FloatType(),\n", - " batch_size=64)" - ] - }, - { - "cell_type": "code", - "execution_count": 56, "id": "e6ff0356-becd-421f-aebb-272497d5ad6a", "metadata": {}, "outputs": [ @@ -1797,15 +1778,22 @@ "name": "stderr", "output_type": "stream", "text": [ - " \r" + "[Stage 12:> (0 + 8) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 23 ms, sys: 8.87 ms, total: 31.8 ms\n", - "Wall time: 6.34 s\n" + "CPU times: user 17.3 ms, sys: 7.75 ms, total: 25.1 ms\n", + "Wall time: 6.35 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" ] } ], @@ -1817,7 +1805,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 56, "id": "ce18ee7c-5958-4986-b200-6d986fcc6243", "metadata": {}, "outputs": [ @@ -1825,22 +1813,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 12:==================================================> (7 + 1) / 8]\r" + " \r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 16.6 ms, sys: 3.1 ms, total: 19.7 ms\n", - "Wall time: 5.83 s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" + "CPU times: user 15.2 ms, sys: 4.2 ms, total: 19.4 ms\n", + "Wall time: 5.86 s\n" ] } ], @@ -1852,7 +1833,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 57, "id": "0888ce40-b2c4-4aed-8ccb-6a8bcd00abc8", "metadata": {}, "outputs": [ @@ -1860,15 +1841,22 @@ "name": "stderr", "output_type": "stream", "text": [ - " \r" + "[Stage 14:===========================================> (6 + 2) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 85.3 ms, sys: 928 μs, total: 86.2 ms\n", - "Wall time: 5.8 s\n" + "CPU times: user 93.4 ms, sys: 3.4 ms, total: 96.8 ms\n", + "Wall time: 5.87 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" ] } ], @@ -1880,7 +1868,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 58, "id": "d45812b5-f584-41a4-a821-2b59e065671c", "metadata": {}, "outputs": [ @@ -1891,26 +1879,26 @@ "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+------------+---+--------+------+-----------+\n", "|Type|Age| Breed1|Gender|Color1| Color2|MaturitySize|FurLength|Vaccinated|Sterilized| Health|Fee|PhotoAmt|target| preds|\n", "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+------------+---+--------+------+-----------+\n", - "| Dog| 3| Mixed Breed| Male| Black|No Color| Small| Medium| Not Sure| Not Sure| Healthy| 0| 2| 0| 0.1276686|\n", - "| Dog| 9| Mixed Breed| Male| Gray|No Color| Medium| Short| Not Sure| No| Healthy| 0| 4| 1| 0.48716992|\n", - "| Cat| 4| Domestic Short Hair| Male| Black| Gray| Medium| Short| Not Sure| Not Sure| Healthy| 0| 4| 1| 0.80358183|\n", - "| Cat| 6| Domestic Short Hair| Male|Yellow| White| Medium| Short| No| No| Healthy| 0| 3| 1| 0.8270739|\n", - "| Cat| 6|Domestic Medium Hair| Male| Gray|No Color| Small| Medium| No| No| Healthy| 0| 4| 1| 0.89303124|\n", - "| Cat| 5|Domestic Medium Hair|Female| Gray|No Color| Medium| Medium| Yes| Not Sure| Healthy| 0| 1| 0| 0.21722752|\n", - "| Dog| 24| Beagle|Female| Black| Golden| Medium| Short| Not Sure| Not Sure|Minor Injury| 0| 1| 1| 0.09258729|\n", - "| Cat| 29| Tabby| Male| Brown| Golden| Medium| Short| No| No| Healthy| 0| 1| 0| 0.74957174|\n", - "| Dog| 9| Mixed Breed|Female| Black| Brown| Medium| Short| Yes| Yes| Healthy| 0| 2| 0|-0.28708756|\n", - "| Dog| 2| Mixed Breed|Female| Cream| White| Medium| Short| No| No| Healthy| 0| 1| 0| 1.632988|\n", - "| Dog| 2| Mixed Breed| Male| Brown| White| Medium| Short| Yes| No| Healthy| 0| 1| 1| 1.512331|\n", - "| Dog| 60| Golden Retriever| Male| Brown| Yellow| Medium| Medium| Yes| Yes| Healthy| 0| 5| 1| 1.2067251|\n", - "| Cat| 9| Siamese| Male| White|No Color| Medium| Short| Yes| No| Healthy| 0| 2| 1| 1.07974|\n", - "| Dog| 19| Doberman Pinscher|Female| Black| Brown| Large| Short| Yes| Yes| Healthy|500| 2| 1| 0.48468828|\n", - "| Cat| 11| Domestic Short Hair| Male| Cream|No Color| Medium| Short| Yes| Yes| Healthy|100| 6| 0| 0.4290621|\n", - "| Dog| 18| Mixed Breed|Female| Brown| White| Small| Short| Yes| No| Healthy| 0| 5| 0| 0.26826438|\n", - "| Dog| 4| Mixed Breed|Female| Brown| White| Medium| Medium| Not Sure| Not Sure| Healthy| 0| 3| 0| 0.2859868|\n", - "| Dog| 96| Golden Retriever| Male|Golden|No Color| Large| Long| Yes| Yes| Healthy| 0| 2| 1| 1.5675049|\n", - "| Dog| 54| Golden Retriever| Male|Golden|No Color| Large| Medium| Yes| No| Healthy|350| 20| 1| 2.913511|\n", - "| Cat| 5|Domestic Medium Hair|Female| Brown| White| Medium| Medium| No| No| Healthy| 0| 5| 1| 1.0410445|\n", + "| Dog| 3| Mixed Breed| Male| Black|No Color| Small| Medium| Not Sure| Not Sure| Healthy| 0| 2| 0| 0.4963937|\n", + "| Dog| 9| Mixed Breed| Male| Gray|No Color| Medium| Short| Not Sure| No| Healthy| 0| 4| 1| 0.6780287|\n", + "| Cat| 4| Domestic Short Hair| Male| Black| Gray| Medium| Short| Not Sure| Not Sure| Healthy| 0| 4| 1| 0.58800673|\n", + "| Cat| 6| Domestic Short Hair| Male|Yellow| White| Medium| Short| No| No| Healthy| 0| 3| 1| 0.7378843|\n", + "| Cat| 6|Domestic Medium Hair| Male| Gray|No Color| Small| Medium| No| No| Healthy| 0| 4| 1| 1.2695599|\n", + "| Cat| 5|Domestic Medium Hair|Female| Gray|No Color| Medium| Medium| Yes| Not Sure| Healthy| 0| 1| 0|0.060457088|\n", + "| Dog| 24| Beagle|Female| Black| Golden| Medium| Short| Not Sure| Not Sure|Minor Injury| 0| 1| 1| 0.28160828|\n", + "| Cat| 29| Tabby| Male| Brown| Golden| Medium| Short| No| No| Healthy| 0| 1| 0| 0.6928505|\n", + "| Dog| 9| Mixed Breed|Female| Black| Brown| Medium| Short| Yes| Yes| Healthy| 0| 2| 0|-0.10125986|\n", + "| Dog| 2| Mixed Breed|Female| Cream| White| Medium| Short| No| No| Healthy| 0| 1| 0| 1.3703903|\n", + "| Dog| 2| Mixed Breed| Male| Brown| White| Medium| Short| Yes| No| Healthy| 0| 1| 1| 1.3243997|\n", + "| Dog| 60| Golden Retriever| Male| Brown| Yellow| Medium| Medium| Yes| Yes| Healthy| 0| 5| 1| 0.9026731|\n", + "| Cat| 9| Siamese| Male| White|No Color| Medium| Short| Yes| No| Healthy| 0| 2| 1| 0.8207382|\n", + "| Dog| 19| Doberman Pinscher|Female| Black| Brown| Large| Short| Yes| Yes| Healthy|500| 2| 1| 0.85343015|\n", + "| Cat| 11| Domestic Short Hair| Male| Cream|No Color| Medium| Short| Yes| Yes| Healthy|100| 6| 0| 0.53920615|\n", + "| Dog| 18| Mixed Breed|Female| Brown| White| Small| Short| Yes| No| Healthy| 0| 5| 0| 0.718272|\n", + "| Dog| 4| Mixed Breed|Female| Brown| White| Medium| Medium| Not Sure| Not Sure| Healthy| 0| 3| 0| 0.16185221|\n", + "| Dog| 96| Golden Retriever| Male|Golden|No Color| Large| Long| Yes| Yes| Healthy| 0| 2| 1| 0.8156965|\n", + "| Dog| 54| Golden Retriever| Male|Golden|No Color| Large| Medium| Yes| No| Healthy|350| 20| 1| 3.5315154|\n", + "| Cat| 5|Domestic Medium Hair|Female| Brown| White| Medium| Medium| No| No| Healthy| 0| 5| 1| 1.1725564|\n", "+----+---+--------------------+------+------+--------+------------+---------+----------+----------+------------+---+--------+------+-----------+\n", "only showing top 20 rows\n", "\n" @@ -1933,22 +1921,16 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 59, "id": "6914f44f-677f-4db3-be09-783df8d11b8a", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "2025-02-04 14:00:18,330 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-04 14:00:28,520 - INFO - Sucessfully stopped 1 servers. \n" ] }, { @@ -1957,20 +1939,18 @@ "[True]" ] }, - "execution_count": 60, + "execution_count": 59, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", - "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" + "server_manager.stop_servers()" ] }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 60, "id": "f8c6ee43-8891-4446-986e-1447c5d48bac", "metadata": {}, "outputs": [], diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_resnet50_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_resnet50_tf.ipynb index c90a835b..97eb600c 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_resnet50_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/keras_resnet50_tf.ipynb @@ -33,13 +33,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-01-27 12:17:01.457688: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2025-01-27 12:17:01.464818: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2025-01-27 12:17:01.472654: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2025-01-27 12:17:01.474956: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2025-01-27 12:17:01.481116: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "2025-02-04 14:00:35.457924: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2025-02-04 14:00:35.465639: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2025-02-04 14:00:35.473515: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2025-02-04 14:00:35.475792: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2025-02-04 14:00:35.482106: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2025-01-27 12:17:01.844814: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" + "2025-02-04 14:00:35.843263: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" ] } ], @@ -70,7 +70,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "75175140", "metadata": {}, "outputs": [ @@ -86,9 +86,9 @@ "output_type": "stream", "text": [ "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", - "I0000 00:00:1738009022.171730 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738009022.193480 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738009022.196408 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" + "I0000 00:00:1738706436.174805 3714100 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706436.197467 3714100 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706436.200398 3714100 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" ] } ], @@ -169,12 +169,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/27 20:17:02 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "25/01/27 20:17:02 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "25/02/04 14:00:36 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/02/04 14:00:36 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "25/01/27 20:17:03 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", - "25/01/27 20:17:03 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n" + "25/02/04 14:00:37 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" ] } ], @@ -268,13 +267,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "I0000 00:00:1738009023.868557 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738009023.871444 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738009023.874007 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738009023.990970 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738009023.992104 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "I0000 00:00:1738009023.993027 3067117 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", - "2025-01-27 12:17:03.993936: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 40005 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" + "I0000 00:00:1738706437.771948 3714100 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706437.774792 3714100 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706437.777387 3714100 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706437.894244 3714100 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706437.895287 3714100 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706437.896207 3714100 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2025-02-04 14:00:37.897142: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 40337 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" ] } ], @@ -520,8 +519,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 48.6 ms, sys: 19.9 ms, total: 68.5 ms\n", - "Wall time: 15 s\n" + "CPU times: user 49.7 ms, sys: 17.6 ms, total: 67.3 ms\n", + "Wall time: 15.1 s\n" ] } ], @@ -551,26 +550,26 @@ "+----------------------------------------------------------------------------------------------------+\n", "| prediction|\n", "+----------------------------------------------------------------------------------------------------+\n", - "|[1.296447E-4, 2.465122E-4, 6.7463385E-5, 1.2231144E-4, 5.731739E-5, 3.9644213E-4, 7.0297688E-6, 4...|\n", - "|[4.4481887E-5, 3.526653E-4, 4.683818E-5, 8.1168495E-5, 3.178377E-5, 1.9188467E-4, 7.885617E-6, 1....|\n", - "|[1.05946536E-4, 2.2744355E-4, 3.0219735E-5, 6.548672E-5, 2.3649674E-5, 3.7177472E-4, 3.353236E-6,...|\n", - "|[2.0392703E-5, 2.2817637E-4, 7.840744E-5, 6.9875685E-5, 4.702542E-5, 9.8244425E-5, 5.5829764E-6, ...|\n", - "|[1.1312391E-4, 2.31244E-4, 5.279228E-5, 1.0859927E-4, 4.0202678E-5, 3.721753E-4, 5.563934E-6, 3.4...|\n", - "|[9.126345E-5, 2.0679034E-4, 4.5165678E-5, 7.679106E-5, 3.234611E-5, 3.3994843E-4, 3.84E-6, 4.1930...|\n", - "|[1.07930486E-4, 3.7741542E-4, 7.613175E-5, 1.2414041E-4, 4.7409427E-5, 3.332554E-4, 1.05853915E-5...|\n", - "|[2.2216762E-5, 2.7354853E-4, 3.8192928E-5, 6.2340725E-5, 1.7952003E-5, 1.7253387E-4, 6.020507E-6,...|\n", - "|[1.10480236E-4, 2.89734E-4, 4.239379E-5, 1.0727814E-4, 3.047985E-5, 4.7992737E-4, 6.4530495E-6, 3...|\n", - "|[9.6864875E-5, 2.0573521E-4, 7.4498465E-5, 1.1323085E-4, 4.6088306E-5, 2.8680824E-4, 5.604823E-6,...|\n", - "|[7.4198484E-5, 3.2886668E-4, 1.3441108E-4, 1.7755068E-4, 8.469927E-5, 2.2534095E-4, 1.3617541E-5,...|\n", - "|[8.7561886E-5, 2.7312653E-4, 3.5959012E-5, 7.7946424E-5, 2.3565723E-5, 3.6881721E-4, 3.5630535E-6...|\n", - "|[9.743975E-5, 2.7615853E-4, 5.74148E-5, 1.10329434E-4, 3.83045E-5, 3.500394E-4, 6.167429E-6, 4.42...|\n", - "|[6.9320704E-5, 2.53287E-4, 5.0612853E-5, 1.14936556E-4, 3.0210098E-5, 2.7870742E-4, 5.031114E-6, ...|\n", - "|[4.2203726E-5, 2.4911022E-4, 1.2378568E-4, 1.4274308E-4, 7.32259E-5, 1.6058519E-4, 7.9425035E-6, ...|\n", - "|[2.7190901E-5, 3.8381666E-4, 1.2918573E-4, 1.570463E-4, 7.310112E-5, 8.554618E-5, 1.2614603E-5, 1...|\n", - "|[3.0573912E-5, 3.5561546E-4, 1.5945674E-4, 2.1361349E-4, 8.046549E-5, 1.0269262E-4, 1.3862439E-5,...|\n", - "|[3.3117096E-5, 2.8073433E-4, 1.7961214E-4, 2.020287E-4, 1.3662946E-4, 1.0117796E-4, 3.4090703E-5,...|\n", - "|[4.5728237E-5, 2.8880237E-4, 2.3783019E-4, 2.4589908E-4, 1.2160292E-4, 1.3812551E-4, 1.6343482E-5...|\n", - "|[1.2280059E-4, 2.806991E-4, 6.3642765E-5, 1.02471764E-4, 4.351664E-5, 3.9150563E-4, 8.235125E-6, ...|\n", + "|[1.2964063E-4, 2.4653607E-4, 6.7508765E-5, 1.2236452E-4, 5.7346635E-5, 3.9642912E-4, 7.033199E-6,...|\n", + "|[4.4486973E-5, 3.5260408E-4, 4.684452E-5, 8.12069E-5, 3.179397E-5, 1.9187202E-4, 7.887208E-6, 1.3...|\n", + "|[1.059436E-4, 2.2737762E-4, 3.0225037E-5, 6.550149E-5, 2.3658315E-5, 3.7172026E-4, 3.353684E-6, 2...|\n", + "|[2.0393689E-5, 2.2818097E-4, 7.841931E-5, 6.991323E-5, 4.704759E-5, 9.822018E-5, 5.5858673E-6, 2....|\n", + "|[1.13108545E-4, 2.3128217E-4, 5.283139E-5, 1.0866656E-4, 4.0229144E-5, 3.7223354E-4, 5.5677583E-6...|\n", + "|[9.1271184E-5, 2.0681013E-4, 4.5193243E-5, 7.6812066E-5, 3.2361808E-5, 3.399333E-4, 3.8415465E-6,...|\n", + "|[1.0792112E-4, 3.7743401E-4, 7.618583E-5, 1.24259E-4, 4.7426664E-5, 3.3307416E-4, 1.0592865E-5, 9...|\n", + "|[2.2220212E-5, 2.7357432E-4, 3.8200575E-5, 6.235621E-5, 1.7954999E-5, 1.7249273E-4, 6.021971E-6, ...|\n", + "|[1.1044029E-4, 2.8961376E-4, 4.2384647E-5, 1.0728626E-4, 3.0468744E-5, 4.796082E-4, 6.4537376E-6,...|\n", + "|[9.68494E-5, 2.0567125E-4, 7.450887E-5, 1.13256065E-4, 4.609738E-5, 2.8675792E-4, 5.603957E-6, 5....|\n", + "|[7.420906E-5, 3.2883475E-4, 1.3444667E-4, 1.7758778E-4, 8.4717096E-5, 2.2534849E-4, 1.3623082E-5,...|\n", + "|[8.755989E-5, 2.7312606E-4, 3.59614E-5, 7.7967066E-5, 2.3571063E-5, 3.6875304E-4, 3.5629025E-6, 3...|\n", + "|[9.7425895E-5, 2.7611412E-4, 5.74094E-5, 1.1035101E-4, 3.8303257E-5, 3.4981826E-4, 6.167147E-6, 4...|\n", + "|[6.92996E-5, 2.5326438E-4, 5.063317E-5, 1.1494952E-4, 3.0212495E-5, 2.7857954E-4, 5.0324948E-6, 5...|\n", + "|[4.2184765E-5, 2.4904116E-4, 1.237565E-4, 1.4271903E-4, 7.3208634E-5, 1.6054673E-4, 7.938735E-6, ...|\n", + "|[2.719573E-5, 3.8372327E-4, 1.291892E-4, 1.5711001E-4, 7.3108524E-5, 8.553368E-5, 1.2617156E-5, 1...|\n", + "|[3.0565643E-5, 3.55542E-4, 1.5949155E-4, 2.1368133E-4, 8.043127E-5, 1.02662845E-4, 1.3859853E-5, ...|\n", + "|[3.311506E-5, 2.8069926E-4, 1.7956384E-4, 2.0205336E-4, 1.3665091E-4, 1.0115404E-4, 3.409792E-5, ...|\n", + "|[4.573667E-5, 2.888326E-4, 2.3792271E-4, 2.460216E-4, 1.2164583E-4, 1.3814335E-4, 1.6352218E-5, 2...|\n", + "|[1.2279079E-4, 2.8073761E-4, 6.365874E-5, 1.0251792E-4, 4.3527238E-5, 3.914249E-4, 8.236801E-6, 6...|\n", "+----------------------------------------------------------------------------------------------------+\n", "only showing top 20 rows\n", "\n" @@ -688,8 +687,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 71.1 ms, sys: 32.7 ms, total: 104 ms\n", - "Wall time: 16.6 s\n" + "CPU times: user 61.7 ms, sys: 23.1 ms, total: 84.8 ms\n", + "Wall time: 16.7 s\n" ] } ], @@ -717,8 +716,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 135 ms, sys: 27.5 ms, total: 163 ms\n", - "Wall time: 15.5 s\n" + "CPU times: user 141 ms, sys: 22.2 ms, total: 163 ms\n", + "Wall time: 16 s\n" ] } ], @@ -745,8 +744,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 53 ms, sys: 13.7 ms, total: 66.7 ms\n", - "Wall time: 16.2 s\n" + "CPU times: user 52.8 ms, sys: 14.3 ms, total: 67.1 ms\n", + "Wall time: 15.5 s\n" ] } ], @@ -776,26 +775,26 @@ "+----------------------------------------------------------------------------------------------------+\n", "| prediction|\n", "+----------------------------------------------------------------------------------------------------+\n", - "|[1.296447E-4, 2.465122E-4, 6.7463385E-5, 1.2231144E-4, 5.731739E-5, 3.9644213E-4, 7.0297688E-6, 4...|\n", - "|[4.4481887E-5, 3.526653E-4, 4.683818E-5, 8.1168495E-5, 3.178377E-5, 1.9188467E-4, 7.885617E-6, 1....|\n", - "|[1.05946536E-4, 2.2744355E-4, 3.0219735E-5, 6.548672E-5, 2.3649674E-5, 3.7177472E-4, 3.353236E-6,...|\n", - "|[2.0392703E-5, 2.2817637E-4, 7.840744E-5, 6.9875685E-5, 4.702542E-5, 9.8244425E-5, 5.5829764E-6, ...|\n", - "|[1.1312391E-4, 2.31244E-4, 5.279228E-5, 1.0859927E-4, 4.0202678E-5, 3.721753E-4, 5.563934E-6, 3.4...|\n", - "|[9.126345E-5, 2.0679034E-4, 4.5165678E-5, 7.679106E-5, 3.234611E-5, 3.3994843E-4, 3.84E-6, 4.1930...|\n", - "|[1.07930486E-4, 3.7741542E-4, 7.613175E-5, 1.2414041E-4, 4.7409427E-5, 3.332554E-4, 1.05853915E-5...|\n", - "|[2.2216762E-5, 2.7354853E-4, 3.8192928E-5, 6.2340725E-5, 1.7952003E-5, 1.7253387E-4, 6.020507E-6,...|\n", - "|[1.10480236E-4, 2.89734E-4, 4.239379E-5, 1.0727814E-4, 3.047985E-5, 4.7992737E-4, 6.4530495E-6, 3...|\n", - "|[9.6864875E-5, 2.0573521E-4, 7.4498465E-5, 1.1323085E-4, 4.6088306E-5, 2.8680824E-4, 5.604823E-6,...|\n", - "|[7.4198484E-5, 3.2886668E-4, 1.3441108E-4, 1.7755068E-4, 8.469927E-5, 2.2534095E-4, 1.3617541E-5,...|\n", - "|[8.7561886E-5, 2.7312653E-4, 3.5959012E-5, 7.7946424E-5, 2.3565723E-5, 3.6881721E-4, 3.5630535E-6...|\n", - "|[9.743975E-5, 2.7615853E-4, 5.74148E-5, 1.10329434E-4, 3.83045E-5, 3.500394E-4, 6.167429E-6, 4.42...|\n", - "|[6.9320704E-5, 2.53287E-4, 5.0612853E-5, 1.14936556E-4, 3.0210098E-5, 2.7870742E-4, 5.031114E-6, ...|\n", - "|[4.2203726E-5, 2.4911022E-4, 1.2378568E-4, 1.4274308E-4, 7.32259E-5, 1.6058519E-4, 7.9425035E-6, ...|\n", - "|[2.7190901E-5, 3.8381666E-4, 1.2918573E-4, 1.570463E-4, 7.310112E-5, 8.554618E-5, 1.2614603E-5, 1...|\n", - "|[3.0573912E-5, 3.5561546E-4, 1.5945674E-4, 2.1361349E-4, 8.046549E-5, 1.0269262E-4, 1.3862439E-5,...|\n", - "|[3.3117096E-5, 2.8073433E-4, 1.7961214E-4, 2.020287E-4, 1.3662946E-4, 1.0117796E-4, 3.4090703E-5,...|\n", - "|[4.5728237E-5, 2.8880237E-4, 2.3783019E-4, 2.4589908E-4, 1.2160292E-4, 1.3812551E-4, 1.6343482E-5...|\n", - "|[1.2280059E-4, 2.806991E-4, 6.3642765E-5, 1.02471764E-4, 4.351664E-5, 3.9150563E-4, 8.235125E-6, ...|\n", + "|[1.293178E-4, 2.4644283E-4, 6.760039E-5, 1.2260793E-4, 5.7431564E-5, 3.9597694E-4, 7.0522524E-6, ...|\n", + "|[4.4487308E-5, 3.5378174E-4, 4.6667028E-5, 8.102564E-5, 3.168566E-5, 1.9189132E-4, 7.903805E-6, 1...|\n", + "|[1.0566196E-4, 2.2684377E-4, 3.00564E-5, 6.5251304E-5, 2.3520754E-5, 3.7116173E-4, 3.331476E-6, 2...|\n", + "|[2.0337258E-5, 2.2749524E-4, 7.8351426E-5, 6.991163E-5, 4.7081656E-5, 9.8092445E-5, 5.564894E-6, ...|\n", + "|[1.12979564E-4, 2.3172122E-4, 5.2946547E-5, 1.0876398E-4, 4.0259067E-5, 3.7143996E-4, 5.5940513E-...|\n", + "|[9.093228E-5, 2.0639994E-4, 4.5151268E-5, 7.666316E-5, 3.2264295E-5, 3.387436E-4, 3.832487E-6, 4....|\n", + "|[1.0783461E-4, 3.7850672E-4, 7.660902E-5, 1.2446321E-4, 4.7591406E-5, 3.3328883E-4, 1.067249E-5, ...|\n", + "|[2.2258617E-5, 2.7345872E-4, 3.814439E-5, 6.229726E-5, 1.79387E-5, 1.7259057E-4, 6.0371217E-6, 1....|\n", + "|[1.1067773E-4, 2.8997674E-4, 4.2570035E-5, 1.0747747E-4, 3.0524247E-5, 4.7921995E-4, 6.489833E-6,...|\n", + "|[9.676251E-5, 2.0588847E-4, 7.467098E-5, 1.1326933E-4, 4.6123736E-5, 2.8609246E-4, 5.627118E-6, 5...|\n", + "|[7.4104944E-5, 3.290917E-4, 1.3448784E-4, 1.7742367E-4, 8.463227E-5, 2.2462371E-4, 1.3614881E-5, ...|\n", + "|[8.7211796E-5, 2.7337394E-4, 3.5953894E-5, 7.7924225E-5, 2.3554327E-5, 3.67775E-4, 3.5652213E-6, ...|\n", + "|[9.7237185E-5, 2.762026E-4, 5.7450008E-5, 1.1019135E-4, 3.831896E-5, 3.4878452E-4, 6.1574788E-6, ...|\n", + "|[6.938849E-5, 2.5376282E-4, 5.0565883E-5, 1.14880335E-4, 3.0061366E-5, 2.7866007E-4, 5.024482E-6,...|\n", + "|[4.2096388E-5, 2.4889092E-4, 1.2363133E-4, 1.4304162E-4, 7.337785E-5, 1.6042824E-4, 7.959722E-6, ...|\n", + "|[2.730248E-5, 3.851789E-4, 1.293143E-4, 1.5753493E-4, 7.302161E-5, 8.547956E-5, 1.26348905E-5, 1....|\n", + "|[3.0354899E-5, 3.5562844E-4, 1.6008675E-4, 2.1440513E-4, 8.062159E-5, 1.02023136E-4, 1.3876455E-5...|\n", + "|[3.3083066E-5, 2.8158593E-4, 1.7979987E-4, 2.0232225E-4, 1.3704685E-4, 1.0091762E-4, 3.4243407E-5...|\n", + "|[4.5485373E-5, 2.878148E-4, 2.3707838E-4, 2.4493985E-4, 1.21028905E-4, 1.3738636E-4, 1.6280053E-5...|\n", + "|[1.22468E-4, 2.809503E-4, 6.3342835E-5, 1.021957E-4, 4.3373006E-5, 3.905496E-4, 8.212427E-6, 6.20...|\n", "+----------------------------------------------------------------------------------------------------+\n", "only showing top 20 rows\n", "\n" @@ -864,7 +863,7 @@ "id": "cdded12d", "metadata": {}, "source": [ - "Import the utility functions from pytriton_utils.py:" + "Import the helper class from pytriton_utils.py:" ] }, { @@ -876,12 +875,7 @@ "source": [ "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "from pytriton_utils import (\n", - " use_stage_level_scheduling,\n", - " find_ports,\n", - " start_triton,\n", - " stop_triton\n", - ")" + "from pytriton_utils import TritonServerManager" ] }, { @@ -952,6 +946,7 @@ " )\n", "\n", " def _stop_triton(signum, frame):\n", + " # The server manager sends SIGTERM to stop the server; this function ensures graceful cleanup.\n", " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", " triton.stop()\n", "\n", @@ -991,82 +986,38 @@ }, { "cell_type": "markdown", - "id": "f5810c77", + "id": "4bf99bde", "metadata": {}, "source": [ - "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + "The `TritonClusterManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n", + "- Find available ports for HTTP/gRPC/metrics\n", + "- Deploy a server on each node via stage-level scheduling\n", + "- Gracefully shutdown servers across nodes" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "id": "2309a55c", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - } - ], - "source": [ - "sc = spark.sparkContext\n", - "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" - ] - }, - { - "cell_type": "markdown", - "id": "533e2a89", - "metadata": {}, - "source": [ - "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "id": "dfc8834a", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using ports [7000, 7001, 7002]\n" - ] - } - ], + "outputs": [], "source": [ "model_name = \"ResNet50\"\n", - "ports = find_ports()\n", - "assert len(ports) == 3\n", - "print(f\"Using ports {ports}\")" + "server_manager = TritonServerManager(num_nodes=num_nodes, model_name=model_name)" ] }, { "cell_type": "code", - "execution_count": 34, - "id": "ad24bc52", + "execution_count": null, + "id": "205fa1e8", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 15:> (0 + 1) / 1]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Triton Server PIDs:\n", - " {\n", - " \"cb4ae00-lcedt\": 3077881\n", - "}\n" + "2025-02-07 11:03:44,809 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-07 11:03:44,810 - INFO - Starting 1 servers.\n" ] }, { @@ -1075,13 +1026,20 @@ "text": [ " \r" ] + }, + { + "data": { + "text/plain": [ + "{'cb4ae00-lcedt': (2020631, [7000, 7001, 7002])}" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", - " ports=ports,\n", - " model_name=model_name)).collectAsMap()\n", - "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n", + "server_manager.start_servers(triton_server)" ] }, { @@ -1092,27 +1050,45 @@ "#### Define client function" ] }, + { + "cell_type": "markdown", + "id": "55c42174", + "metadata": {}, + "source": [ + "Get the hostname -> url mapping from the server manager:" + ] + }, { "cell_type": "code", - "execution_count": 35, - "id": "aa34bebb", + "execution_count": null, + "id": "9e4ff20e", "metadata": {}, "outputs": [], "source": [ - "url = f\"http://localhost:{ports[0]}\"" + "host_to_http_url = server_manager.host_to_http_url # or server_manager.host_to_grpc_url" + ] + }, + { + "cell_type": "markdown", + "id": "481dbd42", + "metadata": {}, + "source": [ + "Define the Triton inference function, which returns a predict function for batch inference through the server:" ] }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 35, "id": "a5ab49bb", "metadata": {}, "outputs": [], "source": [ - "def triton_fn(url, model_name):\n", + "def triton_fn(model_name, host_to_url):\n", + " import socket\n", " import numpy as np\n", " from pytriton.client import ModelClient\n", "\n", + " url = host_to_url[socket.gethostname()]\n", " print(f\"CLIENT: Connecting to {model_name} at {url}\")\n", "\n", " def infer_batch(inputs):\n", @@ -1123,6 +1099,19 @@ " return infer_batch" ] }, + { + "cell_type": "code", + "execution_count": 37, + "id": "9fabcaeb-5a44-42bb-8097-5dbc2d0cee3e", + "metadata": {}, + "outputs": [], + "source": [ + "classify = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_http_url),\n", + " input_tensor_shapes=[[224, 224, 3]],\n", + " return_type=ArrayType(FloatType()),\n", + " batch_size=50)" + ] + }, { "cell_type": "markdown", "id": "fcd2328e", @@ -1133,7 +1122,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 36, "id": "bbfc9009", "metadata": {}, "outputs": [], @@ -1152,21 +1141,6 @@ { "cell_type": "code", "execution_count": 38, - "id": "9fabcaeb-5a44-42bb-8097-5dbc2d0cee3e", - "metadata": {}, - "outputs": [], - "source": [ - "from functools import partial\n", - "\n", - "classify = predict_batch_udf(partial(triton_fn, url=url, model_name=model_name),\n", - " input_tensor_shapes=[[224, 224, 3]],\n", - " return_type=ArrayType(FloatType()),\n", - " batch_size=50)" - ] - }, - { - "cell_type": "code", - "execution_count": 39, "id": "e595473d-1a5d-46a6-a6ba-89d2ea903de9", "metadata": {}, "outputs": [ @@ -1181,7 +1155,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 53 ms, sys: 17.7 ms, total: 70.7 ms\n", + "CPU times: user 60.9 ms, sys: 21.3 ms, total: 82.2 ms\n", "Wall time: 18.4 s\n" ] } @@ -1195,7 +1169,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 39, "id": "5f66d468-e0b1-4589-8606-b3848063a823", "metadata": {}, "outputs": [ @@ -1210,8 +1184,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 55.5 ms, sys: 20.2 ms, total: 75.7 ms\n", - "Wall time: 12.2 s\n" + "CPU times: user 46.3 ms, sys: 16.1 ms, total: 62.4 ms\n", + "Wall time: 12.3 s\n" ] } ], @@ -1223,7 +1197,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 40, "id": "632c4c3a-fa52-4c3d-b71e-7526286e353a", "metadata": {}, "outputs": [ @@ -1238,8 +1212,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 50.6 ms, sys: 18.2 ms, total: 68.8 ms\n", - "Wall time: 12.1 s\n" + "CPU times: user 57.5 ms, sys: 16.4 ms, total: 73.9 ms\n", + "Wall time: 12.4 s\n" ] } ], @@ -1251,7 +1225,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 41, "id": "49870e39", "metadata": {}, "outputs": [ @@ -1259,7 +1233,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 21:===========================================> (3 + 1) / 4]\r" + "[Stage 22:===========================================> (3 + 1) / 4]\r" ] }, { @@ -1269,26 +1243,26 @@ "+----------------------------------------------------------------------------------------------------+\n", "| prediction|\n", "+----------------------------------------------------------------------------------------------------+\n", - "|[1.2964063E-4, 2.4653607E-4, 6.7508765E-5, 1.2236452E-4, 5.7346635E-5, 3.9642912E-4, 7.033199E-6,...|\n", - "|[4.4486973E-5, 3.5260408E-4, 4.684452E-5, 8.12069E-5, 3.179397E-5, 1.9187202E-4, 7.887208E-6, 1.3...|\n", - "|[1.059436E-4, 2.2737762E-4, 3.0225037E-5, 6.550149E-5, 2.3658315E-5, 3.7172026E-4, 3.353684E-6, 2...|\n", - "|[2.0393689E-5, 2.2818097E-4, 7.841931E-5, 6.991323E-5, 4.704759E-5, 9.822018E-5, 5.5858673E-6, 2....|\n", - "|[1.13108545E-4, 2.3128217E-4, 5.283139E-5, 1.0866656E-4, 4.0229144E-5, 3.7223354E-4, 5.5677583E-6...|\n", - "|[9.1271184E-5, 2.0681013E-4, 4.5193243E-5, 7.6812066E-5, 3.2361808E-5, 3.399333E-4, 3.8415465E-6,...|\n", - "|[1.0792112E-4, 3.7743401E-4, 7.618583E-5, 1.24259E-4, 4.7426664E-5, 3.3307416E-4, 1.0592865E-5, 9...|\n", - "|[2.2220212E-5, 2.7357432E-4, 3.8200575E-5, 6.235621E-5, 1.7954999E-5, 1.7249273E-4, 6.021971E-6, ...|\n", - "|[1.1044029E-4, 2.8961376E-4, 4.2384647E-5, 1.0728626E-4, 3.0468744E-5, 4.796082E-4, 6.4537376E-6,...|\n", - "|[9.68494E-5, 2.0567125E-4, 7.450887E-5, 1.13256065E-4, 4.609738E-5, 2.8675792E-4, 5.603957E-6, 5....|\n", - "|[7.420906E-5, 3.2883475E-4, 1.3444667E-4, 1.7758778E-4, 8.4717096E-5, 2.2534849E-4, 1.3623082E-5,...|\n", - "|[8.755989E-5, 2.7312606E-4, 3.59614E-5, 7.7967066E-5, 2.3571063E-5, 3.6875304E-4, 3.5629025E-6, 3...|\n", - "|[9.7425895E-5, 2.7611412E-4, 5.74094E-5, 1.1035101E-4, 3.8303257E-5, 3.4981826E-4, 6.167147E-6, 4...|\n", - "|[6.92996E-5, 2.5326438E-4, 5.063317E-5, 1.1494952E-4, 3.0212495E-5, 2.7857954E-4, 5.0324948E-6, 5...|\n", - "|[4.2184765E-5, 2.4904116E-4, 1.237565E-4, 1.4271903E-4, 7.3208634E-5, 1.6054673E-4, 7.938735E-6, ...|\n", - "|[2.719573E-5, 3.8372327E-4, 1.291892E-4, 1.5711001E-4, 7.3108524E-5, 8.553368E-5, 1.2617156E-5, 1...|\n", - "|[3.0565643E-5, 3.55542E-4, 1.5949155E-4, 2.1368133E-4, 8.043127E-5, 1.02662845E-4, 1.3859853E-5, ...|\n", - "|[3.311506E-5, 2.8069926E-4, 1.7956384E-4, 2.0205336E-4, 1.3665091E-4, 1.0115404E-4, 3.409792E-5, ...|\n", - "|[4.573667E-5, 2.888326E-4, 2.3792271E-4, 2.460216E-4, 1.2164583E-4, 1.3814335E-4, 1.6352218E-5, 2...|\n", - "|[1.2279079E-4, 2.8073761E-4, 6.365874E-5, 1.0251792E-4, 4.3527238E-5, 3.914249E-4, 8.236801E-6, 6...|\n", + "|[1.293178E-4, 2.4644283E-4, 6.760039E-5, 1.2260793E-4, 5.7431564E-5, 3.9597694E-4, 7.0522524E-6, ...|\n", + "|[4.4487308E-5, 3.5378174E-4, 4.6667028E-5, 8.102564E-5, 3.168566E-5, 1.9189132E-4, 7.903805E-6, 1...|\n", + "|[1.0566196E-4, 2.2684377E-4, 3.00564E-5, 6.5251304E-5, 2.3520754E-5, 3.7116173E-4, 3.331476E-6, 2...|\n", + "|[2.0337258E-5, 2.2749524E-4, 7.8351426E-5, 6.991163E-5, 4.7081656E-5, 9.8092445E-5, 5.564894E-6, ...|\n", + "|[1.12979564E-4, 2.3172122E-4, 5.2946547E-5, 1.0876398E-4, 4.0259067E-5, 3.7143996E-4, 5.5940513E-...|\n", + "|[9.093228E-5, 2.0639994E-4, 4.5151268E-5, 7.666316E-5, 3.2264295E-5, 3.387436E-4, 3.832487E-6, 4....|\n", + "|[1.0783461E-4, 3.7850672E-4, 7.660902E-5, 1.2446321E-4, 4.7591406E-5, 3.3328883E-4, 1.067249E-5, ...|\n", + "|[2.2258617E-5, 2.7345872E-4, 3.814439E-5, 6.229726E-5, 1.79387E-5, 1.7259057E-4, 6.0371217E-6, 1....|\n", + "|[1.1067773E-4, 2.8997674E-4, 4.2570035E-5, 1.0747747E-4, 3.0524247E-5, 4.7921995E-4, 6.489833E-6,...|\n", + "|[9.676251E-5, 2.0588847E-4, 7.467098E-5, 1.1326933E-4, 4.6123736E-5, 2.8609246E-4, 5.627118E-6, 5...|\n", + "|[7.4104944E-5, 3.290917E-4, 1.3448784E-4, 1.7742367E-4, 8.463227E-5, 2.2462371E-4, 1.3614881E-5, ...|\n", + "|[8.7211796E-5, 2.7337394E-4, 3.5953894E-5, 7.7924225E-5, 2.3554327E-5, 3.67775E-4, 3.5652213E-6, ...|\n", + "|[9.7237185E-5, 2.762026E-4, 5.7450008E-5, 1.1019135E-4, 3.831896E-5, 3.4878452E-4, 6.1574788E-6, ...|\n", + "|[6.938849E-5, 2.5376282E-4, 5.0565883E-5, 1.14880335E-4, 3.0061366E-5, 2.7866007E-4, 5.024482E-6,...|\n", + "|[4.2096388E-5, 2.4889092E-4, 1.2363133E-4, 1.4304162E-4, 7.337785E-5, 1.6042824E-4, 7.959722E-6, ...|\n", + "|[2.730248E-5, 3.851789E-4, 1.293143E-4, 1.5753493E-4, 7.302161E-5, 8.547956E-5, 1.26348905E-5, 1....|\n", + "|[3.0354899E-5, 3.5562844E-4, 1.6008675E-4, 2.1440513E-4, 8.062159E-5, 1.02023136E-4, 1.3876455E-5...|\n", + "|[3.3083066E-5, 2.8158593E-4, 1.7979987E-4, 2.0232225E-4, 1.3704685E-4, 1.0091762E-4, 3.4243407E-5...|\n", + "|[4.5485373E-5, 2.878148E-4, 2.3707838E-4, 2.4493985E-4, 1.21028905E-4, 1.3738636E-4, 1.6280053E-5...|\n", + "|[1.22468E-4, 2.809503E-4, 6.3342835E-5, 1.021957E-4, 4.3373006E-5, 3.905496E-4, 8.212427E-6, 6.20...|\n", "+----------------------------------------------------------------------------------------------------+\n", "only showing top 20 rows\n", "\n" @@ -1308,7 +1282,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 42, "id": "86cd59f9", "metadata": {}, "outputs": [ @@ -1336,22 +1310,16 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 43, "id": "bbfcaa51-3b9f-43ff-a4a8-4b46766115b8", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "2025-02-04 14:03:34,747 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-04 14:03:39,935 - INFO - Sucessfully stopped 1 servers. \n" ] }, { @@ -1360,20 +1328,18 @@ "[True]" ] }, - "execution_count": 44, + "execution_count": 43, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", - "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" + "server_manager.stop_servers()" ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 44, "id": "0d88639b-d934-4eb4-ae2f-cc13b9b10456", "metadata": {}, "outputs": [], diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/text_classification_tf.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/text_classification_tf.ipynb index bfa5affb..a5dc4d09 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/text_classification_tf.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/tensorflow/text_classification_tf.ipynb @@ -32,13 +32,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-01-07 17:55:03.625173: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2025-01-07 17:55:03.632499: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2025-01-07 17:55:03.640392: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2025-01-07 17:55:03.642797: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2025-01-07 17:55:03.648973: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "2025-02-04 14:05:12.899608: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "2025-02-04 14:05:12.907256: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:485] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2025-02-04 14:05:12.915374: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:8454] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2025-02-04 14:05:12.917743: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1452] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2025-02-04 14:05:12.924372: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX2 AVX_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2025-01-07 17:55:04.012978: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" + "2025-02-04 14:05:13.295411: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" ] } ], @@ -65,6 +65,16 @@ "text": [ "2.17.0\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", + "I0000 00:00:1738706713.692042 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706713.716276 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706713.719037 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n" + ] } ], "source": [ @@ -180,7 +190,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-01-07 17:55:15.035387: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 45468 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" + "I0000 00:00:1738706719.326625 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706719.329542 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706719.332409 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706719.451656 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706719.452700 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "I0000 00:00:1738706719.453630 3744395 cuda_executor.cc:1015] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n", + "2025-02-04 14:05:19.454569: I tensorflow/core/common_runtime/gpu/gpu_device.cc:2021] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 40337 MB memory: -> device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:01:00.0, compute capability: 8.6\n" ] }, { @@ -249,7 +265,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-01-07 17:55:21.572943: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + "2025-02-04 14:05:20.533703: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" ] } ], @@ -358,7 +374,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-01-07 17:55:35.387277: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" + "2025-02-04 14:05:22.003236: I tensorflow/core/framework/local_rendezvous.cc:404] Local rendezvous is aborting with status: OUT_OF_RANGE: End of sequence\n" ] } ], @@ -671,49 +687,49 @@ "output_type": "stream", "text": [ "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", - "I0000 00:00:1736272546.613950 3675377 service.cc:146] XLA service 0x70ca64001f30 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n", - "I0000 00:00:1736272546.613963 3675377 service.cc:154] StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n", - "2025-01-07 17:55:46.627250: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", - "2025-01-07 17:55:46.680543: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8907\n" + "I0000 00:00:1738706722.621647 3744883 service.cc:146] XLA service 0x334cd320 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n", + "I0000 00:00:1738706722.621667 3744883 service.cc:154] StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n", + "2025-02-04 14:05:22.635317: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:268] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable.\n", + "2025-02-04 14:05:22.689182: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:531] Loaded cuDNN version 8907\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m 89/625\u001b[0m \u001b[32m━━\u001b[0m\u001b[37m━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - binary_accuracy: 0.5183 - loss: 0.6920" + "\u001b[1m262/625\u001b[0m \u001b[32m━━━━━━━━\u001b[0m\u001b[37m━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 578us/step - binary_accuracy: 0.5299 - loss: 0.6904" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "I0000 00:00:1736272547.264048 3675377 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n" + "I0000 00:00:1738706723.175401 3744883 device_compiler.h:188] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step - binary_accuracy: 0.5778 - loss: 0.6821 - val_binary_accuracy: 0.7072 - val_loss: 0.6182\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step - binary_accuracy: 0.5692 - loss: 0.6832 - val_binary_accuracy: 0.7020 - val_loss: 0.6195\n", "Epoch 2/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 469us/step - binary_accuracy: 0.7568 - loss: 0.5817 - val_binary_accuracy: 0.8002 - val_loss: 0.4987\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 455us/step - binary_accuracy: 0.7588 - loss: 0.5825 - val_binary_accuracy: 0.7954 - val_loss: 0.5009\n", "Epoch 3/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 453us/step - binary_accuracy: 0.8323 - loss: 0.4656 - val_binary_accuracy: 0.8352 - val_loss: 0.4233\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 536us/step - binary_accuracy: 0.8293 - loss: 0.4681 - val_binary_accuracy: 0.8352 - val_loss: 0.4253\n", "Epoch 4/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 449us/step - binary_accuracy: 0.8535 - loss: 0.3954 - val_binary_accuracy: 0.8520 - val_loss: 0.3789\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 516us/step - binary_accuracy: 0.8523 - loss: 0.3967 - val_binary_accuracy: 0.8516 - val_loss: 0.3802\n", "Epoch 5/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 439us/step - binary_accuracy: 0.8708 - loss: 0.3502 - val_binary_accuracy: 0.8590 - val_loss: 0.3506\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 448us/step - binary_accuracy: 0.8692 - loss: 0.3524 - val_binary_accuracy: 0.8592 - val_loss: 0.3522\n", "Epoch 6/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 433us/step - binary_accuracy: 0.8805 - loss: 0.3183 - val_binary_accuracy: 0.8668 - val_loss: 0.3309\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 530us/step - binary_accuracy: 0.8810 - loss: 0.3199 - val_binary_accuracy: 0.8658 - val_loss: 0.3324\n", "Epoch 7/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 447us/step - binary_accuracy: 0.8908 - loss: 0.2935 - val_binary_accuracy: 0.8698 - val_loss: 0.3170\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 489us/step - binary_accuracy: 0.8919 - loss: 0.2945 - val_binary_accuracy: 0.8666 - val_loss: 0.3188\n", "Epoch 8/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 451us/step - binary_accuracy: 0.9002 - loss: 0.2714 - val_binary_accuracy: 0.8714 - val_loss: 0.3088\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 509us/step - binary_accuracy: 0.8975 - loss: 0.2744 - val_binary_accuracy: 0.8720 - val_loss: 0.3085\n", "Epoch 9/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 427us/step - binary_accuracy: 0.9057 - loss: 0.2557 - val_binary_accuracy: 0.8712 - val_loss: 0.3032\n", + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 389us/step - binary_accuracy: 0.9042 - loss: 0.2565 - val_binary_accuracy: 0.8756 - val_loss: 0.3017\n", "Epoch 10/10\n", - "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 404us/step - binary_accuracy: 0.9113 - loss: 0.2414 - val_binary_accuracy: 0.8766 - val_loss: 0.2969\n" + "\u001b[1m625/625\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 410us/step - binary_accuracy: 0.9121 - loss: 0.2409 - val_binary_accuracy: 0.8750 - val_loss: 0.2972\n" ] } ], @@ -743,9 +759,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 747us/step - binary_accuracy: 0.8727 - loss: 0.3141\n", - "Loss: 0.3163740932941437\n", - "Accuracy: 0.8708400130271912\n" + "\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 573us/step - binary_accuracy: 0.8719 - loss: 0.3147\n", + "Loss: 0.3172186613082886\n", + "Accuracy: 0.8701599836349487\n" ] } ], @@ -794,7 +810,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABWv0lEQVR4nO3dd3gU1f7H8fcmIQ1I6EmAEIpI701AioKChSICQVECFq5IFfEC0gIIqAiCIlUFFUUEQ5cuXBFQuFJERJArTSAUgYQe2Mzvj/llSUghfZLdz+t59tnds7Mz302i++HMmXNshmEYiIiIiDgJN6sLEBEREclMCjciIiLiVBRuRERExKko3IiIiIhTUbgRERERp6JwIyIiIk5F4UZEREScisKNiIiIOBWFGxEREXEqCjciFujevTulS5dO13vDw8Ox2WyZW1AOc/ToUWw2G/PmzcvW427evBmbzcbmzZsdban9XWVVzaVLl6Z79+6Zus/UmDdvHjabjaNHj2b7sUUySuFGJB6bzZaqW/wvP5GM2rZtG+Hh4Vy6dMnqUkScgofVBYjkJF988UWC559//jnr169P1F6pUqUMHWfOnDnExsam673Dhw9nyJAhGTq+pF5GfleptW3bNkaPHk337t0pUKBAgtcOHjyIm5v+HSqSFgo3IvE899xzCZ7/9NNPrF+/PlH73a5du4avr2+qj5MnT5501Qfg4eGBh4f+080uGfldZQYvLy9Ljy+SG+mfAyJp1Lx5c6pWrcovv/xC06ZN8fX15c033wRg2bJlPPHEExQvXhwvLy/KlSvH2LFjsdvtCfZx9ziOuPEa7733HrNnz6ZcuXJ4eXlRr149du7cmeC9SY25sdls9OnTh6VLl1K1alW8vLyoUqUKa9asSVT/5s2bqVu3Lt7e3pQrV45Zs2alehzPli1b6NSpE6VKlcLLy4vg4GBee+01rl+/nujz5cuXj5MnT9K+fXvy5ctH0aJFGTRoUKKfxaVLl+jevTv+/v4UKFCAsLCwVJ2e+e9//4vNZuOzzz5L9NratWux2WysXLkSgGPHjvHqq69SoUIFfHx8KFy4MJ06dUrVeJKkxtyktuZff/2V7t27U7ZsWby9vQkMDOSFF17gn3/+cWwTHh7OG2+8AUCZMmUcpz7jaktqzM1ff/1Fp06dKFSoEL6+vjzwwAOsWrUqwTZx44e++eYbxo0bR8mSJfH29qZFixYcPnz4np87OdOnT6dKlSp4eXlRvHhxevfuneiz//nnnzz99NMEBgbi7e1NyZIl6dKlC1FRUY5t1q9fz4MPPkiBAgXIly8fFSpUcPx3JJJR+uefSDr8888/PPbYY3Tp0oXnnnuOgIAAwByEmS9fPgYOHEi+fPn4/vvvGTlyJNHR0UycOPGe+/3qq6+4fPky//rXv7DZbLz77rt06NCBv/766549CD/++CMRERG8+uqr5M+fnw8++ICnn36a48ePU7hwYQB2795N69atCQoKYvTo0djtdsaMGUPRokVT9bkXLVrEtWvX6NWrF4ULF2bHjh18+OGH/P333yxatCjBtna7nVatWtGgQQPee+89NmzYwKRJkyhXrhy9evUCwDAM2rVrx48//sgrr7xCpUqVWLJkCWFhYfespW7dupQtW5Zvvvkm0fYLFy6kYMGCtGrVCoCdO3eybds2unTpQsmSJTl69CgzZsygefPm/P7772nqdUtLzevXr+evv/6iR48eBAYGsn//fmbPns3+/fv56aefsNlsdOjQgUOHDrFgwQLef/99ihQpApDs7+TMmTM0atSIa9eu0a9fPwoXLsxnn31G27ZtWbx4MU899VSC7d9++23c3NwYNGgQUVFRvPvuu3Tt2pWff/451Z85Tnh4OKNHj6Zly5b06tWLgwcPMmPGDHbu3MnWrVvJkycPMTExtGrVips3b9K3b18CAwM5efIkK1eu5NKlS/j7+7N//36efPJJqlevzpgxY/Dy8uLw4cNs3bo1zTWJJMkQkWT17t3buPs/k2bNmhmAMXPmzETbX7t2LVHbv/71L8PX19e4ceOGoy0sLMwICQlxPD9y5IgBGIULFzYuXLjgaF+2bJkBGCtWrHC0jRo1KlFNgOHp6WkcPnzY0bZ3714DMD788ENHW5s2bQxfX1/j5MmTjrY///zT8PDwSLTPpCT1+SZMmGDYbDbj2LFjCT4fYIwZMybBtrVq1TLq1KnjeL506VIDMN59911H2+3bt40mTZoYgDF37twU6xk6dKiRJ0+eBD+zmzdvGgUKFDBeeOGFFOvevn27ARiff/65o23Tpk0GYGzatCnBZ4n/u0pLzUkdd8GCBQZg/PDDD462iRMnGoBx5MiRRNuHhIQYYWFhjucDBgwwAGPLli2OtsuXLxtlypQxSpcubdjt9gSfpVKlSsbNmzcd206dOtUAjH379iU6Vnxz585NUNPZs2cNT09P49FHH3UcwzAMY9q0aQZgfPrpp4ZhGMbu3bsNwFi0aFGy+37//fcNwDh37lyKNYikl05LiaSDl5cXPXr0SNTu4+PjeHz58mXOnz9PkyZNuHbtGn/88cc99xsaGkrBggUdz5s0aQKYpyHupWXLlpQrV87xvHr16vj5+Tnea7fb2bBhA+3bt6d48eKO7e677z4ee+yxe+4fEn6+q1evcv78eRo1aoRhGOzevTvR9q+88kqC502aNEnwWb777js8PDwcPTkA7u7u9O3bN1X1hIaGcuvWLSIiIhxt69at49KlS4SGhiZZ961bt/jnn3+47777KFCgALt27UrVsdJTc/zj3rhxg/Pnz/PAAw8ApPm48Y9fv359HnzwQUdbvnz56NmzJ0ePHuX3339PsH2PHj3w9PR0PE/L31R8GzZsICYmhgEDBiQY4Pzyyy/j5+fnOC3m7+8PmKcGr127luS+4gZNL1u2LMsHa4trUrgRSYcSJUok+MKIs3//fp566in8/f3x8/OjaNGijsHI8ccbJKdUqVIJnscFnYsXL6b5vXHvj3vv2bNnuX79Ovfdd1+i7ZJqS8rx48fp3r07hQoVcoyjadasGZD483l7eyc6tRK/HjDHwgQFBZEvX74E21WoUCFV9dSoUYOKFSuycOFCR9vChQspUqQIDz/8sKPt+vXrjBw5kuDgYLy8vChSpAhFixbl0qVLqfq9xJeWmi9cuED//v0JCAjAx8eHokWLUqZMGSB1fw/JHT+pY8VdwXfs2LEE7Rn5m7r7uJD4c3p6elK2bFnH62XKlGHgwIF8/PHHFClShFatWvHRRx8l+LyhoaE0btyYl156iYCAALp06cI333yjoCOZRmNuRNIh/r/I41y6dIlmzZrh5+fHmDFjKFeuHN7e3uzatYvBgwen6n/c7u7uSbYbhpGl700Nu93OI488woULFxg8eDAVK1Ykb968nDx5ku7duyf6fMnVk9lCQ0MZN24c58+fJ3/+/CxfvpxnnnkmwRVlffv2Ze7cuQwYMICGDRvi7++PzWajS5cuWfqF2rlzZ7Zt28Ybb7xBzZo1yZcvH7GxsbRu3Trbvsiz+u8iKZMmTaJ79+4sW7aMdevW0a9fPyZMmMBPP/1EyZIl8fHx4YcffmDTpk2sWrWKNWvWsHDhQh5++GHWrVuXbX874rwUbkQyyebNm/nnn3+IiIigadOmjvYjR45YWNUdxYoVw9vbO8krZVJz9cy+ffs4dOgQn332Gd26dXO0r1+/Pt01hYSEsHHjRq5cuZKgJ+TgwYOp3kdoaCijR4/m22+/JSAggOjoaLp06ZJgm8WLFxMWFsakSZMcbTdu3EjXpHmprfnixYts3LiR0aNHM3LkSEf7n3/+mWifaZlxOiQkJMmfT9xpz5CQkFTvKy3i9nvw4EHKli3raI+JieHIkSO0bNkywfbVqlWjWrVqDB8+nG3bttG4cWNmzpzJW2+9BYCbmxstWrSgRYsWTJ48mfHjxzNs2DA2bdqUaF8iaaXTUiKZJO5fm/H/RRwTE8P06dOtKikBd3d3WrZsydKlSzl16pSj/fDhw6xevTpV74eEn88wDKZOnZrumh5//HFu377NjBkzHG12u50PP/ww1fuoVKkS1apVY+HChSxcuJCgoKAE4TKu9rt7Kj788MNEl6VnZs1J/bwApkyZkmifefPmBUhV2Hr88cfZsWMH27dvd7RdvXqV2bNnU7p0aSpXrpzaj5ImLVu2xNPTkw8++CDBZ/rkk0+IioriiSeeACA6Oprbt28neG+1atVwc3Pj5s2bgHm67m41a9YEcGwjkhHquRHJJI0aNaJgwYKEhYXRr18/bDYbX3zxRZZ2/6dVeHg469ato3HjxvTq1Qu73c60adOoWrUqe/bsSfG9FStWpFy5cgwaNIiTJ0/i5+fHt99+m+axG/G1adOGxo0bM2TIEI4ePUrlypWJiIhI83iU0NBQRo4cibe3Ny+++GKiGX2ffPJJvvjiC/z9/alcuTLbt29nw4YNjkvks6JmPz8/mjZtyrvvvsutW7coUaIE69atS7Inr06dOgAMGzaMLl26kCdPHtq0aeMIPfENGTKEBQsW8Nhjj9GvXz8KFSrEZ599xpEjR/j222+zbDbjokWLMnToUEaPHk3r1q1p27YtBw8eZPr06dSrV88xtuz777+nT58+dOrUifvvv5/bt2/zxRdf4O7uztNPPw3AmDFj+OGHH3jiiScICQnh7NmzTJ8+nZIlSyYYKC2SXgo3IpmkcOHCrFy5ktdff53hw4dTsGBBnnvuOVq0aOGYb8VqderUYfXq1QwaNIgRI0YQHBzMmDFjOHDgwD2v5sqTJw8rVqxwjJ/w9vbmqaeeok+fPtSoUSNd9bi5ubF8+XIGDBjA/PnzsdlstG3blkmTJlGrVq1U7yc0NJThw4dz7dq1BFdJxZk6dSru7u58+eWX3Lhxg8aNG7Nhw4Z0/V7SUvNXX31F3759+eijjzAMg0cffZTVq1cnuFoNoF69eowdO5aZM2eyZs0aYmNjOXLkSJLhJiAggG3btjF48GA+/PBDbty4QfXq1VmxYoWj9ySrhIeHU7RoUaZNm8Zrr71GoUKF6NmzJ+PHj3fMw1SjRg1atWrFihUrOHnyJL6+vtSoUYPVq1c7rhRr27YtR48e5dNPP+X8+fMUKVKEZs2aMXr0aMfVViIZYTNy0j8rRcQS7du3Z//+/UmOBxERyW005kbExdy9VMKff/7Jd999R/Pmza0pSEQkk6nnRsTFBAUFOdY7OnbsGDNmzODmzZvs3r2b8uXLW12eiEiGacyNiItp3bo1CxYsIDIyEi8vLxo2bMj48eMVbETEaajnRkRERJyKxtyIiIiIU1G4EREREaficmNuYmNjOXXqFPnz50/TlOciIiJiHcMwuHz5MsWLF7/nZJUuF25OnTpFcHCw1WWIiIhIOpw4cYKSJUumuI3LhZv8+fMD5g/Hz8/P4mpEREQkNaKjowkODnZ8j6fE5cJN3KkoPz8/hRsREZFcJjVDSjSgWERERJyKwo2IiIg4FYUbERERcSouN+ZGREQyl91u59atW1aXIU7A09Pznpd5p4bCjYiIpIthGERGRnLp0iWrSxEn4ebmRpkyZfD09MzQfhRuREQkXeKCTbFixfD19dXEqJIhcZPsnj59mlKlSmXo70nhRkRE0sxutzuCTeHCha0uR5xE0aJFOXXqFLdv3yZPnjzp3o8GFIuISJrFjbHx9fW1uBJxJnGno+x2e4b2o3AjIiLpplNRkpky6+9Jp6Uyid0OW7bA6dMQFARNmoC7u9VViYiIuB713GSCiAgoXRoeegiefda8L13abBcREedXunRppkyZkurtN2/ejM1my/IrzebNm0eBAgWy9Bg5kcJNBkVEQMeO8PffCdtPnjTbFXBERFJmt8PmzbBggXmfweEWKbLZbCnewsPD07XfnTt30rNnz1Rv36hRI06fPo2/v3+6jicp02mpDLDboX9/MIzErxkG2GwwYAC0a6dTVCIiSYmIMP8/Gv8fiCVLwtSp0KFD5h/v9OnTjscLFy5k5MiRHDx40NGWL18+x2PDMLDb7Xh43PursmjRommqw9PTk8DAwDS9R1JPPTcZsGVL4h6b+AwDTpwwtxMRkYSs6PkODAx03Pz9/bHZbI7nf/zxB/nz52f16tXUqVMHLy8vfvzxR/73v//Rrl07AgICyJcvH/Xq1WPDhg0J9nv3aSmbzcbHH3/MU089ha+vL+XLl2f58uWO1+8+LRV3+mjt2rVUqlSJfPny0bp16wRh7Pbt2/Tr148CBQpQuHBhBg8eTFhYGO3bt0/Tz2DGjBmUK1cOT09PKlSowBdffOF4zTAMwsPDKVWqFF5eXhQvXpx+/fo5Xp8+fTrly5fH29ubgIAAOnbsmKZjZxeFmwyI9zeXKduJiLiKe/V8g9nznZWnqJIzZMgQ3n77bQ4cOED16tW5cuUKjz/+OBs3bmT37t20bt2aNm3acPz48RT3M3r0aDp37syvv/7K448/TteuXblw4UKy21+7do333nuPL774gh9++IHjx48zaNAgx+vvvPMOX375JXPnzmXr1q1ER0ezdOnSNH22JUuW0L9/f15//XV+++03/vWvf9GjRw82bdoEwLfffsv777/PrFmz+PPPP1m6dCnVqlUD4L///S/9+vVjzJgxHDx4kDVr1tC0adM0HT/bGC4mKirKAIyoqKgM72vTJsMw/zNM+bZpU4YPJSKSo1y/ft34/fffjevXr6fr/Tnh/59z5841/P3949W0yQCMpUuX3vO9VapUMT788EPH85CQEOP99993PAeM4cOHO55fuXLFAIzVq1cnONbFixcdtQDG4cOHHe/56KOPjICAAMfzgIAAY+LEiY7nt2/fNkqVKmW0a9cu1Z+xUaNGxssvv5xgm06dOhmPP/64YRiGMWnSJOP+++83YmJiEu3r22+/Nfz8/Izo6Ohkj5dRKf1dpeX7Wz03GdCkiXluOLnL8m02CA42txMRkTtycs933bp1Ezy/cuUKgwYNolKlShQoUIB8+fJx4MCBe/bcVK9e3fE4b968+Pn5cfbs2WS39/X1pVy5co7nQUFBju2joqI4c+YM9evXd7zu7u5OnTp10vTZDhw4QOPGjRO0NW7cmAMHDgDQqVMnrl+/TtmyZXn55ZdZsmQJt2/fBuCRRx4hJCSEsmXL8vzzz/Pll19y7dq1NB0/uyjcZIC7uznoDRIHnLjnU6ZoMLGIyN2CgjJ3u8yUN2/eBM8HDRrEkiVLGD9+PFu2bGHPnj1Uq1aNmJiYFPdz9/IBNpuN2NjYNG1vJHXeLgsFBwdz8OBBpk+fjo+PD6+++ipNmzbl1q1b5M+fn127drFgwQKCgoIYOXIkNWrUyJELpyrcZFCHDrB4MZQokbC9ZEmzPStG+4uI5Ha5qed769atdO/enaeeeopq1aoRGBjI0aNHs7UGf39/AgIC2Llzp6PNbreza9euNO2nUqVKbN26NUHb1q1bqVy5suO5j48Pbdq04YMPPmDz5s1s376dffv2AeDh4UHLli159913+fXXXzl69Cjff/99Bj5Z1tCl4JmgQwfzcm/NUCwikjpxPd8dO5pBJn4HRU7r+S5fvjwRERG0adMGm83GiBEjUuyBySp9+/ZlwoQJ3HfffVSsWJEPP/yQixcvpmnJgjfeeIPOnTtTq1YtWrZsyYoVK4iIiHBc/TVv3jzsdjsNGjTA19eX+fPn4+PjQ0hICCtXruSvv/6iadOmFCxYkO+++47Y2FgqVKiQVR853RRuMom7OzRvbnUVIiK5R1zPd1Lz3EyZknN6vidPnswLL7xAo0aNKFKkCIMHDyY6Ojrb6xg8eDCRkZF069YNd3d3evbsSatWrXBPQwJs3749U6dO5b333qN///6UKVOGuXPn0vz/v8AKFCjA22+/zcCBA7Hb7VSrVo0VK1ZQuHBhChQoQEREBOHh4dy4cYPy5cuzYMECqlSpkkWfOP1sRnaf0LNYdHQ0/v7+REVF4efnZ3U5IiK50o0bNzhy5AhlypTB29s7Q/vS2nzpExsbS6VKlejcuTNjx461upxMkdLfVVq+v9VzIyIillLPd+ocO3aMdevW0axZM27evMm0adM4cuQIzz77rNWl5TgaUCwiIpILuLm5MW/ePOrVq0fjxo3Zt28fGzZsoFKlSlaXluOo50ZERCQXCA4OTnSlkyRNPTciIiLiVBRuRERExKko3IiIiIhTUbgRERERp6JwIyIiIk5F4UZEREScisKNiIhIGjVv3pwBAwY4npcuXZopU6ak+B6bzcbSpUszfOzM2k9KwsPDqVmzZpYeIysp3IiIiMto06YNrVu3TvK1LVu2YLPZ+PXXX9O83507d9KzZ8+MlpdAcgHj9OnTPPbYY5l6LGejcCMiIi7jxRdfZP369fwdf6XO/zd37lzq1q1L9erV07zfokWL4uvrmxkl3lNgYCBeXl7ZcqzcSuFGRERcxpNPPknRokWZN29egvYrV66waNEiXnzxRf755x+eeeYZSpQoga+vL9WqVWPBggUp7vfu01J//vknTZs2xdvbm8qVK7N+/fpE7xk8eDD3338/vr6+lC1blhEjRnDr1i0A5s2bx+jRo9m7dy82mw2bzeao+e7TUvv27ePhhx/Gx8eHwoUL07NnT65cueJ4vXv37rRv35733nuPoKAgChcuTO/evR3HSo3Y2FjGjBlDyZIl8fLyombNmqxZs8bxekxMDH369CEoKAhvb29CQkKYMGECAIZhEB4eTqlSpfDy8qJ48eL069cv1cdODy2/ICIimcIw4No1a47t6ws227238/DwoFu3bsybN49hw4Zh+/83LVq0CLvdzjPPPMOVK1eoU6cOgwcPxs/Pj1WrVvH8889Trlw56tevf89jxMbG0qFDBwICAvj555+JiopKMD4nTv78+Zk3bx7Fixdn3759vPzyy+TPn59///vfhIaG8ttvv7FmzRo2bNgAgL+/f6J9XL16lVatWtGwYUN27tzJ2bNneemll+jTp0+CALdp0yaCgoLYtGkThw8fJjQ0lJo1a/Lyyy/f+4cGTJ06lUmTJjFr1ixq1arFp59+Stu2bdm/fz/ly5fngw8+YPny5XzzzTeUKlWKEydOcOLECQC+/fZb3n//fb7++muqVKlCZGQke/fuTdVx081wMVFRUQZgREVFWV2KiEiudf36deP33383rl+/7mi7csUwzIiT/bcrV1Jf+4EDBwzA2LRpk6OtSZMmxnPPPZfse5544gnj9ddfdzxv1qyZ0b9/f8fzkJAQ4/333zcMwzDWrl1reHh4GCdPnnS8vnr1agMwlixZkuwxJk6caNSpU8fxfNSoUUaNGjUSbRd/P7NnzzYKFixoXIn3A1i1apXh5uZmREZGGoZhGGFhYUZISIhx+/ZtxzadOnUyQkNDk63l7mMXL17cGDduXIJt6tWrZ7z66quGYRhG3759jYcfftiIjY1NtK9JkyYZ999/vxETE5Ps8eIk9XcVJy3f3zotJSIiLqVixYo0atSITz/9FIDDhw+zZcsWXnzxRQDsdjtjx46lWrVqFCpUiHz58rF27VqOHz+eqv0fOHCA4OBgihcv7mhr2LBhou0WLlxI48aNCQwMJF++fAwfPjzVx4h/rBo1apA3b15HW+PGjYmNjeXgwYOOtipVquDu7u54HhQUxNmzZ1N1jOjoaE6dOkXjxo0TtDdu3JgDBw4A5qmvPXv2UKFCBfr168e6desc23Xq1Inr169TtmxZXn75ZZYsWcLt27fT9DnTSuFGREQyha8vXLlizS2tY3lffPFFvv32Wy5fvszcuXMpV64czZo1A2DixIlMnTqVwYMHs2nTJvbs2UOrVq2IiYnJtJ/V9u3b6dq1K48//jgrV65k9+7dDBs2LFOPEV+ePHkSPLfZbMTGxmba/mvXrs2RI0cYO3Ys169fp3PnznTs2BEwVzM/ePAg06dPx8fHh1dffZWmTZumacxPWmnMjYiIZAqbDeJ1IORonTt3pn///nz11Vd8/vnn9OrVyzH+ZuvWrbRr147nnnsOMMfQHDp0iMqVK6dq35UqVeLEiROcPn2aoKAgAH766acE22zbto2QkBCGDRvmaDt27FiCbTw9PbHb7fc81rx587h69aqj92br1q24ublRoUKFVNV7L35+fhQvXpytW7c6AmDcceKPQfLz8yM0NJTQ0FA6duxI69atuXDhAoUKFcLHx4c2bdrQpk0bevfuTcWKFdm3bx+1a9fOlBrvpnAjIiIuJ1++fISGhjJ06FCio6Pp3r2747Xy5cuzePFitm3bRsGCBZk8eTJnzpxJdbhp2bIl999/P2FhYUycOJHo6OgEISbuGMePH+frr7+mXr16rFq1iiVLliTYpnTp0hw5coQ9e/ZQsmRJ8ufPn+gS8K5duzJq1CjCwsIIDw/n3Llz9O3bl+eff56AgID0/XCS8MYbbzBq1CjKlStHzZo1mTt3Lnv27OHLL78EYPLkyQQFBVGrVi3c3NxYtGgRgYGBFChQgHnz5mG322nQoAG+vr7Mnz8fHx8fQkJCMq2+u+m0lIiIuKQXX3yRixcv0qpVqwTjY4YPH07t2rVp1aoVzZs3JzAwkPbt26d6v25ubixZsoTr169Tv359XnrpJcaNG5dgm7Zt2/Laa6/Rp08fatasybZt2xgxYkSCbZ5++mlat27NQw89RNGiRZO8HN3X15e1a9dy4cIF6tWrR8eOHWnRogXTpk1L2w/jHvr168fAgQN5/fXXqVatGmvWrGH58uWUL18eMK/8evfdd6lbty716tXj6NGjfPfdd7i5uVGgQAHmzJlD48aNqV69Ohs2bGDFihUULlw4U2uMz2YYhpFle8+BoqOj8ff3JyoqCj8/P6vLERHJlW7cuMGRI0coU6YM3t7eVpcjTiKlv6u0fH+r50ZEREScisKNiIiIOBWFGxEREXEqCjciIiLiVBRuREQk3VzsmhTJYpn196RwIyIiaRY34+01q1bKFKcUN0Nz/KUi0kOT+GWimBiIjoYiRayuREQka7m7u1OgQAHH+kS+vr6OGX5F0iM2NpZz587h6+uLh0fG4onCTSbZvBleeAFq14bFi62uRkQk6wUGBgKkegFGkXtxc3OjVKlSGQ7KCjeZpGhROHoUjhyBnTuhXj2rKxIRyVo2m42goCCKFSuWpYsgiuvw9PTEzS3jI2YUbjJJlSrQrRt89hkMHQobNlhdkYhI9nB3d8/wGAmRzKQBxZlo9Gjw9ISNG2H9equrERERcU0KN5koJARefdV8PHQoxMZaW4+IiIgrUrjJZG++CfnywS+/aGCxiIiIFRRuMlnRojBokPl4+HDQGDsREZHspXCTBQYONEPOn3/C3LlWVyMiIuJaFG6yQP78Zq8NQHg4aAJPERGR7KNwk0X+9S8oXRpOn4YPP7S6GhEREdehcJNFvLxgzBjz8dtvw8WL1tYjIiLiKiwPNx999BGlS5fG29ubBg0asGPHjhS3v3TpEr179yYoKAgvLy/uv/9+vvvuu2yqNm2efRaqVoVLl+Cdd6yuRkRExDVYGm4WLlzIwIEDGTVqFLt27aJGjRq0atUq2XVKYmJieOSRRzh69CiLFy/m4MGDzJkzhxIlSmRz5anj7g7jx5uPp06FkyetrUdERMQV2AzDMKw6eIMGDahXrx7Tpk0DzBVBg4OD6du3L0OGDEm0/cyZM5k4cSJ//PEHefLkSdcxo6Oj8ff3JyoqCj8/vwzVnxqGAU2awNat5jicmTOz/JAiIiJOJy3f35b13MTExPDLL7/QsmXLO8W4udGyZUu2b9+e5HuWL19Ow4YN6d27NwEBAVStWpXx48djt9uTPc7NmzeJjo5OcMtONps55gbg44/h0KFsPbyIiIjLsSzcnD9/HrvdTkBAQIL2gIAAIiMjk3zPX3/9xeLFi7Hb7Xz33XeMGDGCSZMm8dZbbyV7nAkTJuDv7++4BQcHZ+rnSI0HH4QnnwS7HUaMyPbDi4iIuBTLBxSnRWxsLMWKFWP27NnUqVOH0NBQhg0bxswUzvUMHTqUqKgox+3EiRPZWPEd48aZvTjffGMuzSAiIiJZw7JwU6RIEdzd3Tlz5kyC9jNnzhAYGJjke4KCgrj//vtxd3d3tFWqVInIyEhiYmKSfI+Xlxd+fn4JblaoXh26djUfDx1qSQkiIiIuwbJw4+npSZ06ddi4caOjLTY2lo0bN9KwYcMk39O4cWMOHz5MbLzltg8dOkRQUBCenp5ZXnNGjRkDefLA+vUQ72OLiIhIJrL0tNTAgQOZM2cOn332GQcOHKBXr15cvXqVHj16ANCtWzeGxuvm6NWrFxcuXKB///4cOnSIVatWMX78eHr37m3VR0iTMmXglVfMx0OHmldSiYiISObysPLgoaGhnDt3jpEjRxIZGUnNmjVZs2aNY5Dx8ePHcXO7k7+Cg4NZu3Ytr732GtWrV6dEiRL079+fwYMHW/UR0mzYMPj0U9i5EyIi4Omnra5IRETEuVg6z40Vsnuem6SMGmWeoqpQAX77DTwsjZgiIiI5X66Y58aVvf46FC4MBw/CvHlWVyMiIuJcFG4s4Odnnp4CCA+H69ctLUdERMSpKNxYpFcvKFXKXG/qo4+srkZERMR5KNxYxNsbRo82H48fb64cLiIiIhmncGOh55+HypXh4kWYONHqakRERJyDwo2F3N3NXhuA99+H06etrUdERMQZKNxYrG1beOABc1Dx2LFWVyMiIpL7KdxYzGaDt982H8+ZA4cPW1uPiIhIbqdwkwM0awaPPQa3b8OIEVZXIyIikrsp3OQQcWNvvv4adu+2thYREZHcTOEmh6hZE5591nz85puWliIiIpKrKdzkIGPGmOtMrVkDmzdbXY2IiEjupHCTg5QrBz17mo+HDAHXWtJUREQkcyjc5DAjRoCvL/z8MyxbZnU1IiIiuY/CTQ4TGAivvWY+fvNN8woqERERST2FmxzojTegUCE4cAC++MLqakRERHIXhZscyN//zhVTo0bBjRvW1iMiIpKbKNzkUK++CiVLwokTMH261dWIiIjkHgo3OZSPD4SHm4/Hj4eoKEvLERERyTUUbnKwsDCoWBH++Qfee8/qakRERHIHhZsczMMDxo0zH0+eDGfOWFuPiIhIbqBwk8M99RTUrw/XrsFbb1ldjYiISM6ncJPD2Wzw9tvm41mz4K+/rK1HREQkp1O4yQUeeggefRRu3YKRI62uRkREJGdTuMklJkww77/6CvbutbYWERGRnEzhJpeoXRtCQ83FNOMm+BMREZHEFG5ykbFjzSuovvsOfvgha45ht8PmzbBggXlvt2fNcURERLKKwk0uUr48vPSS+XjIELMXJzNFREDp0uYYn2efNe9LlzbbRUREcguFm1xmxAhz9uLt22HFiszbb0QEdOwIf/+dsP3kSbNdAUdERHILhZtcpnhx6N/ffPzmm5lz2shuN/eZVE9QXNuAATpFJSIiuYPCTS40eDAULAj798P8+Rnf35YtiXts4jMMcwHPLVsyfiwREZGspnCTCxUoYI65AXPem5s3M7a/06czdzsRERErKdzkUn36mKeojh+HmTMztq+goMzdTkRExEoKN7mUry+Eh5uP33oLoqPTv68mTaBkSXOph6TYbBAcbG4nIiKS0ync5GI9esD998P58+aq4enl7g5Tp5qP7w44cc+nTDG3ExERyekUbnIxD487K4VPmgRnz6Z/Xx06wOLFUKJEwvaSJc32Dh3Sv28REZHsZDOMzJ4KLmeLjo7G39+fqKgo/Pz8rC4nwwwD6tWDX36Bfv3u9MCkl91uXhV1+rQ5xqZJE/XYiIiI9dLy/a1w4wQ2bIBHHoE8eeDQIXNWYREREWeSlu9vnZZyAi1bQosWcOuWeWm4iIiIK1O4cRITJpj38+fDvn3W1iIiImIlhRsnUa+euQaUYcCwYVZXIyIiYh2FGyfy1lvm4N8VK+DHH62uRkRExBoKN06kQgV44QXz8ZAhSS+EKSIi4uwUbpzMqFHg7Q1bt8J331ldjYiISPZTuHEyJUqY890ADB1qzlsjIiLiShRunNDgweDvb141tWCB1dWIiIhkL4UbJ1SokBlwAEaMgJgYa+sRERHJTgo3Tqp/f3P5hKNHYdYsq6sRERHJPgo3TsrX985sxWPHwuXL1tYjIiKSXRRunNiLL8J998G5c/D++1ZXIyIikj0UbpxYnjzmxH4A771nhhwRERFnp3Dj5Dp1glq1zNNScetPiYiIODOFGyfn5nYn1Hz0ERw7Zm09IiIiWU3hxgU8+ig89JB5SXh4uNXViIiIZC2FGxdgs93pvfn8c9i/39p6REREspLCjYto0AA6dIDYWBg2zOpqREREso7CjQt56y1zDM6yZbB9u9XViIiIZA2FGxdSqRJ0724+HjIEDMPSckRERLKEwo2LCQ8HLy/44QdYs8bqakRERDKfwo2LCQ6GPn3Mx0OHmmNwREREnInCjQsaOhT8/GDvXvj6a6urERERyVwKNy6ocGH497/NxyNGmPPfiIiIOAuFGxfVvz8EBMBff8HHH1tdjYiISOZRuHFR+fKZvTYAY8bAlSvW1iMiIpJZFG5c2MsvQ9mycOYMTJ1qdTUiIiKZQ+HGhXl6wtix5uN334V//rG2HhERkcygcOPiunSBGjUgOvrO+lMiIiK5mcKNi3NzuxNqpk2DEyesrUdERCSjckS4+eijjyhdujTe3t40aNCAHTt2JLvtvHnzsNlsCW7e3t7ZWK3zad0amjaFmzdh9GirqxEREckYy8PNwoULGThwIKNGjWLXrl3UqFGDVq1acfbs2WTf4+fnx+nTpx23Y8eOZWPFzsdmg7ffNh/PnQsHDlhbj4iISEZYHm4mT57Myy+/TI8ePahcuTIzZ87E19eXTz/9NNn32Gw2AgMDHbeAgIBsrNg5NWwI7dqZyzEMH251NSIiIulnabiJiYnhl19+oWXLlo42Nzc3WrZsyfbt25N935UrVwgJCSE4OJh27dqxf//+ZLe9efMm0dHRCW6StHHjzDE4ERHw889WVyMiIpI+loab8+fPY7fbE/W8BAQEEBkZmeR7KlSowKeffsqyZcuYP38+sbGxNGrUiL///jvJ7SdMmIC/v7/jFhwcnOmfw1lUqQLdupmP33gD7HZr6xEREUkPy09LpVXDhg3p1q0bNWvWpFmzZkRERFC0aFFmzZqV5PZDhw4lKirKcTuhy4FSFB4OPj6wZYsZcERERHIbS8NNkSJFcHd358yZMwnaz5w5Q2BgYKr2kSdPHmrVqsXhw4eTfN3Lyws/P78EN0leSAjMm2c+fv99SCYzioiI5FiWhhtPT0/q1KnDxo0bHW2xsbFs3LiRhg0bpmofdrudffv2ERQUlFVlupzOne/MXNy7N2zYYG09IiIiaWH5aamBAwcyZ84cPvvsMw4cOECvXr24evUqPXr0AKBbt24MHTrUsf2YMWNYt24df/31F7t27eK5557j2LFjvPTSS1Z9BKc0bBh07WqOu+nYEf74w+qKREREUsfD6gJCQ0M5d+4cI0eOJDIykpo1a7JmzRrHIOPjx4/j5nYng128eJGXX36ZyMhIChYsSJ06ddi2bRuVK1e26iM4JZsNPv4YjhyBbdvgySfhp5+gSBGrKxMREUmZzTAMw+oislN0dDT+/v5ERUVp/E0qnD0LDRrA0aPQpAmsXw9eXlZXJSIiriYt39+Wn5aSnK1YMVi5Evz8zCuo/vUvcK04LCIiuY3CjdxTlSrwzTfmBH+ffQbvvGN1RSIiIslTuJFUadUKPvjAfDx0qDmLsYiISE6kcCOp1rs39OljPn7uOfjlF2vrERERSYrCjaTJ+++bvTjXr0PbtnDypNUViYiIJKRwI2ni4QELF0LlynDqFLRpA1evWl2ViIjIHQo3kmb+/uYVVEWLwu7d5imq2FirqxIRETEp3Ei6lCkDS5eCp6d5/+abVlckIiJiUriRdGvUCD791Hz8zjswd6619YiIiIDCjWRQ164wYoT5uGdP2LzZ0nJEREQUbiTjwsPNlcRv34ann4Y//7S6IhERcWUKN5Jhbm4wbx7Urw8XLpiLbF68aHVVIiLiqhRuJFP4+MCyZRAcDIcOQceOcOuW1VWJiIgrUriRTBMYaF4ini8ffP+9OZuxFtkUEZHspnAjmap6dViwAGw2mD0bpkyxuiIREXE1CjeS6Z58EiZNMh+//jqsWGFtPSIi4loUbiRLDBhgXhpuGPDMM7B3r9UViYiIq1C4kSxhs8G0afDww+baU23awOnTVlclIiKuQOFGskyePLB4MVSoACdOQLt25mriIiIiWSld4ebEiRP8/fffjuc7duxgwIABzJ49O9MKE+dQsKB5BVWhQrBzJ4SFaZFNERHJWukKN88++yybNm0CIDIykkceeYQdO3YwbNgwxowZk6kFSu53330QEWH25CxaZM5oLCIiklXSFW5+++036tevD8A333xD1apV2bZtG19++SXz5s3LzPrESTRrBrNmmY/HjoX5862tR0REnFe6ws2tW7fw8vICYMOGDbRt2xaAihUrclqjRiUZPXrA4MHm4xdfhK1bra1HREScU7rCTZUqVZg5cyZbtmxh/fr1tG7dGoBTp05RuHDhTC1QnMv48dC+PcTEmPd//WV1RSIi4mzSFW7eeecdZs2aRfPmzXnmmWeoUaMGAMuXL3ecrhJJipubeUqqdm04f968RDwqyuqqRETEmdgMI32r/9jtdqKjoylYsKCj7ejRo/j6+lKsWLFMKzCzRUdH4+/vT1RUFH5+flaX47JOnjRXET91Clq1Mq+o8vCwuioREcmp0vL9na6em+vXr3Pz5k1HsDl27BhTpkzh4MGDOTrYSM5RooS5LIOvL6xdC6+9ZnVFIiLiLNIVbtq1a8fnn38OwKVLl2jQoAGTJk2iffv2zJgxI1MLFOdVu/adq6amTTNvIiIiGZWucLNr1y6aNGkCwOLFiwkICODYsWN8/vnnfPDBB5laoDi3p56Ct982H/fvD2vWWFuPiIjkfukKN9euXSN//vwArFu3jg4dOuDm5sYDDzzAsWPHMrVAcX7//jd0727OXNy5M/z2m9UViYhIbpaucHPfffexdOlSTpw4wdq1a3n00UcBOHv2rAbpSprZbOYEf02bwuXL5hVUZ89mbJ92O2zeDAsWmPd2e2ZUKiIiuUG6ws3IkSMZNGgQpUuXpn79+jRs2BAwe3Fq1aqVqQWKa/D0hG+/hXLl4OhR83TVjRvp21dEBJQuDQ89BM8+a96XLm22i4iI80v3peCRkZGcPn2aGjVq4OZmZqQdO3bg5+dHxYoVM7XIzKRLwXO2P/6Ahg3h0iXo2hW++MLs2UmtiAjo2BHu/quO28fixdChQ6aVKyIi2SQt39/pDjdx4lYHL1myZEZ2k20UbnK+jRvNuW/sdnMdquHDU/c+u93soYm3YH0CNhuULAlHjoC7e6aVKyIi2SDL57mJjY1lzJgx+Pv7ExISQkhICAUKFGDs2LHExsamq2iROC1awPTp5uMRI+Cbb1L3vi1bkg82YPbmnDhhbiciIs4rXXPCDhs2jE8++YS3336bxo0bA/Djjz8SHh7OjRs3GDduXKYWKa6nZ0/zFNX770NYGISEQIMGKb8ntWu2am1XERHnlq5w89lnn/Hxxx87VgMHqF69OiVKlODVV19VuJFMMXEiHDoEq1ZBu3awYweUKpX89kFBqdtvarcTEZHcKV2npS5cuJDkoOGKFSty4cKFDBclAua4mAULoHp1OHPGvET88uXkt2/SxBxTk9wAZJsNgoPN7URExHmlK9zUqFGDaUnMlT9t2jSqV6+e4aJE4uTPb65BFRAAv/5qXtqd3Jw17u4wdar5+O6AE/d8yhQNJhYRcXbpulrqP//5D0888QSlSpVyzHGzfft2Tpw4wXfffedYmiEn0tVSudPPP0Pz5ubcNwMHwqRJyW8bEWEu5RB/cHFwsBlsdBm4iEjulOVXSzVr1oxDhw7x1FNPcenSJS5dukSHDh3Yv38/X3zxRbqKFklJgwbw2Wfm48mTYfbs5Lft0MGcCHDTJvjqK/P+yBEFGxERV5HheW7i27t3L7Vr18aeg+e6V89N7jZ2LIwcCR4e5iKbLVpYXZGIiGSHLO+5EbHK8OHmuJvbt+Hpp83LxUVEROJTuJFcxWaDTz6BRo0gKgqefBL++cfqqkREJCdRuJFcx9sbliwxl1r43//MsTQxMVZXJSIiOUWaJvHrcI8RmZcuXcpILSKpVqwYrFxpLrL5ww/wyitmj05aFtkUERHnlKZw4+/vf8/Xu3XrlqGCRFKrShVz3aknnoC5c6FiRfj3v62uSkRErJapV0vlBrpayvlMmwZ9+5q9Nt9+C089ZXVFIiKS2XS1lLiUPn2gd29z1e/nnoNdu6yuSERErKRwI05hyhR49FG4ds1cg+rkSasrEhERqyjciFPw8DDH31SuDKdOQdu2cPWq1VWJiIgVFG7Eafj7m1dQFSlinpp6/nmIjbW6KhERyW4KN+JUypSBpUvB09OcC+ff/zbH4oiIiOtQuBGn07ixOecNmKuHt25tnqoSERHXoHAjTum552DmTHM243XroGpVWLTI6qpERCQ7KNyI0/rXv8yxN7Vrw8WL0LmzOQ5HE2mLiDg3hRtxapUqwfbtMGwYuLnB/PlQvTps3mx1ZSIiklUUbsTpeXrCW2/Bli1QtiycOAEPPwyDBsGNG1ZXJyIimU3hRlxGo0awdy+89JJ5BdWkSVC/Pvz6q9WViYhIZlK4EZeSLx/MmQPLlkHRorBvH9SrBxMngt1udXUiIpIZFG7EJbVtC7/9Zt7HxJjz4Tz8MBw7ZnVlIiKSUQo34rKKFTMn/JszB/LmhR9+MAcbf/65Jv4TEcnNFG7Epdls5hicvXuhYUOIjoawMOjUCc6ft7o6ERFJD4UbEaBcObPn5q23zEU4v/0WqlWDNWusrkxERNJK4Ubk/3l4mPPh/PQTVKwIkZHw2GPQuzdcu2Z1dSIikloKNyJ3qVPHnNm4b1/z+fTpUKsW7NxpbV0iIpI6CjciSfDxgQ8+gLVroXhxOHTIHJMzZgzcvm11dSIikhKFG5EUPPqoORdO587mPDijRsGDD8Kff1pdmYiIJCdHhJuPPvqI0qVL4+3tTYMGDdixY0eq3vf1119js9lo37591hYoLq1QIfj6a/jyS/D3h59/hpo1YdYsXTIuIpITWR5uFi5cyMCBAxk1ahS7du2iRo0atGrVirNnz6b4vqNHjzJo0CCaNGmSTZWKK7PZ4NlnzaUaHnrIHGD8yivQpo058FhERHIOy8PN5MmTefnll+nRoweVK1dm5syZ+Pr68umnnyb7HrvdTteuXRk9ejRly5bNxmrF1ZUqBRs2wOTJ4OUFq1aZl4wvXWp1ZSIiEsfScBMTE8Mvv/xCy5YtHW1ubm60bNmS7du3J/u+MWPGUKxYMV588cV7HuPmzZtER0cnuIlkhJsbvPYa/Pe/UKOGOdnfU0/BCy/A5ctWVyciIpaGm/Pnz2O32wkICEjQHhAQQGQyff0//vgjn3zyCXPmzEnVMSZMmIC/v7/jFhwcnOG6RQCqVjXH3/z73+Zpq7lzzbDz449WVyYi4tosPy2VFpcvX+b5559nzpw5FClSJFXvGTp0KFFRUY7biRMnsrhKcSVeXvDOO7B5M4SEwJEj0LQpDB1qLsgpIiLZz8PKgxcpUgR3d3fOnDmToP3MmTMEBgYm2v5///sfR48epU2bNo622NhYADw8PDh48CDlypVL8B4vLy+8vLyyoHqRO5o2NQcb9+8P8+bB22+bSzfMnw9VqlhdnYiIa7G058bT05M6deqwceNGR1tsbCwbN26kYcOGibavWLEi+/btY8+ePY5b27Zteeihh9izZ49OOYml/PzMU1PffguFC8OePeZsx1OmwP9ncBERyQaW9twADBw4kLCwMOrWrUv9+vWZMmUKV69epUePHgB069aNEiVKMGHCBLy9valatWqC9xcoUAAgUbuIVTp0MGczfvFFWL3aHHy8cqXZo1OypNXViYg4P8vDTWhoKOfOnWPkyJFERkZSs2ZN1qxZ4xhkfPz4cdzcctXQIBGCgszLxGfNgoEDYeNG85Lx6dPhmWesrk5ExLnZDMO15liNjo7G39+fqKgo/Pz8rC5HXMChQ/Dcc3cW3uzSxQw5BQtaW5eISG6Slu9vdYmIZLH774etWyE8HNzdzaUcqlUzJwMUEZHMp3Ajkg3y5DEX3dy6FcqXh5Mn4ZFHYMAAuH49+ffZ7eZl5gsWmPd2ezYVLCKSiynciGSjBg1g927o1ct8PnUq1K1rtt0tIgJKlzbXsnr2WfO+dGmzXUREkqdwI5LN8uY1x9ysWgUBAfD772bomTDhTs9MRAR07Ah//53wvSdPmu0KOCIiyVO4EbHI44/Db7+Z61LdugVvvgnNmsGff5qTASY11D+ubcAAnaISEUmOwo2IhYoUMSf9mzcP8uc3x+TUqJG4xyY+w4ATJ2DLlmwrU0QkV1G4EbGYzQZhYbB3LzRpkvIA4/hOn87aukREciuFG5EcokwZ2LQJevZM3fZBQVlbj4hIbqVwI5KDuLubg42LFUt+G5sNgoPNXh4REUlM4UYkh3F3hxkzkn/dMMzFON3ds60kEZFcReFGJAfq0MEcaJzcQpsTJ8KiRXD7dvbWJSKSG2htKZEczG43r4o6fRpu3IAff4Qvv4SbN83XQ0LMy8ZffBH05ywiziwt398KNyK5zJkz5mmr6dPh3DmzLX9+ePll6NfPDDwiIs5GC2eKOLGAAHMRzmPHYM4cqFQJLl+GyZOhbFkIDYWff7a6ShER6yjciORSPj7w0kuwfz+sXm0uxBkbC998Aw88AI0bm+N2NJOxiLgahRuRXM5mg9atYd06+PVX6NEDPD1h2zZzHary5c0FOi9ftrpSEZHsoXAj4kSqVYNPPzVPWY0YAYULw5Ej5lpUJUvCG2/A8eNWVykikrUUbkScUGAgjBljrkE1axZUrAjR0fDee+a4nGeegZ07ra5SRCRrKNyIODEfH3M5h/37YdUqaNHCHIPz9ddQv745y/GSJRqXIyLOReFGxAW4ucHjj8OGDbBnj7lQZ5485rw5HTrA/ffDhx/ClStWVyoiknEKNyIupkYNmDfPHJczbBgUKgR//WXOkRMcDIMHw99/W12liEj6KdyIuKigIHjrLXNczowZZu/NpUvw7rvmCuVdu8Ivv1hdpYhI2inciLg4X1945RU4cABWrICHHjLXrPrqK6hbF5o3h+XLzTl0RERyA4UbEQHMcTlPPgnffw+7dsHzz4OHB/znP9CuHVSoYC75cPWq1ZWKiKRM4UZEEqlVCz7/HI4ehaFDoWBBOHwYevc2x+W8+SacOmV1lSIiSVO4EZFklSgB48eb43KmTYP77oOLF2HCBChdGrp1g927ra5SRCQhhRsRuae8ec1emz/+gKVLoWlTuHULvvgCateGhx+GlSs1LkdEcgaFGxFJNXd3c/zNf/5jznD87LPmuJxNm6BNG3OF8pkz4do1qysVEVemcCMi6VK3Lnz5pTlHzr//Df7+cOgQ9OpljssZPhxOn7a6ShFxRQo3IpIhwcHwzjvmxH8ffGCuXXXhAowbByEh0L077N1rdZUi4koUbkQkU+TLB337mr03ERHw4IPmuJzPPoOaNaFlS3MenZs3ra5URJydwo2IZCp3d3jqKdiyBX7+Gbp0Mds2boS2baFoUQgNNScJvHTJ6mpFxBnZDMMwrC4iO0VHR+Pv709UVBR+fn5WlyPiEo4fNxfmnD8fIiPvtHt4QLNm5iDltm3N01giIklJy/e3wo2IZJvYWPMqq2XLzNvvvyd8vWZNM+i0a2c+ttmsqFJEciKFmxQo3IhkP7vdPE11+rS5YGeTJuapqsOH7wSdrVsTzpNTqpTZm9Oundm7kyePdfWLiPUUblKgcCOSvSIioH9/82qqOCVLwtSp0KHDnbbz582JAJctg3XrEs6V4+8Pjz9uBp3HHgP9pyviehRuUqBwI5J9IiKgY0e4+/8ycaebFi9OGHDiXL8OGzaYQWfFCjh79s5refKYK5fHjdMpWTLr6heRnEPhJgUKNyLZw24315+K32MTn81mBpMjR8xTVCnt5+ef75y+Ongw4et16twZp1OtmsbpiDgrhZsUKNyIZI/Nm80elnvZtAmaN0/9fg8evBN0tm9P2CtUuvSdoNOkiXk1log4h7R8f2ueGxHJEqldeiGtSzRUqGAu97B1q/nejz8217Xy9oajR82xPA8/DMWKwfPPm6e+Ll9Oc/kikosp3IhIlggKytztkhIQAC++CMuXmwOSlywxl3soUgQuXjTn1enUyXz++OMwa5bWuxJxBTotJSJZIm7MzcmTiQcUQ+rH3KT32Nu23Tl9dfhwwtfr179z+qpyZY3TEckNNOYmBQo3Itkn7mopSBhw7nW1VGYyDDhw4E7Q+fnnhK+XK3cn6DRqpHE6IjmVwk0KFG5EsldS89wEB8OUKVkfbJJy+rR5efmyZeZ6V/EX8ixcGJ580gw6jz4KefNmf30ikjSFmxQo3Ihkv+RmKLbalSuwdq0ZdFatggsX7rzm7W2uZN6unTlgOSDAujpFROEmRQo3IpKU27fhxx/vnL46cuTOazYbPPDAndNXFStaV6eIq1K4SYHCjYjci2HAb7/dCTr//W/C1++7Dx580Aw8DzwAVaporI5IVlO4SYHCjYik1cmT5uXmy5bB99/DrVsJX8+bF+rVuxN2GjSAwEBrahVxVgo3KVC4EZGMiI6GH34wr7r66SfzPqlJAkNC7oSdBx6AWrXAyyv76xVxFgo3KVC4EZHMZLfDH3+YQSfutn9/4rl9PD3NgBM/8ISEaI4dkdRSuEmBwo2IZLXoaHOcTvzAc+5c4u0CAhKeyqpXD/Lly/56RXIDhZsUKNyISHYzDPPqq/hhZ8+exGN33NygatWEvTsVKpjtIq5O4SYFCjcikhPcuAG7dycMPMePJ97O39/s1YkLO/Xrm5MNirgahZsUKNyISE516tSdgco//QQ7d8L164m3K18+Ye9OtWqQJ0/21yuSnRRuUqBwIyK5xe3bsG/fnbDz889w8GDi7Xx8oG7dhIGnePHsr1ckKyncpEDhRkRyswsXYMeOhIHn0qXE2wUHJzydVbu2GYJEciuFmxQo3IiIM4mNhUOHEo7d2bfPbI/PwwNq1kzYu1O2rC5Fl9xD4SYFCjcikl45dQHQu125Yl6KHjd+Z/t2OHMm8XaFCpnjd8qUgdKlzfu4W6lS5tw8IjmFwk0KFG5EJD0iIqB/f/j77zttJUvC1KnQoYN1daWGYZhXYsXv3dm1C2Jikn+PmxuUKJEw8MS/FS+uS9QleyncpEDhRkTSKiICOnZMPOtw3CmdxYtzfsC5282b8Pvv5vw7d9+OHk36Kq34PD3NGZaTCz+FC+uUl2QuhZsUKNyISFrY7eYpm/g9NvHZbGYPzpEjOfMUVXoYhnkaK6ngc+SI2Qtkt6e8j3z5kg8+ZcpoJmZJO4WbFCjciEhabN4MDz107+02bYLmzbO6mpzh9m0z7CUXfk6fvvc+ihRJPviUKqVFRiWxtHx/e2RTTSIiuVJqvqjTsp0z8PAwe7NKl046+F2/DseOJTzNFT/8XLgA58+bt507E7/fZrv3eB9n6SWTrKFwIyKSgqCgzN3OFfj4QMWK5i0pUVGJA0/827VrZs/Q33+bV6fdLU8es3cnfuAJCoJixcxbQIB5r94f16XTUiIiKYgbc3PyZOIBxeCcY26sZBjmCurJBZ9jx8zTYqnh75848MTd393m768B0DmdTkuJiGQSd3fzcu+OHc0vv/gBJ+7LcMoUBZvMYrPdCR8NGiR+3W43g+bdgefMGTh79s79rVtmD1FUFPz5572PmydP6kJQsWJQtKjW8srp1HMjIpIKSc1zExxsBpvcdhm4szMMc0mKs2cTBp64+7vboqPTfoxChe4dguIe58unXqHMkOuulvroo4+YOHEikZGR1KhRgw8//JD69esnuW1ERATjx4/n8OHD3Lp1i/Lly/P666/z/PPPp+pYCjcikl65ZYZiSZsbN+6EnnuFoXPn7n0Z/N28vVMXgooVM68i099U0nJVuFm4cCHdunVj5syZNGjQgClTprBo0SIOHjxIsWLFEm2/efNmLl68SMWKFfH09GTlypW8/vrrrFq1ilatWt3zeAo3IiKSXrGx5tVeqekROnsWrl5N2/7d3SEw0LwiLCjIvE/qcdGirjdDdK4KNw0aNKBevXpMmzYNgNjYWIKDg+nbty9DhgxJ1T5q167NE088wdixY++5rcKNiIhkl6tXEwefpELQmTPwzz9JD1pPioeHGYLih56kQlCRIs4TgnLNgOKYmBh++eUXhg4d6mhzc3OjZcuWbN++/Z7vNwyD77//noMHD/LOO+9kZakiIiJpljfvncvV7+X2bTPonDpl3k6fvvM4/vOzZ+9MpJjczNlxPDzMkHOvEFS4sPOEILA43Jw/fx673U5AQECC9oCAAP74449k3xcVFUWJEiW4efMm7u7uTJ8+nUceeSTJbW/evMnNmzcdz6PTM3JMREQki3l43AkcKbl1K/kQFP/xuXNmCDpxwrylJE+exCEoqUCUW9YMy5WXgufPn589e/Zw5coVNm7cyMCBAylbtizNk5j7fMKECYwePTr7ixQREckCefKYMziXKJHydrdumae7UgpAp0/fuXT++HHzlhJPz5RDUNzjQoWsDUGWjrmJiYnB19eXxYsX0759e0d7WFgYly5dYtmyZanaz0svvcSJEydYu3ZtoteS6rkJDg7WmBsREREgJiZ1IejcudTvs2ZN2L07c+vMNWNuPD09qVOnDhs3bnSEm9jYWDZu3EifPn1SvZ/Y2NgEASY+Ly8vvDQHt4iISJI8Pc05m4KDU94uJgYiI1MOQKdOmWuGFS2aPbUnx/LTUgMHDiQsLIy6detSv359pkyZwtWrV+nRowcA3bp1o0SJEkyYMAEwTzPVrVuXcuXKcfPmTb777ju++OILZsyYYeXHEBERcWqenuaaXqVKpbxdTAxcuZI9NSXH8nATGhrKuXPnGDlyJJGRkdSsWZM1a9Y4BhkfP34ct3hDuK9evcqrr77K33//jY+PDxUrVmT+/PmEhoZa9RFERHIVTUYoWcnT0xxzYyXL57nJbprnRkRcWVLLSJQsaa6fpWUkJCdLy/e3E13VLiIiKYmIMBcAvXtulJMnzfaICGvqEslsCjciIi7Abjd7bJLqq49rGzAg7esmieRECjciIi5gy5aUZ7M1DHOity1bsq8mkayicCMi4gJOn87c7URyMoUbEREXEBSUuduJ5GQKNyIiLqBJE/OqqOSmxLfZzEncmjTJ3rpEsoLCjYiIC3B3Ny/3hsQBJ+75lCma70acg8KNiIiL6NABFi9OvOBiyZJmu+a5EWdh+QzFIiKSfTp0gHbtNEOxODeFGxERF+PuDs2bW12FSNbRaSkRERFxKgo3IiIi4lQUbkRERMSpKNyIiIiIU9GAYhERybXsdl35JYkp3IiISK4UEWGudB5/QdCSJc3JCjVnj2vTaSkREcl1IiKgY8fEK52fPGm2R0RYU5fkDAo3IiKSq9jtZo+NYSR+La5twABzO3FNCjciIpKrbNmSuMcmPsOAEyfM7cQ1KdyIiEiucvp05m4nzkfhRkREcpWgoMzdTpyPwo2IiOQqTZqYV0XZbEm/brNBcLC5nbgmhRsREclV3N3Ny70hccCJez5liua7cWUKNyIikut06ACLF0OJEgnbS5Y02zXPjWvTJH4iIpIrdegA7dpphmJJTOFGRERyLXd3aN7c6iokp1G4ERERsZjWyMpcCjciIiIW0hpZmU8DikVERCyiNbKyhsKNiIiIBbRGVtZRuBEREbGA1sjKOgo3IiIiFtAaWVlH4UZERMQCWiMr6yjciIiIWEBrZGUdhRsRERELaI2srKNwIyIiYhGtkZU1NImfiIiIhbRGVuZTuBEREbGYs6yRlVOWkVC4ERERkQzLSctIaMyNiIiIZEhOW0ZC4UZERETSLScuI6FwIyIiIumWE5eRULgRERGRdMuJy0go3IiIiEi65cRlJBRuREREJN1y4jISCjciIiKSbjlxGQmFGxEREcmQnLaMhCbxExERkQzLSctIKNyIiIhIpsgpy0jotJSIiIg4FYUbERERcSoKNyIiIuJUFG5ERETEqSjciIiIiFNRuBERERGnonAjIiIiTkXhRkRERJyKwo2IiIg4FZebodgwDACio6MtrkRERERSK+57O+57PCUuF24uX74MQHBwsMWViIiISFpdvnwZf3//FLexGamJQE4kNjaWU6dOkT9/fmx3r80ugJmOg4ODOXHiBH5+flaX4/L0+8hZ9PvIefQ7yVmy6vdhGAaXL1+mePHiuLmlPKrG5Xpu3NzcKFmypNVl5Ap+fn76H0UOot9HzqLfR86j30nOkhW/j3v12MTRgGIRERFxKgo3IiIi4lQUbiQRLy8vRo0ahZeXl9WlCPp95DT6feQ8+p3kLDnh9+FyA4pFRETEuannRkRERJyKwo2IiIg4FYUbERERcSoKNyIiIuJUFG7EYcKECdSrV4/8+fNTrFgx2rdvz8GDB60uS4C3334bm83GgAEDrC7FpZ08eZLnnnuOwoUL4+PjQ7Vq1fjvf/9rdVkuyW63M2LECMqUKYOPjw/lypVj7NixqVp3SDLuhx9+oE2bNhQvXhybzcbSpUsTvG4YBiNHjiQoKAgfHx9atmzJn3/+mW31KdyIw3/+8x969+7NTz/9xPr167l16xaPPvooV69etbo0l7Zz505mzZpF9erVrS7FpV28eJHGjRuTJ08eVq9eze+//86kSZMoWLCg1aW5pHfeeYcZM2Ywbdo0Dhw4wDvvvMO7777Lhx9+aHVpLuHq1avUqFGDjz76KMnX3333XT744ANmzpzJzz//TN68eWnVqhU3btzIlvp0Kbgk69y5cxQrVoz//Oc/NG3a1OpyXNKVK1eoXbs206dP56233qJmzZpMmTLF6rJc0pAhQ9i6dStbtmyxuhQBnnzySQICAvjkk08cbU8//TQ+Pj7Mnz/fwspcj81mY8mSJbRv3x4we22KFy/O66+/zqBBgwCIiooiICCAefPm0aVLlyyvST03kqyoqCgAChUqZHElrqt379488cQTtGzZ0upSXN7y5cupW7cunTp1olixYtSqVYs5c+ZYXZbLatSoERs3buTQoUMA7N27lx9//JHHHnvM4srkyJEjREZGJvj/lr+/Pw0aNGD79u3ZUoPLLZwpqRMbG8uAAQNo3LgxVatWtbocl/T111+za9cudu7caXUpAvz111/MmDGDgQMH8uabb7Jz50769euHp6cnYWFhVpfncoYMGUJ0dDQVK1bE3d0du93OuHHj6Nq1q9WlubzIyEgAAgICErQHBAQ4XstqCjeSpN69e/Pbb7/x448/Wl2KSzpx4gT9+/dn/fr1eHt7W12OYAb+unXrMn78eABq1arFb7/9xsyZMxVuLPDNN9/w5Zdf8tVXX1GlShX27NnDgAEDKF68uH4fotNSklifPn1YuXIlmzZtomTJklaX45J++eUXzp49S+3atfHw8MDDw4P//Oc/fPDBB3h4eGC3260u0eUEBQVRuXLlBG2VKlXi+PHjFlXk2t544w2GDBlCly5dqFatGs8//zyvvfYaEyZMsLo0lxcYGAjAmTNnErSfOXPG8VpWU7gRB8Mw6NOnD0uWLOH777+nTJkyVpfkslq0aMG+ffvYs2eP41a3bl26du3Knj17cHd3t7pEl9O4ceNEUyMcOnSIkJAQiypybdeuXcPNLeFXmLu7O7GxsRZVJHHKlClDYGAgGzdudLRFR0fz888/07Bhw2ypQaelxKF379589dVXLFu2jPz58zvOjfr7++Pj42Nxda4lf/78icY65c2bl8KFC2sMlEVee+01GjVqxPjx4+ncuTM7duxg9uzZzJ492+rSXFKbNm0YN24cpUqVokqVKuzevZvJkyfzwgsvWF2aS7hy5QqHDx92PD9y5Ah79uyhUKFClCpVigEDBvDWW29Rvnx5ypQpw4gRIyhevLjjiqosZ4j8PyDJ29y5c60uTQzDaNasmdG/f3+ry3BpK1asMKpWrWp4eXkZFStWNGbPnm11SS4rOjra6N+/v1GqVCnD29vbKFu2rDFs2DDj5s2bVpfmEjZt2pTk90VYWJhhGIYRGxtrjBgxwggICDC8vLyMFi1aGAcPHsy2+jTPjYiIiDgVjbkRERERp6JwIyIiIk5F4UZEREScisKNiIiIOBWFGxEREXEqCjciIiLiVBRuRERExKko3IiIS7LZbCxdutTqMkQkCyjciEi26969OzabLdGtdevWVpcmIk5Aa0uJiCVat27N3LlzE7R5eXlZVI2IOBP13IiIJby8vAgMDExwK1iwIGCeMpoxYwaPPfYYPj4+lC1blsWLFyd4/759+3j44Yfx8fGhcOHC9OzZkytXriTY5tNPP6VKlSp4eXkRFBREnz59Erx+/vx5nnrqKXx9fSlfvjzLly93vHbx4kW6du1K0aJF8fHxoXz58onCmIjkTAo3IpIjjRgxgqeffpq9e/fStWtXunTpwoEDBwC4evUqrVq1omDBguzcuZNFixaxYcOGBOFlxowZ9O7dm549e7Jv3z6WL1/Offfdl+AYo0ePpnPnzvz66688/vjjdO3alQsXLjiO//vvv7N69WoOHDjAjBkzKFKkSPb9AEQk/bJtiU4Rkf8XFhZmuLu7G3nz5k1wGzdunGEY5gr1r7zySoL3NGjQwOjVq5dhGIYxe/Zso2DBgsaVK1ccr69atcpwc3MzIiMjDcMwjOLFixvDhg1LtgbAGD58uOP5lStXDMBYvXq1YRiG0aZNG6NHjx6Z84FFJFtpzI2IWOKhhx5ixowZCdoKFSrkeNywYcMErzVs2JA9e/YAcODAAWrUqEHevHkdrzdu3JjY2FgOHjyIzWbj1KlTtGjRIsUaqlev7nicN29e/Pz8OHv2LAC9evXi6aefZteuXTz66KO0b9+eRo0apeuzikj2UrgREUvkzZs30WmizOLj45Oq7fLkyZPguc1mIzY2FoDHHnuMY8eO8d1337F+/XpatGhB7969ee+99zK9XhHJXBpzIyI50k8//ZToeaVKlQCoVKkSe/fu5erVq47Xt27dipubGxUqVCB//vyULl2ajRs3ZqiGokWLEhYWxvz585kyZQqzZ8/O0P5EJHuo50ZELHHz5k0iIyMTtHl4eDgG7S5atIi6devy4IMP8uWXX7Jjxw4++eQTALp27cqoUaMICwsjPDycc+fO0bdvX55//nkCAgIACA8P55VXXqFYsWI89thjXL58ma1bt9K3b99U1Tdy5Ejq1KlDlSpVuHnzJitXrnSEKxHJ2RRuRMQSa9asISgoKEFbhQoV+OOPPwDzSqavv/6aV199laCgIBYsWEDlypUB8PX1Ze3atfTv35969erh6+vL008/zeTJkx37CgsL48aNG7z//vsMGjSIIkWK0LFjx1TX5+npydChQzl69Cg+Pj40adKEr7/+OhM+uYhkNZthGIbVRYiIxGez2ViyZAnt27e3uhQRyYU05kZEREScisKNiIiIOBWNuRGRHEdny0UkI9RzIyIiIk5F4UZEREScisKNiIiIOBWFGxEREXEqCjciIiLiVBRuRERExKko3IiIiIhTUbgRERERp6JwIyIiIk7l/wDiUeFgZaekkgAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABUfUlEQVR4nO3deZyNdf/H8deZGbNhBoNZzDBI9n2LuS2VQmVJNEoMdetO1qQbt52izRYhLbRKaSyVXZRQ3EmppNzZwqAwYx/OXL8/rt8cc8wYs5yZa+ac9/PxOA/nXOc65/ocM3Xevtf3e31shmEYiIiIiLgJL6sLEBEREXElhRsRERFxKwo3IiIi4lYUbkRERMStKNyIiIiIW1G4EREREbeicCMiIiJuReFGRERE3IrCjYiIiLgVhRsRC/Tu3Zvo6OgcvXb8+PHYbDbXFlTAHDhwAJvNxsKFC/P1uJs2bcJms7Fp0ybHtqz+rPKq5ujoaHr37u3S98yKhQsXYrPZOHDgQL4fWyS3FG5E0rDZbFm6pf3yE8mtrVu3Mn78eM6cOWN1KSJuwcfqAkQKknfffdfp8TvvvMO6devSba9evXqujvP666+TkpKSo9eOHj2aESNG5Or4knW5+Vll1datW5kwYQK9e/emRIkSTs/t3bsXLy/9O1QkOxRuRNJ45JFHnB5/8803rFu3Lt326124cIHAwMAsH6dIkSI5qg/Ax8cHHx/9p5tfcvOzcgU/Pz9Ljy9SGOmfAyLZ1Lp1a2rVqsV3331Hy5YtCQwM5D//+Q8Ay5cv59577yUiIgI/Pz8qV67MpEmTsNvtTu9x/TyO1PkaL7/8MvPnz6dy5cr4+fnRuHFjduzY4fTajObc2Gw2BgwYwLJly6hVqxZ+fn7UrFmT1atXp6t/06ZNNGrUCH9/fypXrsxrr72W5Xk8mzdvplu3bpQvXx4/Pz+ioqJ46qmnuHjxYrrPV6xYMY4cOULnzp0pVqwYZcqUYdiwYen+Ls6cOUPv3r0JDg6mRIkSxMXFZen0zH//+19sNhtvv/12uufWrFmDzWbjs88+A+DgwYM8+eSTVK1alYCAAEJCQujWrVuW5pNkNOcmqzX/+OOP9O7dm0qVKuHv709YWBiPPvoof//9t2Of8ePH88wzzwBQsWJFx6nP1NoymnPzxx9/0K1bN0qVKkVgYCC33XYbn3/+udM+qfOHPvroI5577jkiIyPx9/fnzjvvZN++fTf93DcyZ84catasiZ+fHxEREfTv3z/dZ//999954IEHCAsLw9/fn8jISLp3705iYqJjn3Xr1vGPf/yDEiVKUKxYMapWrer470gkt/TPP5Ec+Pvvv2nfvj3du3fnkUceITQ0FDAnYRYrVoyhQ4dSrFgxvvjiC8aOHUtSUhIvvfTSTd/3gw8+4OzZs/zrX//CZrPx4osv0qVLF/7444+bjiB8/fXXxMfH8+STT1K8eHFeeeUVHnjgAQ4dOkRISAgA33//Pe3atSM8PJwJEyZgt9uZOHEiZcqUydLn/vjjj7lw4QL9+vUjJCSE7du3M2vWLP78808+/vhjp33tdjtt27aladOmvPzyy6xfv56pU6dSuXJl+vXrB4BhGHTq1Imvv/6aJ554gurVq7N06VLi4uJuWkujRo2oVKkSH330Ubr9Fy9eTMmSJWnbti0AO3bsYOvWrXTv3p3IyEgOHDjA3Llzad26Nb/88ku2Rt2yU/O6dev4448/6NOnD2FhYfz888/Mnz+fn3/+mW+++QabzUaXLl347bffWLRoEdOnT6d06dIAN/yZHD9+nObNm3PhwgUGDRpESEgIb7/9Nh07dmTJkiXcf//9Tvs///zzeHl5MWzYMBITE3nxxRfp0aMH3377bZY/c6rx48czYcIE2rRpQ79+/di7dy9z585lx44dbNmyhSJFipCcnEzbtm25fPkyAwcOJCwsjCNHjvDZZ59x5swZgoOD+fnnn7nvvvuoU6cOEydOxM/Pj3379rFly5Zs1ySSIUNEbqh///7G9f+ZtGrVygCMefPmpdv/woUL6bb961//MgIDA41Lly45tsXFxRkVKlRwPN6/f78BGCEhIcapU6cc25cvX24AxqeffurYNm7cuHQ1AYavr6+xb98+x7YffvjBAIxZs2Y5tnXo0MEIDAw0jhw54tj2+++/Gz4+PuneMyMZfb4pU6YYNpvNOHjwoNPnA4yJEyc67Vu/fn2jYcOGjsfLli0zAOPFF190bLt69arRokULAzAWLFiQaT0jR440ihQp4vR3dvnyZaNEiRLGo48+mmnd27ZtMwDjnXfecWzbuHGjARgbN250+ixpf1bZqTmj4y5atMgAjK+++sqx7aWXXjIAY//+/en2r1ChghEXF+d4PGTIEAMwNm/e7Nh29uxZo2LFikZ0dLRht9udPkv16tWNy5cvO/adOXOmARi7d+9Od6y0FixY4FTTiRMnDF9fX+Puu+92HMMwDGP27NkGYLz11luGYRjG999/bwDGxx9/fMP3nj59ugEYJ0+ezLQGkZzSaSmRHPDz86NPnz7ptgcEBDjunz17lr/++osWLVpw4cIFfv3115u+b2xsLCVLlnQ8btGiBWCehriZNm3aULlyZcfjOnXqEBQU5Hit3W5n/fr1dO7cmYiICMd+t9xyC+3bt7/p+4Pz5zt//jx//fUXzZs3xzAMvv/++3T7P/HEE06PW7Ro4fRZVq5ciY+Pj2MkB8Db25uBAwdmqZ7Y2FiuXLlCfHy8Y9vatWs5c+YMsbGxGdZ95coV/v77b2655RZKlCjBzp07s3SsnNSc9riXLl3ir7/+4rbbbgPI9nHTHr9Jkyb84x//cGwrVqwYjz/+OAcOHOCXX35x2r9Pnz74+vo6Hmfndyqt9evXk5yczJAhQ5wmOPft25egoCDHabHg4GDAPDV44cKFDN8rddL08uXL83yytngmhRuRHChXrpzTF0aqn3/+mfvvv5/g4GCCgoIoU6aMYzJy2vkGN1K+fHmnx6lB5/Tp09l+berrU1974sQJLl68yC233JJuv4y2ZeTQoUP07t2bUqVKOebRtGrVCkj/+fz9/dOdWklbD5hzYcLDwylWrJjTflWrVs1SPXXr1qVatWosXrzYsW3x4sWULl2aO+64w7Ht4sWLjB07lqioKPz8/ChdujRlypThzJkzWfq5pJWdmk+dOsXgwYMJDQ0lICCAMmXKULFiRSBrvw83On5Gx0pdwXfw4EGn7bn5nbr+uJD+c/r6+lKpUiXH8xUrVmTo0KG88cYblC5dmrZt2/Lqq686fd7Y2FhiYmL45z//SWhoKN27d+ejjz5S0BGX0ZwbkRxI+y/yVGfOnKFVq1YEBQUxceJEKleujL+/Pzt37mT48OFZ+h+3t7d3htsNw8jT12aF3W7nrrvu4tSpUwwfPpxq1apRtGhRjhw5Qu/evdN9vhvV42qxsbE899xz/PXXXxQvXpwVK1bw0EMPOa0oGzhwIAsWLGDIkCE0a9aM4OBgbDYb3bt3z9Mv1AcffJCtW7fyzDPPUK9ePYoVK0ZKSgrt2rXLty/yvP69yMjUqVPp3bs3y5cvZ+3atQwaNIgpU6bwzTffEBkZSUBAAF999RUbN27k888/Z/Xq1SxevJg77riDtWvX5tvvjrgvhRsRF9m0aRN///038fHxtGzZ0rF9//79FlZ1TdmyZfH3989wpUxWVs/s3r2b3377jbfffptevXo5tq9bty7HNVWoUIENGzZw7tw5p5GQvXv3Zvk9YmNjmTBhAp988gmhoaEkJSXRvXt3p32WLFlCXFwcU6dOdWy7dOlSji6al9WaT58+zYYNG5gwYQJjx451bP/999/TvWd2rjhdoUKFDP9+Uk97VqhQIcvvlR2p77t3714qVark2J6cnMz+/ftp06aN0/61a9emdu3ajB49mq1btxITE8O8efN49tlnAfDy8uLOO+/kzjvvZNq0aUyePJlRo0axcePGdO8lkl06LSXiIqn/2kz7L+Lk5GTmzJljVUlOvL29adOmDcuWLePo0aOO7fv27WPVqlVZej04fz7DMJg5c2aOa7rnnnu4evUqc+fOdWyz2+3MmjUry+9RvXp1ateuzeLFi1m8eDHh4eFO4TK19utHKmbNmpVuWbora87o7wtgxowZ6d6zaNGiAFkKW/fccw/bt29n27Ztjm3nz59n/vz5REdHU6NGjax+lGxp06YNvr6+vPLKK06f6c033yQxMZF7770XgKSkJK5ever02tq1a+Pl5cXly5cB83Td9erVqwfg2EckNzRyI+IizZs3p2TJksTFxTFo0CBsNhvvvvtung7/Z9f48eNZu3YtMTEx9OvXD7vdzuzZs6lVqxa7du3K9LXVqlWjcuXKDBs2jCNHjhAUFMQnn3yS7bkbaXXo0IGYmBhGjBjBgQMHqFGjBvHx8dmejxIbG8vYsWPx9/fnscceS3dF3/vuu493332X4OBgatSowbZt21i/fr1jiXxe1BwUFETLli158cUXuXLlCuXKlWPt2rUZjuQ1bNgQgFGjRtG9e3eKFClChw4dHKEnrREjRrBo0SLat2/PoEGDKFWqFG+//Tb79+/nk08+ybOrGZcpU4aRI0cyYcIE2rVrR8eOHdm7dy9z5syhcePGjrllX3zxBQMGDKBbt27ceuutXL16lXfffRdvb28eeOABACZOnMhXX33FvffeS4UKFThx4gRz5swhMjLSaaK0SE4p3Ii4SEhICJ999hlPP/00o0ePpmTJkjzyyCPceeedjuutWK1hw4asWrWKYcOGMWbMGKKiopg4cSJ79uy56WquIkWK8OmnnzrmT/j7+3P//fczYMAA6tatm6N6vLy8WLFiBUOGDOG9997DZrPRsWNHpk6dSv369bP8PrGxsYwePZoLFy44rZJKNXPmTLy9vXn//fe5dOkSMTExrF+/Pkc/l+zU/MEHHzBw4EBeffVVDMPg7rvvZtWqVU6r1QAaN27MpEmTmDdvHqtXryYlJYX9+/dnGG5CQ0PZunUrw4cPZ9asWVy6dIk6derw6aefOkZP8sr48eMpU6YMs2fP5qmnnqJUqVI8/vjjTJ482XEdprp169K2bVs+/fRTjhw5QmBgIHXr1mXVqlWOlWIdO3bkwIEDvPXWW/z111+ULl2aVq1aMWHCBMdqK5HcsBkF6Z+VImKJzp078/PPP2c4H0REpLDRnBsRD3N9q4Tff/+dlStX0rp1a2sKEhFxMY3ciHiY8PBwR7+jgwcPMnfuXC5fvsz3339PlSpVrC5PRCTXNOdGxMO0a9eORYsWkZCQgJ+fH82aNWPy5MkKNiLiNjRyIyIiIm5Fc25ERETErSjciIiIiFvxuDk3KSkpHD16lOLFi2frkuciIiJiHcMwOHv2LBERETe9WKXHhZujR48SFRVldRkiIiKSA4cPHyYyMjLTfTwu3BQvXhww/3KCgoIsrkZERESyIikpiaioKMf3eGY8LtyknooKCgpSuBERESlksjKlRBOKRURExK0o3IiIiIhbUbgRERERt+Jxc25ERMS17HY7V65csboMcQO+vr43XeadFQo3IiKSI4ZhkJCQwJkzZ6wuRdyEl5cXFStWxNfXN1fvo3AjIiI5khpsypYtS2BgoC6MKrmSepHdY8eOUb58+Vz9PinciIhIttntdkewCQkJsboccRNlypTh6NGjXL16lSJFiuT4fTShWEREsi11jk1gYKDFlYg7ST0dZbfbc/U+CjciIpJjOhUlruSq3yedlnIRux02b4ZjxyA8HFq0AG9vq6sSERHxPBq5cYH4eIiOhttvh4cfNv+Mjja3i4iI+4uOjmbGjBlZ3n/Tpk3YbLY8X2m2cOFCSpQokafHKIgUbnIpPh66doU//3TefuSIuV0BR0Qkc3Y7bNoEixaZf+ZyukWmbDZbprfx48fn6H137NjB448/nuX9mzdvzrFjxwgODs7R8SRzOi2VC3Y7DB4MhpH+OcMAmw2GDIFOnXSKSkQkI/Hx5v9H0/4DMTISZs6ELl1cf7xjx4457i9evJixY8eyd+9ex7ZixYo57huGgd1ux8fn5l+VZcqUyVYdvr6+hIWFZes1knUaucmFzZvTj9ikZRhw+LC5n4iIOLNi5DssLMxxCw4OxmazOR7/+uuvFC9enFWrVtGwYUP8/Pz4+uuv+d///kenTp0IDQ2lWLFiNG7cmPXr1zu97/WnpWw2G2+88Qb3338/gYGBVKlShRUrVjiev/60VOrpozVr1lC9enWKFStGu3btnMLY1atXGTRoECVKlCAkJIThw4cTFxdH586ds/V3MHfuXCpXroyvry9Vq1bl3XffdTxnGAbjx4+nfPny+Pn5ERERwaBBgxzPz5kzhypVquDv709oaChdu3bN1rHzi8JNLqT5nXPJfiIinuJmI99gjnzn5SmqGxkxYgTPP/88e/bsoU6dOpw7d4577rmHDRs28P3339OuXTs6dOjAoUOHMn2fCRMm8OCDD/Ljjz9yzz330KNHD06dOnXD/S9cuMDLL7/Mu+++y1dffcWhQ4cYNmyY4/kXXniB999/nwULFrBlyxaSkpJYtmxZtj7b0qVLGTx4ME8//TQ//fQT//rXv+jTpw8bN24E4JNPPmH69Om89tpr/P777yxbtozatWsD8N///pdBgwYxceJE9u7dy+rVq2nZsmW2jp9vDA+TmJhoAEZiYmKu32vjRsMw/zPM/LZxY64PJSJSoFy8eNH45ZdfjIsXL+bo9QXh/58LFiwwgoOD09S00QCMZcuW3fS1NWvWNGbNmuV4XKFCBWP69OmOx4AxevRox+Nz584ZgLFq1SqnY50+fdpRC2Ds27fP8ZpXX33VCA0NdTwODQ01XnrpJcfjq1evGuXLlzc6deqU5c/YvHlzo2/fvk77dOvWzbjnnnsMwzCMqVOnGrfeequRnJyc7r0++eQTIygoyEhKSrrh8XIrs9+r7Hx/a+QmF1q0MM8N32hZvs0GUVHmfiIick1BHvlu1KiR0+Nz584xbNgwqlevTokSJShWrBh79uy56chNnTp1HPeLFi1KUFAQJ06cuOH+gYGBVK5c2fE4PDzcsX9iYiLHjx+nSZMmjue9vb1p2LBhtj7bnj17iImJcdoWExPDnj17AOjWrRsXL16kUqVK9O3bl6VLl3L16lUA7rrrLipUqEClSpXo2bMn77//PhcuXMjW8fOLwk0ueHubk94gfcBJfTxjhiYTi4hcLzzctfu5UtGiRZ0eDxs2jKVLlzJ58mQ2b97Mrl27qF27NsnJyZm+z/XtA2w2GykpKdna38jovF0eioqKYu/evcyZM4eAgACefPJJWrZsyZUrVyhevDg7d+5k0aJFhIeHM3bsWOrWrVsgG6cq3ORSly6wZAmUK+e8PTLS3J4Xs/1FRAq7wjTyvWXLFnr37s39999P7dq1CQsL48CBA/laQ3BwMKGhoezYscOxzW63s3Pnzmy9T/Xq1dmyZYvTti1btlCjRg3H44CAADp06MArr7zCpk2b2LZtG7t37wbAx8eHNm3a8OKLL/Ljjz9y4MABvvjii1x8sryhpeAu0KWLudxbVygWEcma1JHvrl3NIJN2gKKgjXxXqVKF+Ph4OnTogM1mY8yYMZmOwOSVgQMHMmXKFG655RaqVavGrFmzOH36dLZaFjzzzDM8+OCD1K9fnzZt2vDpp58SHx/vWP21cOFC7HY7TZs2JTAwkPfee4+AgAAqVKjAZ599xh9//EHLli0pWbIkK1euJCUlhapVq+bVR84xhRsX8faG1q2trkJEpPBIHfnO6Do3M2YUnJHvadOm8eijj9K8eXNKly7N8OHDSUpKyvc6hg8fTkJCAr169cLb25vHH3+ctm3b4p2NBNi5c2dmzpzJyy+/zODBg6lYsSILFiyg9f9/gZUoUYLnn3+eoUOHYrfbqV27Np9++ikhISGUKFGC+Ph4xo8fz6VLl6hSpQqLFi2iZs2aefSJc85m5PcJPYslJSURHBxMYmIiQUFBVpcjIlIoXbp0if3791OxYkX8/f1z9V7qzZczKSkpVK9enQcffJBJkyZZXY5LZPZ7lZ3vb43ciIiIpTTynTUHDx5k7dq1tGrVisuXLzN79mz279/Pww8/bHVpBY4mFIuIiBQCXl5eLFy4kMaNGxMTE8Pu3btZv3491atXt7q0AkcjNyIiIoVAVFRUupVOkjGN3IiIiIhbUbgRERERt6JwIyIiIm5F4UZERETcisKNiIiIuBWFGxEREXErCjciIiLZ1Lp1a4YMGeJ4HB0dzYwZMzJ9jc1mY9myZbk+tqveJzPjx4+nXr16eXqMvKRwIyIiHqNDhw60a9cuw+c2b96MzWbjxx9/zPb77tixg8cffzy35Tm5UcA4duwY7du3d+mx3I3CjYiIeIzHHnuMdevW8WfaTp3/b8GCBTRq1Ig6depk+33LlClDYGCgK0q8qbCwMPz8/PLlWIWVwo2IiHiM++67jzJlyrBw4UKn7efOnePjjz/mscce4++//+ahhx6iXLlyBAYGUrt2bRYtWpTp+15/Wur333+nZcuW+Pv7U6NGDdatW5fuNcOHD+fWW28lMDCQSpUqMWbMGK5cuQLAwoULmTBhAj/88AM2mw2bzeao+frTUrt37+aOO+4gICCAkJAQHn/8cc6dO+d4vnfv3nTu3JmXX36Z8PBwQkJC6N+/v+NYWZGSksLEiROJjIzEz8+PevXqsXr1asfzycnJDBgwgPDwcPz9/alQoQJTpkwBwDAMxo8fT/ny5fHz8yMiIoJBgwZl+dg5ofYLIiLiEoYBFy5Yc+zAQLDZbr6fj48PvXr1YuHChYwaNQrb/7/o448/xm6389BDD3Hu3DkaNmzI8OHDCQoK4vPPP6dnz55UrlyZJk2a3PQYKSkpdOnShdDQUL799lsSExOd5uekKl68OAsXLiQiIoLdu3fTt29fihcvzr///W9iY2P56aefWL16NevXrwcgODg43XucP3+etm3b0qxZM3bs2MGJEyf45z//yYABA5wC3MaNGwkPD2fjxo3s27eP2NhY6tWrR9++fW/+lwbMnDmTqVOn8tprr1G/fn3eeustOnbsyM8//0yVKlV45ZVXWLFiBR999BHly5fn8OHDHD58GIBPPvmE6dOn8+GHH1KzZk0SEhL44YcfsnTcHDM8TGJiogEYiYmJVpciIlJoXbx40fjll1+MixcvOradO2cYZsTJ/9u5c1mvfc+ePQZgbNy40bGtRYsWxiOPPHLD19x7773G008/7XjcqlUrY/DgwY7HFSpUMKZPn24YhmGsWbPG8PHxMY4cOeJ4ftWqVQZgLF269IbHeOmll4yGDRs6Ho8bN86oW7duuv3Svs/8+fONkiVLGufS/AV8/vnnhpeXl5GQkGAYhmHExcUZFSpUMK5everYp1u3bkZsbOwNa7n+2BEREcZzzz3ntE/jxo2NJ5980jAMwxg4cKBxxx13GCkpKenea+rUqcatt95qJCcn3/B4qTL6vUqVne9vnZYSERGPUq1aNZo3b85bb70FwL59+9i8eTOPPfYYAHa7nUmTJlG7dm1KlSpFsWLFWLNmDYcOHcrS++/Zs4eoqCgiIiIc25o1a5Zuv8WLFxMTE0NYWBjFihVj9OjRWT5G2mPVrVuXokWLOrbFxMSQkpLC3r17Hdtq1qyJt7e343F4eDgnTpzI0jGSkpI4evQoMTExTttjYmLYs2cPYJ762rVrF1WrVmXQoEGsXbvWsV+3bt24ePEilSpVom/fvixdupSrV69m63Nml8KNiIi4RGAgnDtnzS27c3kfe+wxPvnkE86ePcuCBQuoXLkyrVq1AuCll15i5syZDB8+nI0bN7Jr1y7atm1LcnKyy/6utm3bRo8ePbjnnnv47LPP+P777xk1apRLj5FWkSJFnB7bbDZSUlJc9v4NGjRg//79TJo0iYsXL/Lggw/StWtXwOxmvnfvXubMmUNAQABPPvkkLVu2zNacn+zSnBsREXEJmw3SDCAUaA8++CCDBw/mgw8+4J133qFfv36O+TdbtmyhU6dOPPLII4A5h+a3336jRo0aWXrv6tWrc/jwYY4dO0Z4eDgA33zzjdM+W7dupUKFCowaNcqx7eDBg077+Pr6Yrfbb3qshQsXcv78ecfozZYtW/Dy8qJq1apZqvdmgoKCiIiIYMuWLY4AmHqctHOQgoKCiI2NJTY2lq5du9KuXTtOnTpFqVKlCAgIoEOHDnTo0IH+/ftTrVo1du/eTYMGDVxS4/UUbkRExOMUK1aM2NhYRo4cSVJSEr1793Y8V6VKFZYsWcLWrVspWbIk06ZN4/jx41kON23atOHWW28lLi6Ol156iaSkJKcQk3qMQ4cO8eGHH9K4cWM+//xzli5d6rRPdHQ0+/fvZ9euXURGRlK8ePF0S8B79OjBuHHjiIuLY/z48Zw8eZKBAwfSs2dPQkNDc/aXk4FnnnmGcePGUblyZerVq8eCBQvYtWsX77//PgDTpk0jPDyc+vXr4+Xlxccff0xYWBglSpRg4cKF2O12mjZtSmBgIO+99x4BAQFUqFDBZfVdT6elRETEIz322GOcPn2atm3bOs2PGT16NA0aNKBt27a0bt2asLAwOnfunOX39fLyYunSpVy8eJEmTZrwz3/+k+eee85pn44dO/LUU08xYMAA6tWrx9atWxkzZozTPg888ADt2rXj9ttvp0yZMhkuRw8MDGTNmjWcOnWKxo0b07VrV+68805mz56dvb+Mmxg0aBBDhw7l6aefpnbt2qxevZoVK1ZQpUoVwFz59eKLL9KoUSMaN27MgQMHWLlyJV5eXpQoUYLXX3+dmJgY6tSpw/r16/n0008JCQlxaY1p2QzDMPLs3QugpKQkgoODSUxMJCgoyOpyREQKpUuXLrF//34qVqyIv7+/1eWIm8js9yo7398auRERERG3onAjIiIibkXhRkRERNyKwo2IiIi4FYUbERHJMQ9bkyJ5zFW/Two3IiKSbalXvL1gVadMcUupV2hO2yoiJ3QRPxc6fhySkuD/l/2LiLgtb29vSpQo4ehPFBgY6LjCr0hOpKSkcPLkSQIDA/HxyV08UbhxkRUr4OGHoWFD2LTJvAy5iIg7CwsLA8hyA0aRm/Hy8qJ8+fK5DsoKNy5Svz5cvQpffQWrVsE991hdkYhI3rLZbISHh1O2bNk8bYIonsPX1xcvr9zPmFG4cZGoKBg0CF56CUaMgLZtIZenDEVECgVvb+9cz5EQcSVNKHahESOgRAnYvRs++MDqakRERDyTwo0LlSoFI0ea90ePhkuXrK1HRETEEyncuNjAgVCuHBw6BHPnWl2NiIiI51G4cbGAAJgwwbz/7LOQmGhtPSIiIp5G4SYPxMVB9epw6hS8+KLV1YiIiHgWhZs84OMDU6aY96dPh2PHrK1HRETEkyjc5JGOHaF5c7h48dppKhEREcl7Cjd5xGaDF14w77/xBuzda209IiIinkLhJg/94x/QoQPY7TBqlNXViIiIeAbLw82rr75KdHQ0/v7+NG3alO3bt2e6/5kzZ+jfvz/h4eH4+flx6623snLlynyqNvsmTwYvL/jkE/j2W6urERERcX+WhpvFixczdOhQxo0bx86dO6lbty5t27a9YRO25ORk7rrrLg4cOMCSJUvYu3cvr7/+OuXKlcvnyrOuVi1z9RTA8OFgGNbWIyIi4u5shmHd123Tpk1p3Lgxs2fPBsx251FRUQwcOJARI0ak23/evHm89NJL/PrrrxQpUiRHx0xKSiI4OJjExESCgoJyVX9WHT4MVarA5cuwciW0b58vhxUREXEb2fn+tmzkJjk5me+++442bdpcK8bLizZt2rBt27YMX7NixQqaNWtG//79CQ0NpVatWkyePBm73X7D41y+fJmkpCSnW35LbaoJ5uhNJuWKiIhILlkWbv766y/sdjuhoaFO20NDQ0lISMjwNX/88QdLlizBbrezcuVKxowZw9SpU3n22WdveJwpU6YQHBzsuEVFRbn0c2SVmmqKiIjkD8snFGdHSkoKZcuWZf78+TRs2JDY2FhGjRrFvHnzbviakSNHkpiY6LgdPnw4Hyu+plQpM+CAmmqKiIjkJcvCTenSpfH29ub48eNO248fP05YWFiGrwkPD+fWW2/F29vbsa169eokJCSQnJyc4Wv8/PwICgpyulll0CA11RQREclrloUbX19fGjZsyIYNGxzbUlJS2LBhA82aNcvwNTExMezbt4+UlBTHtt9++43w8HB8fX3zvObcSttU87nn1FRTREQkL1h6Wmro0KG8/vrrvP322+zZs4d+/fpx/vx5+vTpA0CvXr0YOXKkY/9+/fpx6tQpBg8ezG+//cbnn3/O5MmT6d+/v1UfIdtSm2r+/Te89JLV1YiIiLgfHysPHhsby8mTJxk7diwJCQnUq1eP1atXOyYZHzp0CC+va/krKiqKNWvW8NRTT1GnTh3KlSvH4MGDGT58uFUfIdtSm2p27gzTpkH//hAebnVVIiIi7sPS69xYwYrr3FzPMMzWDFu3wr/+BZnMhxYREREKyXVuPJnNBs8/b95XU00RERHXUrixSIsW15pqjh5tdTUiIiLuQ+HGQqlNNZcsUVNNERERV1G4sZCaaoqIiLiewo3FJkwAPz/48ktYvdrqakRERAo/hRuLRUXBwIHmfTXVFBERyT2FmwJg5EgIDlZTTREREVdQuCkASpUyAw7AmDFw+bK19YiIiBRmCjcFRGpTzYMH1VRTREQkNxRuCoi0TTWffVZNNUVERHJK4aYAUVNNERGR3FO4KUB8fMwL+4HZVPPYMWvrERERKYwUbgqYTp2gWTO4ePHaaSoRERHJOoWbAsZmgxdeMO+/8Qb89pu19YiIiBQ2CjcFUNqmmqNGWV2NiIhI4aJwU0CpqaaIiEjOKNwUULVqQa9e5n011RQREck6hZsCTE01RUREsk/hpgArX/5aU80RIyAlxdp6RERECgOFmwIutanmjz+qqaaIiEhWKNwUcGmbao4eraaaIiIiN6NwUwioqaaIiEjWKdwUAgEBMH68eV9NNUVERDKncFNI9O4N1aqpqaaIiMjNKNwUEj4+MGWKeX/6dDXVFBERuRGFm0IktanmhQswcWLeHMNuh02bYNEi80+7PW+OIyIiklcUbgqRtE01X3/d9U014+MhOhpuvx0eftj8Mzra3C4iIlJYKNwUMi1awH33ub6pZnw8dO0Kf/7pvP3IEXO7Ao6IiBQWCjeF0JQp5ijOkiWwfXvu389uh8GDM+5flbptyBCdohIRkcJB4aYQqlUL4uLM+65oqrl5c/oRm7QMAw4fNvcTEREp6BRuCqnUppqbNsGaNbl7r6yuvNIKLRERKQwUbgqptE01hw/PXVPN8HDX7iciImIlhZtCzFVNNVu0gMhIcx5PRmw2iIoy9xMRESnoFG4KsVKlYMQI835ummp6e8PMmeb96wNO6uMZM8z9RERECjqFm0Ju0CCIiMh9U80uXczVV+XKOW+PjDS3d+mSuzpFRETyi80wcrvWpnBJSkoiODiYxMREgoKCrC7HJd54A/r2hZAQ+N//zFNVOWW3m6uijh0z59i0aKERGxERsV52vr81cuMG0jbVfPnl3L2Xtze0bg0PPWT+qWAjIiKFjcKNG0jbVHPaNC3ZFhERz6Zw4ybyo6mmiIhIYaBw4ybyuqmmiIhIYaFw40byqqmmiIhIYaJw42Zc3VRTRESksFG4cTOubqopIiJS2CjcuCFXNtUUEREpbBRu3FD58jBggHk/t001RUREChuFGzflqqaaIiIihY3CjZsKCbnWVHPMmJw31RQRESlsFG7cWGpTzQMHYN48q6sRERHJHwo3biww0JxcDDBpEiQmWluPiIhIflC4cXOubKopIiJSGCjcuDkfH5g82byvppoiIuIJFG48QOfOcNttaqopIiKeQeHGA6ippoiIeBKFGw/RsuW1ppqjR1tdjYiISN5RuPEgqU01P/5YTTVFRMR9Kdx4kFq1oFcv876aaoqIiLtSuPEwEyeqqaaIiLg3hRsPk7ap5ogRaqopIiLuR+HGA6U21fzhB1i0yOpqREREXEvhxgOlbao5erSaaoqIiHtRuPFQaqopIiLuSuHGQwUGwvjx5n011RQREXeicOPB+vSBqlXVVFNERNyLwo0H8/ExL+wHZlPNhARr6xEREXEFhRsPp6aaIiLibhRuPFzapprz56uppoiIFH4KN6KmmiIi4lYKRLh59dVXiY6Oxt/fn6ZNm7I9k66OCxcuxGazOd38/f3zsVr3NHnytaaaO3ZYXY2IiEjOWR5uFi9ezNChQxk3bhw7d+6kbt26tG3blhMnTtzwNUFBQRw7dsxxO3jwYD5W7J5q11ZTTRERcQ+Wh5tp06bRt29f+vTpQ40aNZg3bx6BgYG89dZbN3yNzWYjLCzMcQsNDc3Hit1XalPNjRth7VqrqxEREckZS8NNcnIy3333HW3atHFs8/Lyok2bNmzbtu2Grzt37hwVKlQgKiqKTp068fPPP+dHuW4vbVPNp582V1CJiIgUNpaGm7/++gu73Z5u5CU0NJSEG1x0pWrVqrz11lssX76c9957j5SUFJo3b86ff/6Z4f6XL18mKSnJ6SY3NnIklCkDP/8MPXuqa7iIiBQ+lp+Wyq5mzZrRq1cv6tWrR6tWrYiPj6dMmTK89tprGe4/ZcoUgoODHbeoqKh8rrhwCQmB+Hjw9TX/1OopEREpbCwNN6VLl8bb25vjx487bT9+/DhhYWFZeo8iRYpQv3599u3bl+HzI0eOJDEx0XE7fPhwrut2d//4B7zxhnl/yhR4+21r6xEREckOS8ONr68vDRs2ZMOGDY5tKSkpbNiwgWbNmmXpPex2O7t37yY8PDzD5/38/AgKCnK6yc317AmjRpn3+/aFzZutrUdERCSrLD8tNXToUF5//XXefvtt9uzZQ79+/Th//jx9+vQBoFevXowcOdKx/8SJE1m7di1//PEHO3fu5JFHHuHgwYP885//tOojuK2JE6FrV7hyBe6/H/73P6srEhERuTkfqwuIjY3l5MmTjB07loSEBOrVq8fq1asdk4wPHTqEl9e1DHb69Gn69u1LQkICJUuWpGHDhmzdupUaNWpY9RHclpeXeUrq4EHzwn733QfbtkGJElZXJiIicmM2w/Csy7UlJSURHBxMYmKiTlFl0bFj0KQJ/PkntGkDK1dCkSJWVyUiIp4kO9/flp+WkoIvPBw+/RSKFoX162HgQF3BWERECi6FG8mSevVg0SKz/9Rrr8Err1hdkYiISMYUbiTLOnSAl1827w8dCp9/bm09IiIiGVG4kWx56ilzaXhKCnTvDj/+aHVFIiIizhRuJFtsNnj1VbjjDjh3zhzNuUGnDBEREUso3Ei2FSkCS5bArbfCoUPQuTNcvGh1VSIiIiaFG8mRkiXhs8+gVCn49lvo00crqEREpGBQuJEcq1LFbK5ZpAgsXgzjx1tdkYiIiMKN5FKrVubScDDbNbz/vrX1iIiIKNxIrvXpA//+t3n/0Udh61Zr6xEREc+mcCMuMWWKObE4Odn888ABiwsSERGPpXAjLuHlBe+9B/Xrw8mTZpPNpCSrqxIREU+kcCMuU7So2YMqIgJ+/hliY+HqVaurEhERT6NwIy5VrhysWAEBAbB6tdmmQUREJD8p3IjLNWxonqICmDXLvKKxiIhIflG4kTzRpYs5yRhg8GBYs8baekRExHMo3EieGT4cevcGux0efBB++cXqikRExBMo3EiesdnMC/y1bGmunLrvPnMllYiISF5SuJE85esLn3wClSvD/v3mNXAuXbK6KhERcWc5CjeHDx/mzz//dDzevn07Q4YMYf78+S4rTNxH6dJmk83gYPPqxX37qsmmiIjknRyFm4cffpiNGzcCkJCQwF133cX27dsZNWoUEydOdGmB4h6qVYMlS8Db21xJNXmy1RWJiIi7ylG4+emnn2jSpAkAH330EbVq1WLr1q28//77LFy40JX1iRtp0wbmzDHvjx4NH31kbT0iIuKechRurly5gp+fHwDr16+nY8eOAFSrVo1jx465rjpxO48/Dk89Zd6Pi4Pt262tR0RE3E+Owk3NmjWZN28emzdvZt26dbRr1w6Ao0ePEhIS4tICxf289BLce685sbhjRzh0yOqKRETEneQo3Lzwwgu89tprtG7dmoceeoi6desCsGLFCsfpKpEb8faGRYugTh04fhw6dICzZ62uSkRE3IXNMHK2bsVut5OUlETJkiUd2w4cOEBgYCBly5Z1WYGulpSURHBwMImJiQQFBVldjkc7dAiaNLkWcJYuNYOPiIjI9bLz/Z2jkZuLFy9y+fJlR7A5ePAgM2bMYO/evQU62EjBUr48LF8O/v5mN/F//9vqikRExB3kKNx06tSJd955B4AzZ87QtGlTpk6dSufOnZk7d65LCxT31rQpvP22eX/aNNClkkREJLdyFG527txJixYtAFiyZAmhoaEcPHiQd955h1deecWlBYr7e/BBSL08Uv/+sGGDtfWIiEjhlqNwc+HCBYoXLw7A2rVr6dKlC15eXtx2220cPHjQpQWKZxg9Gnr0gKtXoWtX2Ls3d+9nt8OmTebE5U2bzMciIuIZchRubrnlFpYtW8bhw4dZs2YNd999NwAnTpzQJF3JEZsN3ngDmjeHM2fMJpt//52z94qPh+houP12ePhh88/oaHO7iIi4vxyFm7FjxzJs2DCio6Np0qQJzZo1A8xRnPr167u0QPEc/v7miqnoaNi3Dx54AJKTs/ce8fHmyE+a1mcAHDliblfAERFxfzleCp6QkMCxY8eoW7cuXl5mRtq+fTtBQUFUq1bNpUW6kpaCF3w//wzNmpnXvunTB9580xzZuRm73QxG1webVDYbREaa3cm15FxEpHDJ86XgAGFhYdSvX5+jR486OoQ3adKkQAcbKRxq1jT7Tnl5wYIF5hWNs2Lz5hsHGzA7kR8+bO4nIiLuK0fhJiUlhYkTJxIcHEyFChWoUKECJUqUYNKkSaSkpLi6RvFA7drBzJnm/REjzNNVN5PVtmZqfyYi4t58cvKiUaNG8eabb/L8888TExMDwNdff8348eO5dOkSzz33nEuLFM80YIC5amr2bHjkEXPEpUGDG+8fHp61983qfiIiUjjlaM5NREQE8+bNc3QDT7V8+XKefPJJjhw54rICXU1zbgqXq1fNlVNr1kBEhNlFvFy5jPdNnXNz5Ih5Cup6mnMjIlJ45fmcm1OnTmU4t6ZatWqcOnUqJ28pkiEfH1i8GGrUgKNHzS7i589nvK+397VTWddPQE59PGOGgo2IiLvLUbipW7cus2fPTrd99uzZ1KlTJ9dFiaQVHAyffQZlysDOndCzJ9xoaleXLrBkSfrRnchIc3uXLnlfr4iIWCtHp6W+/PJL7r33XsqXL++4xs22bds4fPgwK1eudLRmKIh0Wqrw2rrVvCBfcrI5yXjKlBvva7ebc3SOHTPn2LRooREbEZHCLM9PS7Vq1YrffvuN+++/nzNnznDmzBm6dOnCzz//zLvvvpujokVupnlzeOst8/7zz5vLxG/E2xtat4aHHjL/VLAREfEcOb6IX0Z++OEHGjRogL0AN/LRyE3hN3YsTJoERYrAunXQqpXVFYmISF7Ll4v4iVhl/Hizk/iVK+Ycmn37rK5IREQKEoUbKXS8vGDhQmjSBE6dMpeKnz5tdVUiIlJQKNxIoRQQAMuXQ1SUeaG/bt3MkRwREZFsXaG4y03W0Z45cyY3tYhkS1iYuUQ8JgY2bDCvaDxvXtaabIqIiPvKVrgJDg6+6fO9evXKVUEi2VGnDixaZF7cb/58qFYNnnrK6qpERMRKLl0tVRhotZR7mj4dhg41R22WL4cOHayuSEREXEmrpcTjDBkC//qX2VPqoYfghx+srkhERKyicCNuwWaDWbPgzjvN3lMdOkBCgtVViYiIFRRuxG0UKQIffwxVq8Lhw9CpE1y8aHVVIiKS3xRuxK2ULGmuoCpVCrZvhx494MIFq6sSEZH8pHAjbueWW2DpUnMkZ+lSaNjQ7CYuIiKeQeFG3FLLlrB6NUREwK+/QtOmZrPNAtz2TEREXEThRtzWHXfAjz/CAw/A1aswcqQ54fjQIasrExGRvKRwI24tJMScZPzWW1CsGHz5pXnhvw8+sLoyERHJKwo34vZsNujTB3btgttug8REc6Jxjx6gjiEiIu5H4UY8RuXKsHkzjB8P3t7m6E3duvDVV1ZXJiIirqRwIx7FxwfGjYOvv4ZKlcz5N61bm/NxkpOtrk5ERFxB4UY80m23maepHn3UbNnw/PPQrJm5skpERAo3hRvxWMWLw5tvwiefmBf927kTGjSAefPMwCMiIoWTwo14vC5dYPduuOsus11Dv37QsSOcOGF1ZSIikhMKNyKYF/tbvRqmTwc/P7OFQ+3a8PnnVlcmIiLZpXAj8v+8vGDIENixwww2J07AfffBk0+qP5WISGGicCNyndq1zaabTz1lPp47V/2pREQKE4UbkQz4+8O0abB27bX+VLfdBi+8oP5UIiIFncKNSCbuuutaf6orV2DECPWnEhEp6BRuRG7iRv2pFi2yujIREclIgQg3r776KtHR0fj7+9O0aVO2b9+epdd9+OGH2Gw2OnfunLcFisfLqD/Vww/DI4+oP5WISEFjebhZvHgxQ4cOZdy4cezcuZO6devStm1bTtzkIiMHDhxg2LBhtGjRIp8qFUnfn+r999WfSkSkoLE83EybNo2+ffvSp08fatSowbx58wgMDOStt9664Wvsdjs9evRgwoQJVKpUKR+rFVF/KhGRgs7ScJOcnMx3331HmzZtHNu8vLxo06YN27Ztu+HrJk6cSNmyZXnsscdueozLly+TlJTkdBNxhYz6UzVvrv5UIiJWszTc/PXXX9jtdkJDQ522h4aGkpCQkOFrvv76a958801ef/31LB1jypQpBAcHO25RUVG5rlsk1fX9qb77Tv2pRESsZvlpqew4e/YsPXv25PXXX6d06dJZes3IkSNJTEx03A4fPpzHVYon6tLFXDLepo36U4mIWM3HyoOXLl0ab29vjh8/7rT9+PHjhIWFpdv/f//7HwcOHKBDhw6ObSkpKQD4+Piwd+9eKleu7PQaPz8//Pz88qB6EWflysGaNfDKK+b1cFL7U731Ftx7r9XViYh4DktHbnx9fWnYsCEbNmxwbEtJSWHDhg00a9Ys3f7VqlVj9+7d7Nq1y3Hr2LEjt99+O7t27dIpJ7Fc2v5UtWpd60/Vv7/6U4mI5BdLR24Ahg4dSlxcHI0aNaJJkybMmDGD8+fP06dPHwB69epFuXLlmDJlCv7+/tSqVcvp9SVKlABIt13ESrVrmwHnP/8xO43PmQNffGEuHW/QwOrqRETcm+XhJjY2lpMnTzJ27FgSEhKoV68eq1evdkwyPnToEF5ehWpqkAhwrT9V+/bQu/e1/lSTJsGwYeZ1ckRExPVshuFZazqSkpIIDg4mMTGRoKAgq8sRD/H33/D44xAfbz5u1QreeQfKl7e2LhGRwiI7398aEhHJByEhsGRJ9vtT2e2waZO536ZN6kguIpIVCjci+SS7/ani4yE6Gm6/3dzv9tvNx6mjPyIikjGFG5F8lpX+VPHx0LUr/Pmn82uPHDG3K+CIiNyYwo2IBVL7U23e7Nyf6j//MS8COHhwxlc4Tt02ZIhOUYmI3IjCjYiFmjVz7k81ZQrUq5d+xCYtw4DDh81gJCIi6SnciFjs+v5Uv/2WtdcdO5a3dYmIFFYKNyIFRGp/qoYNs7Z/eHje1iMiUlgp3IgUIOXKwbZtEBx8431sNoiKghYt8q8uEZHCROFGpIApUsS8Hs6NGAbMmKErHIuI3IjCjUgB1KWLOQenXLn0zwUHw+7dZlNOERFJT+0XRAowu91cFfX77/Df/8LKlddWUvn6Qo8e5rLxunWtrVNEJK9l5/tb4UakELlyxbyA3/Tp8O2317bffjs89RTcey+oz6yIuCP1lhJxU0WKQGwsfPONOfE4Ntace7NxI3TsCFWrwuzZcO6c1ZWKiFhH4UakkLrtNvjwQ/jjD/j3v6FECdi3DwYOhMhIeOYZOHjQ6ipFRPKfwo1IIVe+PLzwgjkX59VX4dZbzaacL79stnbo1g22bs24nYOIiDtSuBFxE0WLwpNPwp498Pnn0KYNpKTAkiUQEwNNm8IHH5jzdkRE3JnCjYib8fKCe+6BdevMJeOPPQZ+frBjh7m6Kjra7GH1999WVyoikjcUbkTcWK1a8MYbZqPNSZMgLAyOHjW7j0dFwRNPmCM9IiLuROFGxAOUKQOjR8OBA/DOO1C/Ply8CK+9BjVqQPv2sGaN5uWIiHtQuBHxIH5+0LMnfPcdfPkl3H+/2atq9Wpo1w5q1jQDz4ULVlcqIpJzCjciHshmg5YtzQsC7tsHQ4ZA8eLmKaonnjBPWf3nP3DkiNWViohkn8KNiIerVMm84vGff5p/VqwIp06Zk46jo81JyDt2WF2liEjWKdyICABBQeYIzu+/myM6LVvC1avm8vEmTeAf/zCXlV+9anWlIiKZU7gRESfe3uZcnC+/NOfm9Oxptn3YssW8IOAtt8DUqXDmjNWViohkTOFGRG6oQQNzddXBgzBmDJQubd4fNsxs8TBokDlnR0SkIFG4EZGbCg+HiRPh0CHzujm1asH58zBrltnuoWNHs3mnlpKLSEGgcCMiWRYQYF7x+McfzSsg33uvGWg+/RTuuMO8fs7ChXDpktWViognU7gRkWyz2czeVZ99Br/+ava0CgyEH36APn2gQgUYPx6OH7e6UhHxRAo3IpIrVaua3cj//NPsTh4ZCSdOwIQJZsfyPn3M0CMikl8UbkTEJUqWhH//G/74AxYvhttug+Rk8zRVvXrmaaulS3XKSkTynsKNiLhUkSLw4IOwbZt5697dXF6+cSN06WKuuOraFd57D06ftrpaEXFHNsPwrPUNSUlJBAcHk5iYSFBQkNXliHiEw4fNU1fvvefc0sHbG1q3hk6dzFv58paVKCIFXHa+vxVuRCTfGIZ5YcBly2D5cvjpJ+fnGzQwQ07nzlC7tjlxWUQEFG4ypXAjkv/sdti8GY4dM6+Z06KFOWqzb58ZcpYvh6+/dr5OTsWK14JOTAz4+FhWvogUAAo3mVC4Eclf8fEweLC5mipVZCTMnGnOwUl14oS5tHz5cli71nnicUgIdOhghp277zaXnYuIZ1G4yYTCjUj+iY83Jw9f/3+Z1NNNS5Y4B5xU58+bAWfZMjPwnDp17bmAADPgdOoE990HZcrkWfkiUoAo3GRC4UYkf9jtEB3tPGKTls1mjuDs32+eorqRq1fNU1ap83QOHLj2nJeX2a089fRVpUquq19EChaFm0wo3Ijkj02b4Pbbb77fxo3miqmsMAyz9UNq0Pn+e+fna9e+FnQaNNCEZBF3kp3vb13nRkTyxLFjrt0PzLBSty6MGwc7d5qjODNnmiHK2xt274Znn4VGjcwWEAMHwvr1cOVKjj6CiBRSCjcikifCw127X0YqVIBBg+CLL8wJye+8Y87hCQw0r60zezbcdReULQuPPAIffwxnz+b8eCJSOOi0lIjkidQ5N0eOpJ9QDFmfc5MTFy/Chg3m6asVK+DkyWvP+fqaTT87dYKOHSEszLXHFpG8oTk3mVC4Eck/qaulwDng3Gy1lCvZ7fDNN2bQWbbMvLZO2jpuu82co9Opk9kEVEQKJoWbTCjciOSvjK5zExUFM2bkfbC5nmHAnj3Xgs6OHc7PV6tmBp3OnaFxY3M1logUDAo3mVC4Ecl/N7pCsdWOHDFPWy1bZq7aSjvxODzcPG3VqZPZ0dzPz7IyRQSFm0wp3IhIRhITYdUqM+isXOk88bh4cWjf3hzRad8eSpSwqEgRD6ZwkwmFGxG5mcuXzev0pF5PJ+1ydR8faNXK7HfVpIl501WSRfKewk0mFG5EJDtSUuC//702T2fPnvT7VKx4Leg0bQr166v/lYirKdxkQuFGRHLjt9/MCwNu327eMgo73t7m1ZKbNr0WeqpXLxjzjEQKK4WbTCjciIgrJSaaIzvbt8O335q3hIT0+xUrZl45OXV0p0kTKFdOLSJEskrhJhMKNyKSlwzDXIX17bfXRnd27DA7nV8vPNx5dKdxY9D/lkQypnCTCYUbEclvdrt5+ip1dGf7drMPlt3uvJ/NZl5rJ+3oTu3a5lWVRTydwk0mFG5EpCC4cMFs/pk6uvPtt2Yj0Ov5+ZkdztNOWK5USaezxPMo3GRC4UZECqoTJ8xTWGlPaZ0+nX6/UqWcR3caN9ZydHF/CjeZULgRkcLCMMxeWKlBZ/t2+P578zo816tUKf1y9ICA/K9ZJK8o3GRC4UZECrPkZPjxR+f5O7/+mn4/H5/0y9GrVdNydCm8FG4yoXAjIu7mzJlry9FTQ09Gy9GLF3dejt64sZajS+GhcJMJhRsRyamC2gD0eoZhdmFPO7rz3/9mvBw9ONi8wGCNGs5/VqigruhSsCjcZELhRkRyIj4eBg82Q0OqyEiYORO6dLGurqyy2+GXX5xHd376Kf1y9FQBAeZprOtDT+XKUKRI/tYuAgo3mVK4EZHsio+Hrl3NEZG0Uk/nLFlSOALO9S5fht9/N0PPnj3X/ty715zbk5EiRaBKlfSjPbfeqgnMkrcUbjKhcCMi2WG3Q3S084hNWjabOYKzf3/BPEWVE1evmp/n+tCzZ0/Gp7bA/HuoVCl96KlWTVddFtdQuMmEwo2IZMemTXD77Tffb+NGaN06r6uxVkqKGfKuDz2//JLx9XhSRUZmPK+ndOn8q10Kv+x8f/vkU00iIoXSsWOu3a8w8/KC8uXNW7t217YbhnkBwoxGeo4dMwPRn3/CunXO71emTMahJyJCK7gkdxRuREQyER7u2v3ckc0GoaHm7fpRrtOnzevwXB98DhyAkyfN21dfOb8mKCjj0BMdrRVckjU6LSUikonUOTdHjqSfUAzuOecmP5w/b05c/uUX5+Dzv/9lvoKratX0weeWW7SCyxNozk0mFG5EJLtSV0uBc8Ap7KulCqLUFVypp7XSruDKqO0EmKEyIsK8IGFmt8DA/P0s4loKN5lQuBGRnMjoOjdRUTBjhoJNfrDbM17B9csvN17Bdb0SJZzDTmRk+gBUurROfRVUCjeZULgRkZwqLFco9iSGAUePmqHzyJEb37IagIoUydookL9/3n4uSa/QhZtXX32Vl156iYSEBOrWrcusWbNo0qRJhvvGx8czefJk9u3bx5UrV6hSpQpPP/00PXv2zNKxFG5ERDyLYUBiYubh58gRc8VXVr8RQ0JuHoBCQrTqy5UK1VLwxYsXM3ToUObNm0fTpk2ZMWMGbdu2Ze/evZQtWzbd/qVKlWLUqFFUq1YNX19fPvvsM/r06UPZsmVp27atBZ9AREQKMpvNPCVVogTUrHnj/a5cMUflbhR+UkeHLl2Cv/82bz/+eOP38/NLPwp0/amwiAjw9XX1JxbLR26aNm1K48aNmT17NgApKSlERUUxcOBARowYkaX3aNCgAffeey+TJk266b4auRERkZwyDHN5+81GgU6ezPp7lilzLeyUKQOlSmV+CwryzBGhQjNyk5yczHfffcfIkSMd27y8vGjTpg3btm276esNw+CLL75g7969vPDCC3lZqoiICDbbtZBRu/aN97t8OeNRoLRzg44eNfdLvd7Prl1Zq8Hb++YBKO0tJMT8MzjYcyZLWxpu/vrrL+x2O6GhoU7bQ0ND+fXXX2/4usTERMqVK8fly5fx9vZmzpw53HXXXRnue/nyZS6nWT+YlJTkmuJFRERuwM/PvD5SdPSN9zEM89RW2rDz999w6lT6W+r2ixfNie2pgSg7bDYoWTJrQSjtrUQJ8LF8Ekv2FLJyTcWLF2fXrl2cO3eODRs2MHToUCpVqkTrDBq7TJkyhQkTJuR/kSIiIpmw2cyl56VLQ926WXvNxYvmabGMAlDaEHT97dw5M0ylPs6u4OCbh6C0t9KlzVNsVrF0zk1ycjKBgYEsWbKEzp07O7bHxcVx5swZli9fnqX3+ec//8nhw4dZs2ZNuucyGrmJiorSnBsREfEYycnXQtGNAlBGt8TEnB2vfn3YudO1n6HQzLnx9fWlYcOGbNiwwRFuUlJS2LBhAwMGDMjy+6SkpDgFmLT8/Pzw8/NzRbkiIiKFkq/vtf5f2XHlCpw5c/MQdH1gsrrju+WnpYYOHUpcXByNGjWiSZMmzJgxg/Pnz9OnTx8AevXqRbly5ZgyZQpgnmZq1KgRlStX5vLly6xcuZJ3332XuXPnWvkxREQKDV2MULKqSBHz9FJ2TzFZfQU9y8NNbGwsJ0+eZOzYsSQkJFCvXj1Wr17tmGR86NAhvNJM7z5//jxPPvkkf/75JwEBAVSrVo333nuP2NhYqz6CiEihkVEbichImDlTbSTEdaxeqm75dW7ym65zIyKeKrUB6PX/11cDUCkMsvP97SEr3kVEPJvdbo7YZPTP2dRtQ4aY+4kUdgo3IiIeYPNm51NR1zMMOHzY3E+ksFO4ERHxAMeOuXY/kYJM4UZExAOEh7t2P5GCTOFGRMQDtGhhroq60SoWmw2iosz9RAo7hRsREQ/g7W0u94b0ASf18YwZut6NuAeFGxERD9Gli7ncu1w55+2RkVoGLu7F8ov4iYhI/unSBTp10hWKxb0p3IiIeBhvb2jd2uoqRPKOTkuJiIiIW1G4EREREbeicCMiIiJuRXNuRESk0LLbNTla0lO4ERGRQik+3mwGmrZnVmSkeT0fLWv3bDotJSIihU58PHTtmr4Z6JEj5vb4eGvqkoJB4UZERAoVu90csTGM9M+lbhsyxNxPPJPCjYiIFCqbN6cfsUnLMODwYXM/8UwKNyIiUqgcO+ba/cT9KNyIiEihEh7u2v3E/SjciIhIodKihbkq6vru5qlsNoiKMvcTz6RwIyIihYq3t7ncG9IHnNTHM2boejeeTOFGREQKnS5dYMkSKFfOeXtkpLld17nxbLqIn4iIFEpdukCnTrpCsaSncCMiIoWWtze0bm11FVLQ6LSUiIiIuBWFGxEREXErOi0lIiJiMXU3dy2FGxEREQupu7nr6bSUiIiIRdTdPG8o3IiIiFhA3c3zjsKNiIiIBdTdPO8o3IiIiFhA3c3zjsKNiIiIBdTdPO8o3IiIiFhA3c3zjsKNiIiIBdTdPO8o3IiIiFhE3c3zhi7iJyIiYiF1N3c9hRsRERGLuUt384LSRkLhRkRERHKtILWR0JwbERERyZWC1kZC4UZERERyrCC2kVC4ERERkRwriG0kFG5EREQkxwpiGwmFGxEREcmxgthGQuFGREREcqwgtpFQuBEREZEcK4htJBRuREREJFcKWhsJXcRPREREcq0gtZFQuBERERGXKChtJHRaSkRERNyKwo2IiIi4FYUbERERcSsKNyIiIuJWFG5ERETErSjciIiIiFtRuBERERG3onAjIiIibkXhRkRERNyKx12h2DAMAJKSkiyuRERERLIq9Xs79Xs8Mx4Xbs6ePQtAVFSUxZWIiIhIdp09e5bg4OBM97EZWYlAbiQlJYWjR49SvHhxbNf3ZhfATMdRUVEcPnyYoKAgq8vxePp5FCz6eRQ8+pkULHn18zAMg7NnzxIREYGXV+azajxu5MbLy4vIyEiryygUgoKC9D+KAkQ/j4JFP4+CRz+TgiUvfh43G7FJpQnFIiIi4lYUbkRERMStKNxIOn5+fowbNw4/Pz+rSxH08yho9PMoePQzKVgKws/D4yYUi4iIiHvTyI2IiIi4FYUbERERcSsKNyIiIuJWFG5ERETErSjciMOUKVNo3LgxxYsXp2zZsnTu3Jm9e/daXZYAzz//PDabjSFDhlhdikc7cuQIjzzyCCEhIQQEBFC7dm3++9//Wl2WR7Lb7YwZM4aKFSsSEBBA5cqVmTRpUpb6DknuffXVV3To0IGIiAhsNhvLli1zet4wDMaOHUt4eDgBAQG0adOG33//Pd/qU7gRhy+//JL+/fvzzTffsG7dOq5cucLdd9/N+fPnrS7No+3YsYPXXnuNOnXqWF2KRzt9+jQxMTEUKVKEVatW8csvvzB16lRKlixpdWke6YUXXmDu3LnMnj2bPXv28MILL/Diiy8ya9Ysq0vzCOfPn6du3bq8+uqrGT7/4osv8sorrzBv3jy+/fZbihYtStu2bbl06VK+1Kel4HJDJ0+epGzZsnz55Ze0bNnS6nI80rlz52jQoAFz5szh2WefpV69esyYMcPqsjzSiBEj2LJlC5s3b7a6FAHuu+8+QkNDefPNNx3bHnjgAQICAnjvvfcsrMzz2Gw2li5dSufOnQFz1CYiIoKnn36aYcOGAZCYmEhoaCgLFy6ke/fueV6TRm7khhITEwEoVaqUxZV4rv79+3PvvffSpk0bq0vxeCtWrKBRo0Z069aNsmXLUr9+fV5//XWry/JYzZs3Z8OGDfz2228A/PDDD3z99de0b9/e4spk//79JCQkOP1/Kzg4mKZNm7Jt27Z8qcHjGmdK1qSkpDBkyBBiYmKoVauW1eV4pA8//JCdO3eyY8cOq0sR4I8//mDu3LkMHTqU//znP+zYsYNBgwbh6+tLXFyc1eV5nBEjRpCUlES1atXw9vbGbrfz3HPP0aNHD6tL83gJCQkAhIaGOm0PDQ11PJfXFG4kQ/379+enn37i66+/troUj3T48GEGDx7MunXr8Pf3t7ocwQz8jRo1YvLkyQDUr1+fn376iXnz5incWOCjjz7i/fff54MPPqBmzZrs2rWLIUOGEBERoZ+H6LSUpDdgwAA+++wzNm7cSGRkpNXleKTvvvuOEydO0KBBA3x8fPDx8eHLL7/klVdewcfHB7vdbnWJHic8PJwaNWo4batevTqHDh2yqCLP9swzzzBixAi6d+9O7dq16dmzJ0899RRTpkyxujSPFxYWBsDx48edth8/ftzxXF5TuBEHwzAYMGAAS5cu5YsvvqBixYpWl+Sx7rzzTnbv3s2uXbsct0aNGtGjRw927dqFt7e31SV6nJiYmHSXRvjtt9+oUKGCRRV5tgsXLuDl5fwV5u3tTUpKikUVSaqKFSsSFhbGhg0bHNuSkpL49ttvadasWb7UoNNS4tC/f38++OADli9fTvHixR3nRoODgwkICLC4Os9SvHjxdHOdihYtSkhIiOZAWeSpp56iefPmTJ48mQcffJDt27czf/585s+fb3VpHqlDhw4899xzlC9fnpo1a/L9998zbdo0Hn30UatL8wjnzp1j3759jsf79+9n165dlCpVivLlyzNkyBCeffZZqlSpQsWKFRkzZgwRERGOFVV5zhD5f0CGtwULFlhdmhiG0apVK2Pw4MFWl+HRPv30U6NWrVqGn5+fUa1aNWP+/PlWl+SxkpKSjMGDBxvly5c3/P39jUqVKhmjRo0yLl++bHVpHmHjxo0Zfl/ExcUZhmEYKSkpxpgxY4zQ0FDDz8/PuPPOO429e/fmW326zo2IiIi4Fc25EREREbeicCMiIiJuReFGRERE3IrCjYiIiLgVhRsRERFxKwo3IiIi4lYUbkRERMStKNyIiEey2WwsW7bM6jJEJA8o3IhIvuvduzc2my3drV27dlaXJiJuQL2lRMQS7dq1Y8GCBU7b/Pz8LKpGRNyJRm5ExBJ+fn6EhYU53UqWLAmYp4zmzp1L+/btCQgIoFKlSixZssTp9bt37+aOO+4gICCAkJAQHn/8cc6dO+e0z1tvvUXNmjXx8/MjPDycAQMGOD3/119/cf/99xMYGEiVKlVYsWKF47nTp0/To0cPypQpQ0BAAFWqVEkXxkSkYFK4EZECacyYMTzwwAP88MMP9OjRg+7du7Nnzx4Azp8/T9u2bSlZsiQ7duzg448/Zv369U7hZe7cufTv35/HH3+c3bt3s2LFCm655RanY0yYMIEHH3yQH3/8kXvuuYcePXpw6tQpx/F/+eUXVq1axZ49e5g7dy6lS5fOv78AEcm5fGvRKSLy/+Li4gxvb2+jaNGiTrfnnnvOMAyzQ/0TTzzh9JqmTZsa/fr1MwzDMObPn2+ULFnSOHfunOP5zz//3PDy8jISEhIMwzCMiIgIY9SoUTesATBGjx7teHzu3DkDMFatWmUYhmF06NDB6NOnj2s+sIjkK825ERFL3H777cydO9dpW6lSpRz3mzVr5vRcs2bN2LVrFwB79uyhbt26FC1a1PF8TEwMKSkp7N27F5vNxtGjR7nzzjszraFOnTqO+0WLFiUoKIgTJ04A0K9fPx544AF27tzJ3XffTefOnWnevHmOPquI5C+FGxGxRNGiRdOdJnKVgICALO1XpEgRp8c2m42UlBQA2rdvz8GDB1m5ciXr1q3jzjvvpH///rz88ssur1dEXEtzbkSkQPrmm2/SPa5evToA1atX54cffuD8+fOO57ds2YKXlxdVq1alePHiREdHs2HDhlzVUKZMGeLi4njvvfeYMWMG8+fPz9X7iUj+0MiNiFji8uXLJCQkOG3z8fFxTNr9+OOPadSoEf/4xz94//332b59O2+++SYAPXr0YNy4ccTFxTF+/HhOnjzJwIED6dmzJ6GhoQCMHz+eJ554grJly9K+fXvOnj3Lli1bGDhwYJbqGzt2LA0bNqRmzZpcvnyZzz77zBGuRKRgU7gREUusXr2a8PBwp21Vq1bl119/BcyVTB9++CFPPvkk4eHhLFq0iBo1agAQGBjImjVrGDx4MI0bNyYwMJAHHniAadOmOd4rLi6OS5cuMX36dIYNG0bp0qXp2rVrluvz9fVl5MiRHDhwgICAAFq0aMGHH37ogk8uInnNZhiGYXURIiJp2Ww2li5dSufOna0uRUQKIc25EREREbeicCMiIiJuRXNuRKTA0dlyEckNjdyIiIiIW1G4EREREbeicCMiIiJuReFGRERE3IrCjYiIiLgVhRsRERFxKwo3IiIi4lYUbkRERMStKNyIiIiIW/k/BPLpH1Zbfp0AAAAASUVORK5CYII=", "text/plain": [ "
" ] @@ -831,7 +847,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABbR0lEQVR4nO3deXhMZ/sH8O9km+wRSWQTSYRaI/bUErRN36BNxR6UWFqllpBqUXsVfVGNrZRXUbWkNFQpGim1L0WQij2ESEIsiQQRk/P74/wyjEyWSSY5M5nv57rmmplnnnPOfWaGufOcZ5EJgiCAiIiIyIAYSR0AERERUUVjAkREREQGhwkQERERGRwmQERERGRwmAARERGRwWECRERERAaHCRAREREZHCZAREREZHCYABEREZHBYQJEpAUDBw6El5dXqbadPn06ZDKZdgPSMTdu3IBMJsOaNWsq9Lj79++HTCbD/v37lWUl/azKK2YvLy8MHDhQq/skIs0xAaJKTSaTlej26g8kUVkdOXIE06dPx6NHj6QOhYgKYSJ1AETlad26dSrPf/rpJ8TExBQor1evXpmOs3LlSuTl5ZVq28mTJ2PChAllOj6VXFk+q5I6cuQIZsyYgYEDB6JKlSoqr126dAlGRvzbk0hqTICoUvvwww9Vnh87dgwxMTEFyl/35MkTWFpalvg4pqampYoPAExMTGBiwn+KFaUsn5U2yOVySY+vL7Kzs2FlZSV1GFSJ8c8QMngdOnRAw4YNcerUKbRr1w6Wlpb48ssvAQC//fYb3nvvPbi5uUEul8PHxwczZ86EQqFQ2cfr/Ury+4/Mnz8fK1asgI+PD+RyOVq0aIGTJ0+qbKuuD5BMJsPIkSOxbds2NGzYEHK5HA0aNMDu3bsLxL9//340b94c5ubm8PHxwQ8//FDifkUHDx5Ez549UaNGDcjlcnh4eGDs2LF4+vRpgfOztrZGcnIyQkJCYG1tDScnJ4wbN67Ae/Ho0SMMHDgQdnZ2qFKlCsLCwkp0Keiff/6BTCbD2rVrC7y2Z88eyGQy7NixAwBw8+ZNfPrpp6hTpw4sLCzg4OCAnj174saNG8UeR10foJLGfO7cOQwcOBA1a9aEubk5XFxcMHjwYNy/f19ZZ/r06fj8888BAN7e3srLrPmxqesDdP36dfTs2RNVq1aFpaUl3nzzTezcuVOlTn5/pl9++QWzZs1C9erVYW5ujnfeeQdXr14t9rw1ec8ePXqEsWPHwsvLC3K5HNWrV8eAAQOQnp6urPPs2TNMnz4db7zxBszNzeHq6opu3brh2rVrKvG+fnlZXd+q/O/XtWvX0LlzZ9jY2KBfv34ASv4dBYCLFy+iV69ecHJygoWFBerUqYNJkyYBAPbt2weZTIatW7cW2G7Dhg2QyWQ4evRose8jVR78s5MIwP3799GpUyeEhobiww8/hLOzMwBgzZo1sLa2RkREBKytrfHXX39h6tSpyMzMxLx584rd74YNG/D48WN88sknkMlkmDt3Lrp164br168X2xJx6NAhREdH49NPP4WNjQ0WLVqE7t27IykpCQ4ODgCAM2fOoGPHjnB1dcWMGTOgUCjw1VdfwcnJqUTnvXnzZjx58gTDhw+Hg4MDTpw4gcWLF+P27dvYvHmzSl2FQoGgoCD4+/tj/vz52Lt3L7799lv4+Phg+PDhAABBENClSxccOnQIw4YNQ7169bB161aEhYUVG0vz5s1Rs2ZN/PLLLwXqR0VFwd7eHkFBQQCAkydP4siRIwgNDUX16tVx48YNLFu2DB06dMCFCxc0ar3TJOaYmBhcv34dgwYNgouLC/7991+sWLEC//77L44dOwaZTIZu3brh8uXL2LhxI7777js4OjoCQKGfSVpaGlq3bo0nT55g9OjRcHBwwNq1a/HBBx9gy5Yt6Nq1q0r9b775BkZGRhg3bhwyMjIwd+5c9OvXD8ePHy/yPEv6nmVlZSEgIAAJCQkYPHgwmjZtivT0dGzfvh23b9+Go6MjFAoF3n//fcTGxiI0NBTh4eF4/PgxYmJiEB8fDx8fnxK///levHiBoKAgtG3bFvPnz1fGU9Lv6Llz5xAQEABTU1MMHToUXl5euHbtGn7//XfMmjULHTp0gIeHB9avX1/gPV2/fj18fHzQqlUrjeMmPSYQGZARI0YIr3/t27dvLwAQli9fXqD+kydPCpR98skngqWlpfDs2TNlWVhYmODp6al8npiYKAAQHBwchAcPHijLf/vtNwGA8PvvvyvLpk2bViAmAIKZmZlw9epVZdnZs2cFAMLixYuVZcHBwYKlpaWQnJysLLty5YpgYmJSYJ/qqDu/OXPmCDKZTLh586bK+QEQvvrqK5W6TZo0EZo1a6Z8vm3bNgGAMHfuXGXZixcvhICAAAGAsHr16iLjmThxomBqaqrynuXk5AhVqlQRBg8eXGTcR48eFQAIP/30k7Js3759AgBh3759Kufy6melSczqjrtx40YBgHDgwAFl2bx58wQAQmJiYoH6np6eQlhYmPL5mDFjBADCwYMHlWWPHz8WvL29BS8vL0GhUKicS7169YScnBxl3YULFwoAhPPnzxc41qtK+p5NnTpVACBER0cXqJ+XlycIgiD8+OOPAgBhwYIFhdZR994Lwst/G6++r/nfrwkTJpQobnXf0Xbt2gk2NjYqZa/GIwji90sulwuPHj1Slt29e1cwMTERpk2bVuA4VLnxEhgRxH4ZgwYNKlBuYWGhfPz48WOkp6cjICAAT548wcWLF4vdb+/evWFvb698HhAQAEC85FGcwMBAlb+kGzVqBFtbW+W2CoUCe/fuRUhICNzc3JT1atWqhU6dOhW7f0D1/LKzs5Geno7WrVtDEAScOXOmQP1hw4apPA8ICFA5lz/++AMmJibKFiEAMDY2xqhRo0oUT+/evZGbm4vo6Ghl2Z9//olHjx6hd+/eauPOzc3F/fv3UatWLVSpUgWnT58u0bFKE/Orx3327BnS09Px5ptvAoDGx331+C1btkTbtm2VZdbW1hg6dChu3LiBCxcuqNQfNGgQzMzMlM9L+p0q6Xv266+/ws/Pr0ArCQDlZdVff/0Vjo6Oat+jskzp8OpnoC7uwr6j9+7dw4EDBzB48GDUqFGj0HgGDBiAnJwcbNmyRVkWFRWFFy9eFNsvkCofJkBEANzd3VV+VPL9+++/6Nq1K+zs7GBrawsnJyflf5QZGRnF7vf1/4zzk6GHDx9qvG3+9vnb3r17F0+fPkWtWrUK1FNXpk5SUhIGDhyIqlWrKvv1tG/fHkDB8zM3Ny9wGefVeACxn4mrqyusra1V6tWpU6dE8fj5+aFu3bqIiopSlkVFRcHR0RFvv/22suzp06eYOnUqPDw8IJfL4ejoCCcnJzx69KhEn8urNIn5wYMHCA8Ph7OzMywsLODk5ARvb28AJfs+FHZ8dcfKH5l48+ZNlfLSfqdK+p5du3YNDRs2LHJf165dQ506dbTaed/ExATVq1cvUF6S72h+8ldc3HXr1kWLFi2wfv16Zdn69evx5ptvlvjfDFUe7ANEBNW/MvM9evQI7du3h62tLb766iv4+PjA3Nwcp0+fxvjx40s0lNrY2FhtuSAI5bptSSgUCrz77rt48OABxo8fj7p168LKygrJyckYOHBggfMrLB5t6927N2bNmoX09HTY2Nhg+/bt6NOnj8qP7ahRo7B69WqMGTMGrVq1gp2dHWQyGUJDQ8t1iHuvXr1w5MgRfP7552jcuDGsra2Rl5eHjh07lvvQ+nyl/V5U9HtWWEvQ653m88nl8gLTA2j6HS2JAQMGIDw8HLdv30ZOTg6OHTuGJUuWaLwf0n9MgIgKsX//fty/fx/R0dFo166dsjwxMVHCqF6qVq0azM3N1Y4AKsmooPPnz+Py5ctYu3YtBgwYoCyPiYkpdUyenp6IjY1FVlaWSovKpUuXSryP3r17Y8aMGfj111/h7OyMzMxMhIaGqtTZsmULwsLC8O233yrLnj17VqqJB0sa88OHDxEbG4sZM2Zg6tSpyvIrV64U2Kcml4E8PT3Vvj/5l1g9PT1LvK+ilPQ98/HxQXx8fJH78vHxwfHjx5Gbm1toZ/78lqnX9/96i1ZRSvodrVmzJgAUGzcAhIaGIiIiAhs3bsTTp09hamqqcnmVDAcvgREVIv8v7Vf/sn7+/Dm+//57qUJSYWxsjMDAQGzbtg137txRll+9ehW7du0q0faA6vkJgoCFCxeWOqbOnTvjxYsXWLZsmbJMoVBg8eLFJd5HvXr14Ovri6ioKERFRcHV1VUlAc2P/fUWj8WLFxfauqCNmNW9XwAQGRlZYJ/589eUJCHr3LkzTpw4oTIEOzs7GytWrICXlxfq169f0lMpUknfs+7du+Ps2bNqh4vnb9+9e3ekp6erbTnJr+Pp6QljY2McOHBA5XVN/v2U9Dvq5OSEdu3a4ccff0RSUpLaePI5OjqiU6dO+Pnnn7F+/Xp07NhROVKPDAtbgIgK0bp1a9jb2yMsLAyjR4+GTCbDunXrtHYJShumT5+OP//8E23atMHw4cOhUCiwZMkSNGzYEHFxcUVuW7duXfj4+GDcuHFITk6Gra0tfv311xL1TypMcHAw2rRpgwkTJuDGjRuoX78+oqOjNe4f07t3b0ydOhXm5uYYMmRIgUsj77//PtatWwc7OzvUr18fR48exd69e5XTA5RHzLa2tmjXrh3mzp2L3NxcuLu7488//1TbItisWTMAwKRJkxAaGgpTU1MEBwerndhvwoQJ2LhxIzp16oTRo0ejatWqWLt2LRITE/Hrr79qbdbokr5nn3/+ObZs2YKePXti8ODBaNasGR48eIDt27dj+fLl8PPzw4ABA/DTTz8hIiICJ06cQEBAALKzs7F37158+umn6NKlC+zs7NCzZ08sXrwYMpkMPj4+2LFjB+7evVvimDX5ji5atAht27ZF06ZNMXToUHh7e+PGjRvYuXNngX8LAwYMQI8ePQAAM2fO1PzNpMqhwsedEUmosGHwDRo0UFv/8OHDwptvvilYWFgIbm5uwhdffCHs2bOn2KHV+UN9582bV2CfAFSG3BY2DH7EiBEFtn19CLUgCEJsbKzQpEkTwczMTPDx8RH+97//CZ999plgbm5eyLvw0oULF4TAwEDB2tpacHR0FD7++GPlcPvXhylbWVkV2F5d7Pfv3xf69+8v2NraCnZ2dkL//v2FM2fOlGgYfL4rV64IAAQAwqFDhwq8/vDhQ2HQoEGCo6OjYG1tLQQFBQkXL14s8P6UZBi8JjHfvn1b6Nq1q1ClShXBzs5O6Nmzp3Dnzp0Cn6kgCMLMmTMFd3d3wcjISGVIvLrP8Nq1a0KPHj2EKlWqCObm5kLLli2FHTt2qNTJP5fNmzerlKsbVq5OSd+z/Pdj5MiRgru7u2BmZiZUr15dCAsLE9LT05V1njx5IkyaNEnw9vYWTE1NBRcXF6FHjx7CtWvXlHXu3bsndO/eXbC0tBTs7e2FTz75RIiPjy/x90sQSv4dFQRBiI+PV34+5ubmQp06dYQpU6YU2GdOTo5gb28v2NnZCU+fPi3yfaPKSyYIOvTnLBFpRUhICP7991+1/VOIDN2LFy/g5uaG4OBgrFq1SupwSCLsA0Sk515fEuDKlSv4448/0KFDB2kCItJx27Ztw71791Q6VpPhYQsQkZ5zdXVVrk918+ZNLFu2DDk5OThz5gxq164tdXhEOuP48eM4d+4cZs6cCUdHx1JPXkmVAztBE+m5jh07YuPGjUhNTYVcLkerVq0we/ZsJj9Er1m2bBl+/vlnNG7cWGUxVjJMbAEiIiIig8M+QERERGRwmAARERGRwWEfIDXy8vJw584d2NjYlGllYyIiIqo4giDg8ePHcHNzK3YSUSZAaty5cwceHh5Sh0FERESlcOvWLVSvXr3IOkyA1LCxsQEgvoG2trYSR0NEREQlkZmZCQ8PD+XveFGYAKmRf9nL1taWCRAREZGeKUn3FXaCJiIiIoPDBIiIiIgMDhMgIiIiMjhMgIiIiMjgMAEiIiIig8MEiIiIiAwOEyAiIiIyOEyAiIiIyOAwASIiIiKDw5mgiYiIqEIoFMDBg0BKCuDqCgQEAMbG0sTCBIiIiIjKXXQ0EB4O3L79sqx6dWDhQqBbt4qPh5fAiIiIqFxFRwM9eqgmPwCQnCyWR0dXfExMgIiIiKjcKBRiy48gFHwtv2zMGLFeRWICREREROXm4MGCLT+vEgTg1i2xXkViAkRERETlJiVFu/W0hQkQERERlRtXV+3W0xaOAiMiItJxujR8XFMBAeJor+Rk9f2AZDLx9YCAio2LLUBEREQ6LDoa8PIC3noL6NtXvPfykmbkVGkYG4tD3QEx2XlV/vPIyIpP6JgAERER6ShdHD5eGt26AVu2AO7uquXVq4vlUswDJBMEdQ1Shi0zMxN2dnbIyMiAra2t1OEQEZEBUijElp7CRlDlXzpKTNSfy2HlfSlPk99v9gEiIiLSQZoMH+/QocLCKhNjY92JlZfAiIiIdJCuDh+vLJgAERER6SBdHT5eWfASGBERVVocPk6FkbwFaOnSpfDy8oK5uTn8/f1x4sSJQuvm5ubiq6++go+PD8zNzeHn54fdu3eXaZ9ERFQ5cfg4FUXSBCgqKgoRERGYNm0aTp8+DT8/PwQFBeHu3btq60+ePBk//PADFi9ejAsXLmDYsGHo2rUrzpw5U+p9EhFR5cPh41QcSYfB+/v7o0WLFliyZAkAIC8vDx4eHhg1ahQmTJhQoL6bmxsmTZqEESNGKMu6d+8OCwsL/Pzzz6XapzocBk9EpL84fNxwafL7LVkL0PPnz3Hq1CkEBga+DMbICIGBgTh69KjabXJycmBubq5SZmFhgUOHDpV6n/n7zczMVLkREZF+0tXVx8sif/h4nz7iPZOfspMsAUpPT4dCoYCzs7NKubOzM1JTU9VuExQUhAULFuDKlSvIy8tDTEwMoqOjkfL/YwBLs08AmDNnDuzs7JQ3Dw+PMp4dERFJhcPHqSQk7wStiYULF6J27dqoW7cuzMzMMHLkSAwaNAhGRmU7jYkTJyIjI0N5u3XrlpYiJiKiisbh41QSkiVAjo6OMDY2Rlpamkp5WloaXFxc1G7j5OSEbdu2ITs7Gzdv3sTFixdhbW2NmjVrlnqfACCXy2Fra6tyIyIi/ZQ/fPz1kVP5ZDLAw4PDxw2dZAmQmZkZmjVrhtjYWGVZXl4eYmNj0apVqyK3NTc3h7u7O168eIFff/0VXbp0KfM+iYiocuDwcSoJSS+BRUREYOXKlVi7di0SEhIwfPhwZGdnY9CgQQCAAQMGYOLEicr6x48fR3R0NK5fv46DBw+iY8eOyMvLwxdffFHifRIRUfEUCmD/fmDjRvFeoZA6Is1w+DgVR9KZoHv37o179+5h6tSpSE1NRePGjbF7925lJ+akpCSV/j3Pnj3D5MmTcf36dVhbW6Nz585Yt24dqlSpUuJ9EhFR0aKjgfBw1ZFU1auLrSr6lDh06wZ06cLh46SepPMA6SrOA0REhip/AsHXfxnyLx2x9YR0mV7MA0RERLpFoRBbftT9WZxfNmaM/l0OI1KHCRAREQGonBMIEhWGCRAREQHgBIJkWJgAERERAE4gSIaFCRAREQHgBIJkWJgAERERAE4gSIaFCRARESlxAkEyFJJOhEhERLqHEwiSIWACREREBRgbAx06SB0FUflhAkREpEUKBVtOiPQBEyAiIi2pLGtoERkCdoImItKC/DW0Xp9JOTlZLI+OliYuIlKPCRARURlxDS0i/cMEiIiojLiGFpH+YQJERFRGXEOLSP8wASIiKiOuoUWkf5gAERGVEdfQItI/TICIiMqIa2gR6R8mQEREWsA1tIj0CydCJCLSEq6hRaQ/mAAREWkR19AiKpwgAGlpQGIi4OICeHtLFwsTICLSCVxDi6hyyMoSE5zr19XfP30q1vvqK2DKFOniZAJERJLjGlpE+iM3V5zYs7AkJz296O3zR0WamVVMvIVhAkREkspfQ+v1ZSTy19BiB2KiiiUIwL17hSc4t24Vv6xL1ari5a2aNQve16ghffIDADJBULd6jWHLzMyEnZ0dMjIyYGtrK3U4RJWWQgF4eRW+jIRMJrYEJSbychiRNmVnAzduFH6ZKju76O3lcvHfrroEx9sbsLOriLMoSJPfb7YAEZFkNFlDix2LiUpOoRD/bRWW4KSlFb29TAa4uRWe4Li6AkZ6PpEOEyAikgzX0CIqHUEAHjwoPMG5eRN48aLofdjZFZ7geHoC5uYVcy5SYQJERJLhGlr0qtxccYTQkyea34rbLidHbNV49WZkpNnz0myj7X3m5IjJzfXrwOPHRb+fpqbiZarC+uLY21fIx6qzmAARkWTy19BKTi7YCRp42QeIa2hJS6HQLDEpbRJTXIsFFeTqWniC4+bGvnNFYQJERJLJX0OrRw8x2Xk1CeIaWtr35Alw/7546eTV+6LKMjLEVoeKJJMBlpaF3ywsin5d3S1/1FFenvg9y78V97wkdSpynyYm4iiqmjXF1h0Li4r9bCoTJkBEJKn8NbTUzQMUGckh8Oo8fy4mJ+oSmKKSmWfPyn5sdclHaRKSoraTywsuKkukbUyAiEhyhrqGlkIBPHpUeAJTWDKTlVX6Y5qYAA4O4jwtr96rK6taFahSBbCyEhMTc3P9H/lDlI8JEBHphMqyhlZWFnDunNhRtbhk5tEj9X2fSkImEzuxFpfEvF5mY8PWFSKACRARUamlpgJxccCZM+J9XBxw5YrmSY2tbclbZV5tmWFrDFHpMQEiIipGXh5w9WrBZCc1VX19NzfgjTcAR8fiExt7e3G4MhFVLCZARESvePYMiI9XTXbOnlW/NIBMBtSpAzRuDDRpIt43bgxUq1ahIRNRKTABIiKD9eDBy9ac/GQnIUH9Qo/m5kCjRqrJjq+v2EGYiPQPEyAiqvQEQeyU/Hqyk5Skvr6Dw8skJ//+jTfEEVREVDnwnzMRVSq5uWIrzuvJzqNH6uvXrPny0lV+suPuzpFSRJUdEyAi0luPH4v9c15NduLjxYkCX2dqCjRooJro+PmJC0ISkeFhAkREOk8QxBFXr47AOnNGHJmljq1twVad+vVfLodARMQEiIh0ikIhJjb5yU7+/d276uu7uxfsr+PlxTlyiKhoTICI9JxCoZ9LSAiCGHN8PPDvvy/vz58XF+18nZERULeuasuOnx/g5FTRkRNRZcAEiEiPRUerX0R04ULdWkT07l3VJCf/vrCOyRYWYnLzarLTsKG4HhURkTYwASLSU9HRQI8eBZddSE4Wy7dsqfgk6MEDMbF5PdlJT1df39gYqFVLTG4aNBDvGzYUh5zrQysWEekvmSCUdim+yiszMxN2dnbIyMiAra2t1OEQFaBQiP1cXm35eZVMJrYEJSaWTyKRmQlcuFCwRSclpfB4atZ8meTk39epA8jl2o+PiAyTJr/fbAEi0kMHDxae/ABiq9CtW2K9sqywnp0tzqnzapLz77+FTyAIADVqqCY5DRoA9erx8hUR6RYmQER6qLCWltLWe/YMuHSpYItOYmLhK5u7uakmOQ0aiEPN2WhKRPqACRCRHnJ1LV293Fzg8uWCLTpXrogrnqvj5KSa5OQ/trcv2zkQEUmJCRCRHgoIEPv4JCcX3kLj4iJ2Pp4582XCc/mymASpY29fsEWnQQOubE5ElZPkU4UtXboUXl5eMDc3h7+/P06cOFFk/cjISNSpUwcWFhbw8PDA2LFj8ezZM+Xr06dPh0wmU7nVrVu3vE+DqEIZG4tD3YuSmgr07AlMnQpERYlJUG4uYG0NvPkmMGQIsGAB8OefYiJ1/77YZ2jZMmDkSOCtt5j8EFHlJWkLUFRUFCIiIrB8+XL4+/sjMjISQUFBuHTpEqqp+Z93w4YNmDBhAn788Ue0bt0aly9fxsCBAyGTybBgwQJlvQYNGmDv3r3K5yZcwpkqobZtgX79gI0bxVFhr7OwEPvkvD7yysODC30SEUmaGSxYsAAff/wxBg0aBABYvnw5du7ciR9//BETJkwoUP/IkSNo06YN+vbtCwDw8vJCnz59cPz4cZV6JiYmcHFxKf8TIKpggvCylebXX19ezrKyEicOrFcPeP99wNdXHCbPuXSIiNST7BLY8+fPcerUKQQGBr4MxsgIgYGBOHr0qNptWrdujVOnTikvk12/fh1//PEHOnfurFLvypUrcHNzQ82aNdGvXz8kFTVmF0BOTg4yMzNVbkS65NEjYNEisRWnfXtg0yYx+WnZEvjxR3Gm5cOHgf/9DwgJAXx8mPwQERVFshag9PR0KBQKODs7q5Q7Ozvj4sWLarfp27cv0tPT0bZtWwiCgBcvXmDYsGH48ssvlXX8/f2xZs0a1KlTBykpKZgxYwYCAgIQHx8PGxsbtfudM2cOZsyYob2TI9ICQQBOngSWLxcTnqdPxXIrK/HS1yefAE2bShsjEZG+krwTtCb279+P2bNn4/vvv8fp06cRHR2NnTt3YubMmco6nTp1Qs+ePdGoUSMEBQXhjz/+wKNHj/DLL78Uut+JEyciIyNDebt161ZFnA6RWllZwMqVQPPmgL8/sHq1mPw0bAgsXSp2WP7hByY/RERlIVkLkKOjI4yNjZGWlqZSnpaWVmj/nSlTpqB///746KOPAAC+vr7Izs7G0KFDMWnSJBgZFcznqlSpgjfeeANXr14tNBa5XA455+MniZ0/L7b2rFsHPH4slsnlQK9ewLBhQKtW7LxMRKQtkrUAmZmZoVmzZoiNjVWW5eXlITY2Fq1atVK7zZMnTwokOcb/39GhsCXNsrKycO3aNbiWdOY4ogr07Bnw88/iiK5GjYDvvxeTn1q1gPnzxeUufvoJaN2ayQ8RkTZJOgosIiICYWFhaN68OVq2bInIyEhkZ2crR4UNGDAA7u7umDNnDgAgODgYCxYsQJMmTeDv74+rV69iypQpCA4OViZC48aNQ3BwMDw9PXHnzh1MmzYNxsbG6NOnj2TnSfS6K1fEy1irV4srqAOAiYnYgXnYMHEOHjUNmkREpCWSJkC9e/fGvXv3MHXqVKSmpqJx48bYvXu3smN0UlKSSovP5MmTIZPJMHnyZCQnJ8PJyQnBwcGYNWuWss7t27fRp08f3L9/H05OTmjbti2OHTsGJyenCj8/olfl5gLbt4uXuV6ZpgoeHsDQoeLEhGyoJCKqGDKhsGtHBiwzMxN2dnbIyMiALVd2pDJKShI7Nf/vf+LszIB4OatzZ7G1p1MnDlknItIGTX6/OUUyUTlQKIA9e8TWnp07Xy406uwMfPSRePPykjREIiKDxgSISItSU8WJCVesAG7efFn+9ttia0+XLoCZmXTxERGRiAkQURkJArB/v9jaEx0NvHghltvbA4MGif176tSRNEQiInoNEyCiUnrwAFi7Vkx8Ll9+Wd6qldja07OnuCApERHpHiZARBoQBOD4cXEx0l9+EefxAQBra6B/f3F5Cj8/aWMkIqLiMQEiKoHHj4H168XWnrNnX5b7+QHDhwN9+wKFLDVHREQ6iAkQURHOnhVbe9avF9foAgBzcyA0VLzM1bIlZ2gmItJHTIDIYCkUwMGDQEqKOAFhQIA4H8/Tp+LlreXLgWPHXtavU0dMesLCxA7ORESkv5gAkUGKjgbCw8W1tvI5OwMtWgCHDwMPH4plpqZAt25i4tO+PVt7iIgqCyZAZHCio4EePcQOza9KSwN27BAfe3mJHZoHDRITIyIiqlyYAJFBUSjElp+iFoBxdAQuXeKEhURElRnXmyaDcvCg6mUvddLTgSNHKiYeIiKSBhMgMih37pSsXkpK+cZBRETSYgJEBuPpU3FF9pJwdS3fWIiISFpMgMggpKWJC5Lu21d0PZkM8PAQh8QTEVHlxQSIKr34eMDfX5zTx94emDFDTHReH9Ke/zwyUpwPiIiIKi8mQFSp7d4NtG4N3LwJ1K4tJkFTpwJbtgDu7qp1q1cXy7t1kyZWIiKqOBwGT5XW0qXA6NFAXp44ieGvvwIODuJr3boBXbqonwmaiIgqPyZAVOm8eAFERACLF4vPBw4Efvih4Lw+xsZAhw4VHR0REekCJkBUqWRmiguV7tolPp8zBxg/nktYEBGRKiZAVGncvAm8/77Y6dnCAli3DujeXeqoiIhIFzEBokrh+HGxT09aGuDiAvz+O9C8udRRERGRruIoMNJ7v/wi9uVJSwP8/IATJ5j8EBFR0ZgAkd4SBODrr4HevYFnz8TLX4cOiRMZEhERFYUJEOmlnBwgLAyYMkV8PnYssG0bYG0taVhERKQn2AeI9E56OtC1q9jaY2wMLFkCDBsmdVRERKRPmACRXrl4UbzUde0aYGcHbN4MvPuu1FEREZG+YQJEeiM2FujRA3j0CPD2BnbsAOrXlzoqIiLSR+wDRHph5UqgY0cx+WnTRhz2zuSHiIhKiwkQ6TSFAhg3Dhg6VFziol8/YO9ewMlJ6siIiEif8RIY6aysLDHh2b5dfP7VV8DkyVzWgoiIyo4JEOmk27eB4GAgLg6Qy4E1a8Q1voiIiLSBCRDpnFOngA8+AO7cES91/fYb0KqV1FEREVFlwj5ApFO2bgXatROTnwYNxGUtmPwQEZG2MQEinSAIwNy54urtT54AQUHA4cOAl5fUkRERUWXEBIgk9/w58PHHwPjxYiI0YoQ4x4+dndSRERFRZcU+QCSphw/FVp99+wAjIyAyEhg1SuqoiIiosmMCRJK5ehV47z3g8mVxEdOoKKBzZ6mjIiIiQ8AEiCRx4IC4oOmDB0CNGuIlL19fqaMiIiJDwT5AVOHWrgUCA8Xkp2VLcVkLJj9ERFSRmABRhcnLA778Ehg4EMjNBXr2BPbvB1xcpI6MiIgMDRMgqhBPngC9ewNz5ojPJ00CNm0CLCykjYuIiAwT+wBRuUtJAbp0AU6eBExNgf/9DxgwQOqoiIjIkDEBonJ19qy4ptetW4CDgzjTc0CA1FEREZGh4yUwKjc7dgBt24rJT506wLFjTH6IiEg3MAEirRMEYOFC8bJXVhbw9tvA0aNArVpSR0ZERCRiAkRa9eKFuJTFmDHiqK+PPwZ27wbs7aWOjIiI6CX2ASKtycgAevUC/vwTkMmAefOAiAjxMRERkS5hAkRakZgIvP8+cOECYGkJbNggXgIjIiLSRUyAqMyOHAFCQoB79wA3N+D334GmTaWOioiIqHCS9wFaunQpvLy8YG5uDn9/f5w4caLI+pGRkahTpw4sLCzg4eGBsWPH4tmzZ2XaJ5Xehg1iJ+d794AmTYATJ5j8EBGR7pM0AYqKikJERASmTZuG06dPw8/PD0FBQbh7967a+hs2bMCECRMwbdo0JCQkYNWqVYiKisKXX35Z6n1S6QgCMH060K8fkJMjtgAdPAi4u0sdGRERUfFkgiAIUh3c398fLVq0wJIlSwAAeXl58PDwwKhRozBhwoQC9UeOHImEhATExsYqyz777DMcP34chw4dKtU+1cnMzISdnR0yMjJga2tb1tOsdHJzgbAwYONG8fnnnwPffAMYSd6eSEREhkyT32/JfrKeP3+OU6dOITAw8GUwRkYIDAzE0aNH1W7TunVrnDp1SnlJ6/r16/jjjz/QuXPnUu8TAHJycpCZmalyo8J9/72Y/JiYACtXAnPnMvkhIiL9Ilkn6PT0dCgUCjg7O6uUOzs74+LFi2q36du3L9LT09G2bVsIgoAXL15g2LBhyktgpdknAMyZMwczZswo4xkZBoUCiIwUH3/3HfDRR5KGQ0REVCp69Xf7/v37MXv2bHz//fc4ffo0oqOjsXPnTsycObNM+504cSIyMjKUt1u3bmkp4spn2zbgxg1xXa/Bg6WOhoiIqHQkawFydHSEsbEx0tLSVMrT0tLg4uKidpspU6agf//++Oj/mx18fX2RnZ2NoUOHYtKkSaXaJwDI5XLI5fIynpFh+O478X7YMHG+HyIiIn2kcQuQl5cXvvrqKyQlJZXpwGZmZmjWrJlKh+a8vDzExsaiVatWard58uQJjF7rbGJsbAwAEAShVPukkjt+HDh8GDA1FZe7ICIi0lcaJ0BjxoxBdHQ0atasiXfffRebNm1CTk5OqQ4eERGBlStXYu3atUhISMDw4cORnZ2NQYMGAQAGDBiAiRMnKusHBwdj2bJl2LRpExITExETE4MpU6YgODhYmQgVt08qvfzWn759AVdXaWMhIiIqE6GUTp06JYwaNUpwdHQU7O3thREjRginTp3SeD+LFy8WatSoIZiZmQktW7YUjh07pnytffv2QlhYmPJ5bm6uMH36dMHHx0cwNzcXPDw8hE8//VR4+PBhifdZEhkZGQIAISMjQ+Pzqaxu3hQEY2NBAAQhLk7qaIiIiArS5Pe7zPMA5ebm4vvvv8f48eORm5sLX19fjB49GoMGDYJMT1fB5DxABX3+OTB/vjjr8ytXGImIiHSGJr/fpe4EnZubi61bt2L16tWIiYnBm2++iSFDhuD27dv48ssvsXfvXmzYsKG0uycd8vgxsGKF+HjsWGljISIi0gaNE6DTp09j9erV2LhxI4yMjDBgwAB89913qFu3rrJO165d0aJFC60GStJZvRrIzATeeAP4/zkniYiI9JrGCVCLFi3w7rvvYtmyZQgJCYGpqWmBOt7e3ggNDdVKgCStVyc+HDuWMz4TEVHloHECdP36dXh6ehZZx8rKCqtXry51UKQ7tm8HEhOBqlWBAQOkjoaIiEg7NP57/u7duzh+/HiB8uPHj+Off/7RSlCkOxYsEO858SEREVUmGidAI0aMULtURHJyMkZwdrxK5eRJ4NAhTnxIRESVj8YJ0IULF9C0adMC5U2aNMGFCxe0EhTphvyJD0NDATc3aWMhIiLSJo0TILlcXmCtLQBISUmBiYlkS4uRlt26Bfzyi/iYQ9+JiKiy0TgB+s9//qNcPT3fo0eP8OWXX+Ldd9/VanAknSVLxBFgHToATZpIHQ0REZF2adxkM3/+fLRr1w6enp5o8v+/jHFxcXB2dsa6deu0HiBVvKws4IcfxMcREdLGQkREVB40ToDc3d1x7tw5rF+/HmfPnoWFhQUGDRqEPn36qJ0TiPTP6tVARgZQuzbw3ntSR0NERKR9peq0Y2VlhaFDh2o7FtIBCgWwcKH4eMwYTnxIRESVU6l7LV+4cAFJSUl4/vy5SvkHH3xQ5qBIOr//Dly7BtjbA2FhUkdDRERUPko1E3TXrl1x/vx5yGQy5C8mn7/yu0Kh0G6EVKHyh74PGwZYWUkbCxERUXnR+AJHeHg4vL29cffuXVhaWuLff//FgQMH0Lx5c+zfv78cQqSK8s8/wIEDgIkJJz4kIqLKTeMWoKNHj+Kvv/6Co6MjjIyMYGRkhLZt22LOnDkYPXo0zpw5Ux5xUgV4deJDd3dpYyEiIipPGrcAKRQK2NjYAAAcHR1x584dAICnpycuXbqk3eiowty+zYkPiYjIcGjcAtSwYUOcPXsW3t7e8Pf3x9y5c2FmZoYVK1agZs2a5REjVYAlS4AXL4D27QE1K50QERFVKhonQJMnT0Z2djYA4KuvvsL777+PgIAAODg4ICoqSusBUvnjxIdERGRoNE6AgoKClI9r1aqFixcv4sGDB7C3t1eOBCP9snYt8OgRUKsW8P77xddXKICDB4GUFMDVFQgIAIyNyz1MIiIirdGoD1Bubi5MTEwQHx+vUl61alUmP3oqLw+IjBQfl2Tiw+howMsLeOstoG9f8d7LSywnIiLSFxolQKampqhRowbn+qlEduwArl4FqlQpfuLD6GigRw+xw/SrkpPFciZBRESkLzQeBTZp0iR8+eWXePDgQXnEQxVswQLx/pNPAGvrwuspFEB4OPD/816qyC8bM0asR0REpOs07gO0ZMkSXL16FW5ubvD09ITVa9MFnz59WmvBUfk6fRr4+29x4sORI4uue/BgwZafVwkCcOuWWK9DB62GSUREpHUaJ0AhISHlEAZJIX/iw169gOrVi66bklKyfZa0HhERkZQ0ToCmTZtWHnFQBUtOBjZtEh+XZOJDV9eS7bek9YiIiKSkcR8gqhyWLhUnPmzXDmjevPj6AQFiK1Fhg/1kMsDDQ6xHRESk6zROgIyMjGBsbFzojXRfdjawfLn4uKTLXhgbAwsXio9fT4Lyn0dGcj4gIiLSDxpfAtu6davK89zcXJw5cwZr167FjBkztBYYlZ+1a4GHDwEfHyA4uOTbdesGbNkijgZ7tUN09epi8tOtm9ZDJSIiKhcyQVA3sFlzGzZsQFRUFH777Tdt7E5SmZmZsLOzQ0ZGBmxtbaUOR6vy8oC6dYErV4BFi4BRozTfB2eCJiIiXaTJ77fWEqDr16+jUaNGyMrK0sbuJFWZE6Dffwc++ACwsxNbcYqa+4eIiEifaPL7rZVO0E+fPsWiRYvg7u6ujd1ROcof+l7cxIdERESVmcZ9gF5f9FQQBDx+/BiWlpb4+eeftRocadeZM8C+feLlquImPiQiIqrMNE6AvvvuO5UEyMjICE5OTvD394e9vb1WgyPtenXiQw8PaWMhIiKSktb6AFUmlbEP0J074qrtubnAyZMlm/uHiIhIn5RrH6DVq1dj8+bNBco3b96MtWvXaro7qiBLl4rJT9u2TH6IiIg0ToDmzJkDR0fHAuXVqlXD7NmztRIUadeTJy8nPoyIkDYWIiIiXaBxApSUlARvb+8C5Z6enkhKStJKUKRdP/0EPHgA1KwpDoEnIiIydBonQNWqVcO5c+cKlJ89exYODg5aCYq0Jy/vZefn8HBOWEhERASUIgHq06cPRo8ejX379kGhUEChUOCvv/5CeHg4QkNDyyNGKoNdu4DLl8WJDwcNkjoaIiIi3aDxMPiZM2fixo0beOedd2BiIm6el5eHAQMGsA+QDlqwQLz/+GPAxkbaWIiIiHRFqYfBX7lyBXFxcbCwsICvry88PT21HZtkKssw+Lg4oEkT8bLX9etAjRpSR0RERFR+NPn91rgFKF/t2rVRu3bt0m5OFSAyUrzv2ZPJDxER0as07gPUvXt3/Pe//y1QPnfuXPTs2VMrQVHZpaQAGzaIj8eOlTYWIiIiXaNxAnTgwAF07ty5QHmnTp1w4MABrQRFZff99+LEh23aAC1bSh0NERGRbtE4AcrKyoKZmVmBclNTU2RmZmolKCqbJ0+AZcvEx2z9ISIiKkjjBMjX1xdRUVEFyjdt2oT69etrJSgqm3XrgPv3AW9vICRE6miIiIh0j8adoKdMmYJu3brh2rVrePvttwEAsbGx2LBhA7Zs2aL1AEkzeXkvOz9z4kMiIiL1NE6AgoODsW3bNsyePRtbtmyBhYUF/Pz88Ndff6Fq1arlESNpYPdu4OJFwNYWGDxY6miIiIh0k8aXwADgvffew+HDh5GdnY3r16+jV69eGDduHPz8/EoVxNKlS+Hl5QVzc3P4+/vjxIkThdbt0KEDZDJZgdt7772nrDNw4MACr3fs2LFUsekbTnxIRERUvFIlQIA4GiwsLAxubm749ttv8fbbb+PYsWMa7ycqKgoRERGYNm0aTp8+DT8/PwQFBeHu3btq60dHRyMlJUV5i4+Ph7GxcYEh+B07dlSpt3HjxlKdpz45dw6IjQWMjIBRo6SOhoiISHdpdAksNTUVa9aswapVq5CZmYlevXohJycH27ZtK3UH6AULFuDjjz/GoP9fqGr58uXYuXMnfvzxR0yYMKFA/dcvs23atAmWlpYFEiC5XA4XF5dSxaSv8hc97dEDqEQTcxMREWldiVuAgoODUadOHZw7dw6RkZG4c+cOFi9eXKaDP3/+HKdOnUJgYODLgIyMEBgYiKNHj5ZoH6tWrUJoaCisrKxUyvfv349q1aqhTp06GD58OO7fv1+mWHVdaurLiQ8jIqSNhYiISNeVuAVo165dGD16NIYPH661JTDS09OhUCjg7OysUu7s7IyLFy8Wu/2JEycQHx+PVatWqZR37NgR3bp1g7e3N65du4Yvv/wSnTp1wtGjR2GsZlhUTk4OcnJylM/1cT6j778Hnj8HWrUC/P2ljoaIiEi3lbgF6NChQ3j8+DGaNWsGf39/LFmyBOnp6eUZW7FWrVoFX19ftHxtquPQ0FB88MEH8PX1RUhICHbs2IGTJ09i//79avczZ84c2NnZKW8eHh4VEL32PH36cuJDtv4QEREVr8QJ0JtvvomVK1ciJSUFn3zyCTZt2gQ3Nzfk5eUhJiYGjx8/1vjgjo6OMDY2Rlpamkp5Wlpasf13srOzsWnTJgwZMqTY49SsWROOjo64evWq2tcnTpyIjIwM5e3WrVslPwkd8PPPQHo64OXFiQ+JiIhKQuNRYFZWVhg8eDAOHTqE8+fP47PPPsM333yDatWq4YMPPtBoX2ZmZmjWrBliY2OVZXl5eYiNjUWrVq2K3Hbz5s3IycnBhx9+WOxxbt++jfv378PV1VXt63K5HLa2tio3fSEILzs/jx4NmGg8sxMREZHhKfUweACoU6cO5s6di9u3b5d6mHlERARWrlyJtWvXIiEhAcOHD0d2drZyVNiAAQMwceLEAtutWrUKISEhcHBwUCnPysrC559/jmPHjuHGjRuIjY1Fly5dUKtWLQQFBZUqRl22Zw+QkCDO+VOCxjAiIiJCKWaCVsfY2BghISEIKcX1l969e+PevXuYOnUqUlNT0bhxY+zevVvZMTopKQlGRqp52qVLl3Do0CH8+eefamM5d+4c1q5di0ePHsHNzQ3/+c9/MHPmTMjl8lKdny7Ln/jwo4/E2Z+JiIioeDJBEASpg9A1mZmZsLOzQ0ZGhk5fDjt/HmjUSJz48No1sQ8QERGRodLk97tMl8BIWvmLnnbvzuSHiIhIE0yA9FRamjj6CwDGjpU2FiIiIn3DBEhPLVsmTnz45pvi5IdERERUckyA9NDTp+LMzwAnPiQiIioNJkB6aP164N49ccHTrl2ljoaIiEj/MAHSM5z4kIiIqOyYAOmZP/8ELlwArK058SEREVFpMQHSM69OfGhnJ20sRERE+ooJkB6JjxdbgIyMxMtfREREVDpMgPRI/sSHXbsC3t6ShkJERKTXmADpibt3X058yKHvREREZcMESE8sWwbk5AAtW3LiQyIiorJiAqQHnj0Dli4VH0dEADKZtPEQERHpOyZAemDDBnHiwxo1xIVPiYiIqGyYAOk4QXg59H3UKE58SEREpA1MgHTc3r3Av/+KEx9+9JHU0RAREVUOTIB0XH7rz5AhQJUqkoZCRERUaTAB0mEXLgC7d4udnjnxIRERkfYwAdJhr058WLOmpKEQERFVKkyAdNS9e8BPP4mPx46VNhYiIqLKhgmQjlq+XJz4sEULoE0bqaMhIiKqXJgA6aBnz4AlS8THnPiQiIhI+5gA6aCNG8W1v6pX58SHRERE5YEJkI4RBOC778THo0cDpqbSxkNERFQZMQHSMbGxwPnzgJUV8PHHUkdDRERUOTEB0jH5Ex8OHsyJD4mIiMoLEyAdkpAA7NoldnoOD5c6GiIiosqLCZAOyZ/4sEsXwMdH0lCIiIgqNSZAOiI9/eXEhxER0sZCRERU2TEB0hHLl4vz/zRvDrRtK3U0RERElRsTIB2Qk/Ny4sOxYznxIRERUXljAqQDNm0C0tIAd3egZ0+poyEiIqr8mABJTBBeDn0fNYoTHxIREVUEJkAS27cPOHcOsLQEhg6VOhoiIiLDwARIYq9OfGhvL20sREREhoIJkIQuXgR27uTEh0RERBWNCZCEFi4U7z/4AKhVS9pYiIiIDAkTIIncvw+sXSs+5sSHREREFYsJkER++AF4+hRo2hQICJA6GiIiIsPCBEgCOTnA4sXi44gITnxIRERU0ZgASSAqCkhNBdzcOPEhERGRFJgAVTBBAL77Tnw8ahRgZiZtPERERIaICVAF278fiIvjxIdERERSYgJUwfInPhw4EKhaVdJQiIiIDBYToAp0+TKwYwcnPiQiIpIaE6AKFBkp3gcHA2+8IWkoREREBs1E6gAMyWefASYmHPlFREQkNSZAFcjHB1i0SOooiIiIiJfAiIiIyOAwASIiIiKDoxMJ0NKlS+Hl5QVzc3P4+/vjxIkThdbt0KEDZDJZgdt7772nrCMIAqZOnQpXV1dYWFggMDAQV65cqYhTISIiIj0geQIUFRWFiIgITJs2DadPn4afnx+CgoJw9+5dtfWjo6ORkpKivMXHx8PY2Bg9X+lZPHfuXCxatAjLly/H8ePHYWVlhaCgIDx79qyiTouIiIh0mEwQBEHKAPz9/dGiRQssWbIEAJCXlwcPDw+MGjUKEyZMKHb7yMhITJ06FSkpKbCysoIgCHBzc8Nnn32GcePGAQAyMjLg7OyMNWvWIDQ0tNh9ZmZmws7ODhkZGbC1tS3bCRIREVGF0OT3W9IWoOfPn+PUqVMIDAxUlhkZGSEwMBBHjx4t0T5WrVqF0NBQWFlZAQASExORmpqqsk87Ozv4+/sXus+cnBxkZmaq3IiIiKjykjQBSk9Ph0KhgLOzs0q5s7MzUlNTi93+xIkTiI+Px0cffaQsy99Ok33OmTMHdnZ2ypuHh4emp0JERER6RPI+QGWxatUq+Pr6omXLlmXaz8SJE5GRkaG83bp1S0sREhERkS6SNAFydHSEsbEx0tLSVMrT0tLg4uJS5LbZ2dnYtGkThgwZolKev50m+5TL5bC1tVW5ERERUeUlaQJkZmaGZs2aITY2VlmWl5eH2NhYtGrVqshtN2/ejJycHHz44Ycq5d7e3nBxcVHZZ2ZmJo4fP17sPomIiMgwSL4URkREBMLCwtC8eXO0bNkSkZGRyM7OxqBBgwAAAwYMgLu7O+bMmaOy3apVqxASEgIHBweVcplMhjFjxuDrr79G7dq14e3tjSlTpsDNzQ0hISEVdVpERESkwyRPgHr37o179+5h6tSpSE1NRePGjbF7925lJ+akpCQYGak2VF26dAmHDh3Cn3/+qXafX3zxBbKzszF06FA8evQIbdu2xe7du2Fubl7u50NERES6T/J5gHQR5wEiIiLSP3ozDxARERGRFJgAERERkcFhAkREREQGhwkQERERGRwmQERERGRwmAARERGRwWECRERERAaHCRAREREZHCZAREREZHCYABEREZHBYQJEREREBocJEBERERkcJkBERERkcJgAERERkcFhAkREREQGhwkQERERGRwmQERERGRwmAARERGRwWECRERERAaHCRAREREZHCZAREREZHCYABEREZHBYQJEREREBocJEBERERkcJkBERERkcJgAERERkcFhAkREREQGhwkQERERGRwmQERERGRwmAARERGRwWECRERERAaHCRAREREZHCZAREREZHCYABEREZHBYQJEREREBsdE6gCIiMiwKBQK5ObmSh0G6SFTU1MYGxtrZV9MgIiIqEIIgoDU1FQ8evRI6lBIj1WpUgUuLi6QyWRl2g8TICIiqhD5yU+1atVgaWlZ5h8wMiyCIODJkye4e/cuAMDV1bVM+2MCRERE5U6hUCiTHwcHB6nDIT1lYWEBALh79y6qVatWpsth7ARNRETlLr/Pj6WlpcSRkL7L/w6VtR8ZEyAiIqowvOxFZaWt7xATICIiogrk5eWFyMjIEtffv38/ZDIZO49rGfsAERGR3lAogIMHgZQUwNUVCAgAtDQquoDiWhqmTZuG6dOna7zfkydPwsrKqsT1W7dujZSUFNjZ2Wl8LCocEyAiItIL0dFAeDhw+/bLsurVgYULgW7dtH+8lJQU5eOoqChMnToVly5dUpZZW1srHwuCAIVCAROT4n9WnZycNIrDzMwMLi4uGm1DxeMlMCIi0nnR0UCPHqrJDwAkJ4vl0dHaP6aLi4vyZmdnB5lMpnx+8eJF2NjYYNeuXWjWrBnkcjkOHTqEa9euoUuXLnB2doa1tTVatGiBvXv3quz39UtgMpkM//vf/9C1a1dYWlqidu3a2L59u/L11y+BrVmzBlWqVMGePXtQr149WFtbo2PHjioJ24sXLzB69GhUqVIFDg4OGD9+PMLCwhASElLo+d6/fx99+vSBu7s7LC0t4evri40bN6rUycvLw9y5c1GrVi3I5XLUqFEDs2bNUr5++/Zt9OnTB1WrVoWVlRWaN2+O48ePl+LdL39MgIiISKcpFGLLjyAUfC2/bMwYsV5FmzBhAr755hskJCSgUaNGyMrKQufOnREbG4szZ86gY8eOCA4ORlJSUpH7mTFjBnr16oVz586hc+fO6NevHx48eFBo/SdPnmD+/PlYt24dDhw4gKSkJIwbN075+n//+1+sX78eq1evxuHDh5GZmYlt27YVGcOzZ8/QrFkz7Ny5E/Hx8Rg6dCj69++PEydOKOtMnDgR33zzDaZMmYILFy5gw4YNcHZ2BgBkZWWhffv2SE5Oxvbt23H27Fl88cUXyMvLK8E7KQGBCsjIyBAACBkZGVKHQkRUKTx9+lS4cOGC8PTpU4233bdPEMRUp+jbvn1aD1tp9erVgp2d3Ssx7RMACNu2bSt22wYNGgiLFy9WPvf09BS+++475XMAwuTJk5XPs7KyBADCrl27VI718OFDZSwAhKtXryq3Wbp0qeDs7Kx87uzsLMybN0/5/MWLF0KNGjWELl26lPSUBUEQhPfee0/47LPPBEEQhMzMTEEulwsrV65UW/eHH34QbGxshPv372t0DE0V9V3S5PebfYCIiEinvXJlRyv1tKl58+Yqz7OysjB9+nTs3LkTKSkpePHiBZ4+fVpsC1CjRo2Uj62srGBra6uc8VgdS0tL+Pj4KJ+7uroq62dkZCAtLQ0tW7ZUvm5sbIxmzZoV2RqjUCgwe/Zs/PLLL0hOTsbz58+Rk5OjnHcnISEBOTk5eOedd9RuHxcXhyZNmqBq1apFnquuYAJEREQ6raQrHpRxZYRSeX0017hx4xATE4P58+ejVq1asLCwQI8ePfD8+fMi92NqaqryXCaTFZmsqKsvqLtGqIF58+Zh4cKFiIyMhK+vL6ysrDBmzBhl7PmzMBemuNd1DfsAERGRTgsIEEd7FTYqXSYDPDzEelI7fPgwBg4ciK5du8LX1xcuLi64ceNGhcZgZ2cHZ2dnnDx5UlmmUChw+vTpIrc7fPgwunTpgg8//BB+fn6oWbMmLl++rHy9du3asLCwQGxsrNrtGzVqhLi4uCL7LukSyROgpUuXwsvLC+bm5vD391fpbKXOo0ePMGLECLi6ukIul+ONN97AH3/8oXx9+vTpkMlkKre6deuW92kQEVE5MTYWh7oDBZOg/OeRkeU3H5AmateujejoaMTFxeHs2bPo27evJJ2AR40ahTlz5uC3337DpUuXEB4ejocPHxY5t1Ht2rURExODI0eOICEhAZ988gnS0tKUr5ubm2P8+PH44osv8NNPP+HatWs4duwYVq1aBQDo06cPXFxcEBISgsOHD+P69ev49ddfcfTo0XI/39KQNAGKiopCREQEpk2bhtOnT8PPzw9BQUGFXvd8/vw53n33Xdy4cQNbtmzBpUuXsHLlSri7u6vUa9CgAVJSUpS3Q4cOVcTpEBFROenWDdiyBXjtv3tUry6Wl8c8QKWxYMEC2Nvbo3Xr1ggODkZQUBCaNm1a4XGMHz8effr0wYABA9CqVStYW1sjKCgI5ubmhW4zefJkNG3aFEFBQejQoYMymXnVlClT8Nlnn2Hq1KmoV68eevfurfzNNjMzw59//olq1aqhc+fO8PX1xTfffFOmBUvLk0wo60XDMvD390eLFi2wZMkSAOL8Ah4eHhg1ahQmTJhQoP7y5csxb948XLx4scD1z3zTp0/Htm3bEBcXV+q4MjMzYWdnh4yMDNja2pZ6P0REJHr27BkSExPh7e1d5I9wcSpyJujKJC8vD/Xq1UOvXr0wc+ZMqcMpk6K+S5r8fkvWAvT8+XOcOnUKgYGBL4MxMkJgYGChzWXbt29Hq1atMGLECDg7O6Nhw4aYPXs2FK9N/nDlyhW4ubmhZs2a6NevX7G973NycpCZmalyIyIi3WNsDHToAPTpI94z+VHv5s2bWLlyJS5fvozz589j+PDhSExMRN++faUOTWdIlgClp6dDoVAoJ1DK5+zsjNTUVLXbXL9+HVu2bIFCocAff/yBKVOm4Ntvv8XXX3+trOPv7481a9Zg9+7dWLZsGRITExEQEIDHjx8XGsucOXNgZ2envHl4eGjnJImIiCRgZGSENWvWoEWLFmjTpg3Onz+PvXv3ol69elKHpjP0ahh8Xl4eqlWrhhUrVijnNEhOTsa8efMwbdo0AECnTp2U9Rs1agR/f394enril19+wZAhQ9Tud+LEiYiIiFA+z8zMZBJERER6y8PDA4cPH5Y6DJ0mWQLk6OgIY2NjlR7mAJCWllboom+urq4wNTVV6VBVr149pKam4vnz5zAzMyuwTZUqVfDGG2/g6tWrhcYil8shl8tLeSZERESkbyS7BGZmZoZmzZqpzCeQl5eH2NhYtGrVSu02bdq0wdWrV1WGFF6+fBmurq5qkx9AnJXz2rVrcJVihiwiIiLSSZIOg4+IiMDKlSuxdu1aJCQkYPjw4cjOzsagQYMAAAMGDMDEiROV9YcPH44HDx4gPDwcly9fxs6dOzF79myMGDFCWWfcuHH4+++/cePGDRw5cgRdu3aFsbEx+vTpU+HnR0RERLpJ0j5AvXv3xr179zB16lSkpqaicePG2L17t7JjdFJSEoyMXuZoHh4e2LNnD8aOHYtGjRrB3d0d4eHhGD9+vLLO7du30adPH9y/fx9OTk5o27Ytjh07Bicnpwo/PyIiItJNks4DpKs4DxARkXZpax4gIr2fB4iIiIhIKkyAiIiIylGHDh0wZswY5XMvLy9ERkYWuY1MJsO2bdvKfGxt7acyYgJERESkRnBwMDp27Kj2tYMHD0Imk+HcuXMa7/fkyZMYOnRoWcNTMX36dDRu3LhAeUpKisr8ePQSEyAiIiI1hgwZgpiYGNy+fbvAa6tXr0bz5s3RqFEjjffr5OQES0tLbYRYLBcXF85zVwgmQERERGq8//77cHJywpo1a1TKs7KysHnzZgwZMgT3799Hnz594O7uDktLS/j6+mLjxo1F7vf1S2BXrlxBu3btYG5ujvr16yMmJqbANuPHj8cbb7wBS0tL1KxZE1OmTEFubi4AYM2aNZgxYwbOnj0LmUwGmUymjPn1S2Dnz5/H22+/DQsLCzg4OGDo0KHIyspSvj5w4ECEhIRg/vz5cHV1hYODA0aMGKE8ljrXrl1Dly5d4OzsDGtra7Ro0QJ79+5VqZOTk4Px48fDw8MDcrkctWrVwqpVq5Sv//vvv3j//fdha2sLGxsbBAQE4Nq1a0W+j2WlV0thEBFR5SEIwJMnFX9cS0tAJiu+nomJCQYMGIA1a9Zg0qRJkP3/Rps3b4ZCoUCfPn2QlZWFZs2aYfz48bC1tcXOnTvRv39/+Pj4oGXLlsUeIy8vD926dYOzszOOHz+OjIwMlf5C+WxsbLBmzRq4ubnh/Pnz+Pjjj2FjY4MvvvgCvXv3Rnx8PHbv3q1MPOzs7ArsIzs7G0FBQWjVqhVOnjyJu3fv4qOPPsLIkSNVkrx9+/bB1dUV+/btw9WrV9G7d280btwYH3/8sdpzyMrKQufOnTFr1izI5XL89NNPCA4OxqVLl1CjRg0A4rx+R48exaJFi+Dn54fExESkp6cDAJKTk9GuXTt06NABf/31F2xtbXH48GG8ePGi2PevTAQqICMjQwAgZGRkaHW/L14Iwr59grBhg3j/4oVWd09EpLOePn0qXLhwQXj69KmyLCtLEMQ0qGJvWVkljzshIUEAIOzbt09ZFhAQIHz44YeFbvPee+8Jn332mfJ5+/bthfDwcOVzT09P4bvvvhMEQRD27NkjmJiYCMnJycrXd+3aJQAQtm7dWugx5s2bJzRr1kz5fNq0aYKfn1+Beq/uZ8WKFYK9vb2Q9cobsHPnTsHIyEhITU0VBEEQwsLCBE9PT+HFKz9QPXv2FHr37l1oLOo0aNBAWLx4sSAIgnDp0iUBgBATE6O27sSJEwVvb2/h+fPnJdq3uu9SPk1+v3kJrIJERwNeXsBbbwF9+4r3Xl5iORER6aa6deuidevW+PHHHwEAV69excGDB5WLaysUCsycORO+vr6oWrUqrK2tsWfPHiQlJZVo/wkJCfDw8ICbm5uyTN1yUFFRUWjTpg1cXFxgbW2NyZMnl/gYrx7Lz88PVlZWyrI2bdogLy8Ply5dUpY1aNBAZc1NV1dX3L17t9D9ZmVlYdy4cahXrx6qVKkCa2trJCQkKOOLi4uDsbEx2rdvr3b7uLg4BAQEwNTUVKPzKSteAqsA0dFAjx7i3x6vSk4Wy7dsAbp1kyY2IiKpWFoCr3Q/qdDjamLIkCEYNWoUli5ditWrV8PHx0f5Yz5v3jwsXLgQkZGR8PX1hZWVFcaMGYPnz59rLd6jR4+iX79+mDFjBoKCgmBnZ4dNmzbh22+/1doxXvV6IiKTyVTW4HzduHHjEBMTg/nz56NWrVqwsLBAjx49lO+BhYVFkccr7vXywgSonCkUQHh4weQHEMtkMmDMGKBLF+CVhJuIqNKTyYBXGiN0Vq9evRAeHo4NGzbgp59+wvDhw5X9gQ4fPowuXbrgww8/BCD26bl8+TLq169fon3Xq1cPt27dQkpKinLR7mPHjqnUOXLkCDw9PTFp0iRl2c2bN1XqmJmZQaFQFHusNWvWIDs7W9kKdPjwYRgZGaFOnToliledw4cPY+DAgejatSsAsUXoxo0bytd9fX2Rl5eHv//+G4GBgQW2b9SoEdauXYvc3NwKbQXiJbBydvAgoGYEpZIgALduifWIiEj3WFtbo3fv3pg4cSJSUlIwcOBA5Wu1a9dGTEwMjhw5goSEBHzyySdIS0sr8b4DAwPxxhtvICwsDGfPnsXBgwdVEp38YyQlJWHTpk24du0aFi1ahK1bt6rU8fLyQmJiIuLi4pCeno6cnJwCx+rXrx/Mzc0RFhaG+Ph47Nu3D6NGjUL//v2Va3CWRu3atREdHY24uDicPXsWffv2VWkx8vLyQlhYGAYPHoxt27YhMTER+/fvxy+//AIAGDlyJDIzMxEaGop//vkHV65cwbp161Quy5UHJkDlLCVFu/WIiKjiDRkyBA8fPkRQUJBKf53JkyejadOmCAoKQocOHeDi4oKQkJAS79fIyAhbt27F06dP0bJlS3z00UeYNWuWSp0PPvgAY8eOxciRI9G4cWMcOXIEU6ZMUanTvXt3dOzYEW+99RacnJzUDsW3tLTEnj178ODBA7Ro0QI9evTAO++8gyVLlmj2ZrxmwYIFsLe3R+vWrREcHIygoCA0bdpUpc6yZcvQo0cPfPrpp6hbty4+/vhjZGdnAwAcHBzw119/ISsrC+3bt0ezZs2wcuXKcm8N4mKoamhzMdT9+8UOz8XZtw/o0KFMhyIi0llcDJW0hYuh6omAAKB69cLnnJDJAA8PsR4RERFVDCZA5czYGFi4UHz8ehKU/zwykh2giYiIKhIToArQrZs41N3dXbW8enUOgSciIpICh8FXkG7dxKHuBw+KHZ5dXcXLXmz5ISIiqnhMgCqQsTE7OhMREekCXgIjIqIKw4HHVFba+g4xASIionKXP6fLEymWf6dKJf87VNZ5gngJjIiIyp2xsTGqVKmiXFTT0tJSuZwEUUkIgoAnT57g7t27qFKlisqCraXBBIiIiCqEi4sLABS5sjhRcapUqaL8LpUFEyAiIqoQMpkMrq6uqFatGnJzc6UOh/SQqalpmVt+8jEBIiKiCmVsbKy1HzGi0mInaCIiIjI4TICIiIjI4DABIiIiIoPDPkBq5E+ylJmZKXEkREREVFL5v9slmSyRCZAajx8/BgB4eHhIHAkRERFp6vHjx7CzsyuyjkzgvOQF5OXl4c6dO7CxseFEXYXIzMyEh4cHbt26BVtbW6nDMXj8PHQLPw/dws9Dt5Tn5yEIAh4/fgw3NzcYGRXdy4ctQGoYGRmhevXqUoehF2xtbfkfig7h56Fb+HnoFn4euqW8Po/iWn7ysRM0ERERGRwmQERERGRwmABRqcjlckybNg1yuVzqUAj8PHQNPw/dws9Dt+jK58FO0ERERGRw2AJEREREBocJEBERERkcJkBERERkcJgAERERkcFhAkQlNmfOHLRo0QI2NjaoVq0aQkJCcOnSJanDov/3zTffQCaTYcyYMVKHYtCSk5Px4YcfwsHBARYWFvD19cU///wjdVgGSaFQYMqUKfD29oaFhQV8fHwwc+bMEq0TRWV34MABBAcHw83NDTKZDNu2bVN5XRAETJ06Fa6urrCwsEBgYCCuXLlSYfExAaIS+/vvvzFixAgcO3YMMTExyM3NxX/+8x9kZ2dLHZrBO3nyJH744Qc0atRI6lAM2sOHD9GmTRuYmppi165duHDhAr799lvY29tLHZpB+u9//4tly5ZhyZIlSEhIwH//+1/MnTsXixcvljo0g5CdnQ0/Pz8sXbpU7etz587FokWLsHz5chw/fhxWVlYICgrCs2fPKiQ+DoOnUrt37x6qVauGv//+G+3atZM6HIOVlZWFpk2b4vvvv8fXX3+Nxo0bIzIyUuqwDNKECRNw+PBhHDx4UOpQCMD7778PZ2dnrFq1SlnWvXt3WFhY4Oeff5YwMsMjk8mwdetWhISEABBbf9zc3PDZZ59h3LhxAICMjAw4OztjzZo1CA0NLfeY2AJEpZaRkQEAqFq1qsSRGLYRI0bgvffeQ2BgoNShGLzt27ejefPm6NmzJ6pVq4YmTZpg5cqVUodlsFq3bo3Y2FhcvnwZAHD27FkcOnQInTp1kjgySkxMRGpqqsr/W3Z2dvD398fRo0crJAYuhkqlkpeXhzFjxqBNmzZo2LCh1OEYrE2bNuH06dM4efKk1KEQgOvXr2PZsmWIiIjAl19+iZMnT2L06NEwMzNDWFiY1OEZnAkTJiAzMxN169aFsbExFAoFZs2ahX79+kkdmsFLTU0FADg7O6uUOzs7K18rb0yAqFRGjBiB+Ph4HDp0SOpQDNatW7cQHh6OmJgYmJubSx0OQfzDoHnz5pg9ezYAoEmTJoiPj8fy5cuZAEngl19+wfr167FhwwY0aNAAcXFxGDNmDNzc3Ph5EC+BkeZGjhyJHTt2YN++fahevbrU4RisU6dO4e7du2jatClMTExgYmKCv//+G4sWLYKJiQkUCoXUIRocV1dX1K9fX6WsXr16SEpKkigiw/b5559jwoQJCA0Nha+vL/r374+xY8dizpw5Uodm8FxcXAAAaWlpKuVpaWnK18obEyAqMUEQMHLkSGzduhV//fUXvL29pQ7JoL3zzjs4f/484uLilLfmzZujX79+iIuLg7GxsdQhGpw2bdoUmBri8uXL8PT0lCgiw/bkyRMYGan+zBkbGyMvL0+iiCift7c3XFxcEBsbqyzLzMzE8ePH0apVqwqJgZfAqMRGjBiBDRs24LfffoONjY3yOq2dnR0sLCwkjs7w2NjYFOh/ZWVlBQcHB/bLksjYsWPRunVrzJ49G7169cKJEyewYsUKrFixQurQDFJwcDBmzZqFGjVqoEGDBjhz5gwWLFiAwYMHSx2aQcjKysLVq1eVzxMTExEXF4eqVauiRo0aGDNmDL7++mvUrl0b3t7emDJlCtzc3JQjxcqdQFRCANTeVq9eLXVo9P/at28vhIeHSx2GQfv999+Fhg0bCnK5XKhbt66wYsUKqUMyWJmZmUJ4eLhQo0YNwdzcXKhZs6YwadIkIScnR+rQDMK+ffvU/maEhYUJgiAIeXl5wpQpUwRnZ2dBLpcL77zzjnDp0qUKi4/zABEREZHBYR8gIiIiMjhMgIiIiMjgMAEiIiIig8MEiIiIiAwOEyAiIiIyOEyAiIiIyOAwASIiIiKDwwSIiKgQMpkM27ZtkzoMIioHTICISCcNHDgQMpmswK1jx45Sh0ZElQDXAiMindWxY0esXr1apUwul0sUDRFVJmwBIiKdJZfL4eLionKzt7cHIF6eWrZsGTp16gQLCwvUrFkTW7ZsUdn+/PnzePvtt2FhYQEHBwcMHToUWVlZKnV+/PFHNGjQAHK5HK6urhg5cqTK6+np6ejatSssLS1Ru3ZtbN++Xfnaw4cP0a9fPzg5OcHCwgK1a9cukLARkW5iAkREemvKlCno3r07zp49i379+iE0NBQJCQkAgOzsbAQFBcHe3h4nT57E5s2bsXfvXpUEZ9myZRgxYgSGDh2K8+fPY/v27ahVq5bKMWbMmIFevXrh3Llz6Ny5M/r164cHDx4oj3/hwgXs2rULCQkJWLZsGRwdHSvuDSCi0quwZVeJiDQQFhYmGBsbC1ZWViq3WbNmCYIgCACEYcOGqWzj7+8vDB8+XBAEQVixYoVgb28vZGVlKV/fuXOnYGRkJKSmpgqCIAhubm7CpEmTCo0BgDB58mTl86ysLAGAsGvXLkEQBCE4OFgYNGiQdk6YiCoU+wARkc566623sGzZMpWyqlWrKh+3atVK5bVWrVohLi4OAJCQkAA/Pz9YWVkpX2/Tpg3y8vJw6dIlyGQy3LlzB++8806RMTRq1Ej52MrKCra2trh79y4AYPjw4ejevTtOnz6N//znPwgJCUHr1q1Lda5EVLGYABGRzrKysipwSUpbLCwsSlTP1NRU5blMJkNeXh4AoFOnTrh58yb++OMPxMTE4J133sGIESMwf/58rcdLRNrFPkBEpLeOHTtW4Hm9evUAAPXq1cPZs2eRnZ2tfP3w4cMwMjJCnTp1YGNjAy8vL8TGxpYpBicnJ4SFheHnn39GZGQkVqxYUab9EVHFYAsQEemsnJwcpKamqpSZmJgoOxpv3rwZzZs3R9u2bbF+/XqcOHECq1atAgD069cP06ZNQ1hYGKZPn4579+5h1KhR6N+/P5ydnQEA06dPx7Bhw1CtWjV06tQJjx8/xuHDhzFq1KgSxTd16lQ0a9YMDRo0QE5ODnbs2KFMwIhItzEBIiKdtXv3bri6uqqU1alTBxcvXgQgjtDatGkTPv30U7i6umLjxo2oX78+AMDS0hJ79uxBeHg4WrRoAUtLS3Tv3h0LFixQ7issLAzPnj3Dd999h3HjxsHR0RE9evQocXxmZmaYOHEibty4AQsLCwQEBGDTpk1aOHMiKm8yQRAEqYMgItKUTCbD1q1bERISInUoRKSH2AeIiIiIDA4TICIiIjI47ANERHqJV++JqCzYAkREREQGhwkQERERGRwmQERERGRwmAARERGRwWECRERERAaHCRAREREZHCZAREREZHCYABEREZHBYQJEREREBuf/AI3YOGCNAw66AAAAAElFTkSuQmCC", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABcgUlEQVR4nO3dd1hT598G8DuEvVWQJQKideJWioraSovaUnGideCottaNtmoVR63SWqu4qrV11Um1aP1Vq1WqdY+690RRFBQHCCpKOO8f5000EjCBwEnI/bmuXCZPTk6+J9Dm5pxnyARBEEBERERkQsykLoCIiIiopDEAERERkclhACIiIiKTwwBEREREJocBiIiIiEwOAxARERGZHAYgIiIiMjkMQERERGRyGICIiIjI5DAAEelB79694evrW6jXTpo0CTKZTL8FGZjr169DJpNh2bJlJfq+u3btgkwmw65du1Rt2v6siqtmX19f9O7dW6/7JCLdMQBRqSaTybS6vfoFSVRU+/fvx6RJk/Do0SOpSyGifJhLXQBRcVqxYoXa419//RXbt2/P0169evUivc/PP/+M3NzcQr12/PjxGDNmTJHen7RXlJ+Vtvbv34/Jkyejd+/ecHZ2Vnvu4sWLMDPj355EUmMAolKtR48eao8PHjyI7du352l/3ZMnT2Bra6v1+1hYWBSqPgAwNzeHuTn/UywpRflZ6YOVlZWk728ssrKyYGdnJ3UZVIrxzxAyeS1btkStWrVw9OhRNG/eHLa2tvjqq68AAH/88Qc++OADeHp6wsrKCv7+/pgyZQoUCoXaPl7vV6LsPzJjxgwsWrQI/v7+sLKyQqNGjXDkyBG112rqAySTyTB48GBs3LgRtWrVgpWVFWrWrImtW7fmqX/Xrl1o2LAhrK2t4e/vj59++knrfkV79uxB586dUbFiRVhZWcHb2xsjRozA06dP8xyfvb09kpOTER4eDnt7e7i6umLUqFF5PotHjx6hd+/ecHJygrOzMyIjI7W6FPTff/9BJpNh+fLleZ7btm0bZDIZ/vzzTwDAjRs38Pnnn6Nq1aqwsbFBuXLl0LlzZ1y/fv2N76OpD5C2NZ86dQq9e/dGpUqVYG1tDXd3d/Tt2xf3799XbTNp0iR88cUXAAA/Pz/VZVZlbZr6AF27dg2dO3dG2bJlYWtri7fffhubN29W20bZn+m3337D1KlTUaFCBVhbW6NVq1a4cuXKG49bl8/s0aNHGDFiBHx9fWFlZYUKFSqgV69eSEtLU23z7NkzTJo0CW+99Rasra3h4eGBDh064OrVq2r1vn55WVPfKuXv19WrV9G2bVs4ODige/fuALT/HQWACxcuoEuXLnB1dYWNjQ2qVq2KcePGAQB27twJmUyGDRs25Hnd6tWrIZPJcODAgTd+jlR68M9OIgD3799HmzZt0LVrV/To0QNubm4AgGXLlsHe3h5RUVGwt7fHP//8gwkTJiAjIwPff//9G/e7evVqPH78GJ9++ilkMhmmT5+ODh064Nq1a288E7F3717Ex8fj888/h4ODA+bMmYOOHTsiKSkJ5cqVAwAcP34crVu3hoeHByZPngyFQoGvv/4arq6uWh33unXr8OTJEwwcOBDlypXD4cOHMXfuXNy6dQvr1q1T21ahUCA0NBSBgYGYMWMGduzYgR9++AH+/v4YOHAgAEAQBLRr1w579+7FZ599hurVq2PDhg2IjIx8Yy0NGzZEpUqV8Ntvv+XZPi4uDmXKlEFoaCgA4MiRI9i/fz+6du2KChUq4Pr161iwYAFatmyJc+fO6XT2Tpeat2/fjmvXrqFPnz5wd3fH2bNnsWjRIpw9exYHDx6ETCZDhw4dcOnSJaxZswazZs2Ci4sLAOT7M0lNTUWTJk3w5MkTDB06FOXKlcPy5cvx0UcfYf369Wjfvr3a9t9++y3MzMwwatQopKenY/r06ejevTsOHTpU4HFq+5llZmYiODgY58+fR9++fVG/fn2kpaVh06ZNuHXrFlxcXKBQKPDhhx8iISEBXbt2xbBhw/D48WNs374dZ86cgb+/v9afv1JOTg5CQ0PRrFkzzJgxQ1WPtr+jp06dQnBwMCwsLDBgwAD4+vri6tWr+N///oepU6eiZcuW8Pb2xqpVq/J8pqtWrYK/vz+CgoJ0rpuMmEBkQgYNGiS8/mvfokULAYCwcOHCPNs/efIkT9unn34q2NraCs+ePVO1RUZGCj4+PqrHiYmJAgChXLlywoMHD1Ttf/zxhwBA+N///qdqmzhxYp6aAAiWlpbClStXVG0nT54UAAhz585VtYWFhQm2trZCcnKyqu3y5cuCubl5nn1qoun4YmJiBJlMJty4cUPt+AAIX3/9tdq29erVExo0aKB6vHHjRgGAMH36dFVbTk6OEBwcLAAQli5dWmA9Y8eOFSwsLNQ+s+zsbMHZ2Vno27dvgXUfOHBAACD8+uuvqradO3cKAISdO3eqHcurPytdatb0vmvWrBEACLt371a1ff/99wIAITExMc/2Pj4+QmRkpOrx8OHDBQDCnj17VG2PHz8W/Pz8BF9fX0GhUKgdS/Xq1YXs7GzVtrNnzxYACKdPn87zXq/S9jObMGGCAECIj4/Ps31ubq4gCIKwZMkSAYAwc+bMfLfR9NkLwsv/Nl79XJW/X2PGjNGqbk2/o82bNxccHBzU2l6tRxDE3y8rKyvh0aNHqra7d+8K5ubmwsSJE/O8D5VuvARGBLFfRp8+ffK029jYqO4/fvwYaWlpCA4OxpMnT3DhwoU37jciIgJlypRRPQ4ODgYgXvJ4k5CQELW/pGvXrg1HR0fVaxUKBXbs2IHw8HB4enqqtqtcuTLatGnzxv0D6seXlZWFtLQ0NGnSBIIg4Pjx43m2/+yzz9QeBwcHqx3Lli1bYG5urjojBAByuRxDhgzRqp6IiAi8ePEC8fHxqra///4bjx49QkREhMa6X7x4gfv376Ny5cpwdnbGsWPHtHqvwtT86vs+e/YMaWlpePvttwFA5/d99f0bN26MZs2aqdrs7e0xYMAAXL9+HefOnVPbvk+fPrC0tFQ91vZ3StvP7Pfff0edOnXynCUBoLqs+vvvv8PFxUXjZ1SUKR1e/Rloqju/39F79+5h9+7d6Nu3LypWrJhvPb169UJ2djbWr1+vaouLi0NOTs4b+wVS6cMARATAy8tL7UtF6ezZs2jfvj2cnJzg6OgIV1dX1f8o09PT37jf1/9nrAxDDx8+1Pm1ytcrX3v37l08ffoUlStXzrOdpjZNkpKS0Lt3b5QtW1bVr6dFixYA8h6ftbV1nss4r9YDiP1MPDw8YG9vr7Zd1apVtaqnTp06qFatGuLi4lRtcXFxcHFxwbvvvqtqe/r0KSZMmABvb29YWVnBxcUFrq6uePTokVY/l1fpUvODBw8wbNgwuLm5wcbGBq6urvDz8wOg3e9Dfu+v6b2UIxNv3Lih1l7Y3yltP7OrV6+iVq1aBe7r6tWrqFq1ql4775ubm6NChQp52rX5HVWGvzfVXa1aNTRq1AirVq1Sta1atQpvv/221v/NUOnBPkBEUP8rU+nRo0do0aIFHB0d8fXXX8Pf3x/W1tY4duwYRo8erdVQarlcrrFdEIRifa02FAoF3nvvPTx48ACjR49GtWrVYGdnh+TkZPTu3TvP8eVXj75FRERg6tSpSEtLg4ODAzZt2oRu3bqpfdkOGTIES5cuxfDhwxEUFAQnJyfIZDJ07dq1WIe4d+nSBfv378cXX3yBunXrwt7eHrm5uWjdunWxD61XKuzvRUl/ZvmdCXq907ySlZVVnukBdP0d1UavXr0wbNgw3Lp1C9nZ2Th48CDmzZun837I+DEAEeVj165duH//PuLj49G8eXNVe2JiooRVvVS+fHlYW1trHAGkzaig06dP49KlS1i+fDl69eqlat++fXuha/Lx8UFCQgIyMzPVzqhcvHhR631ERERg8uTJ+P333+Hm5oaMjAx07dpVbZv169cjMjISP/zwg6rt2bNnhZp4UNuaHz58iISEBEyePBkTJkxQtV++fDnPPnW5DOTj46Px81FeYvXx8dF6XwXR9jPz9/fHmTNnCtyXv78/Dh06hBcvXuTbmV95Zur1/b9+Rqsg2v6OVqpUCQDeWDcAdO3aFVFRUVizZg2ePn0KCwsLtcurZDp4CYwoH8q/tF/9y/r58+f48ccfpSpJjVwuR0hICDZu3Ijbt2+r2q9cuYK//vpLq9cD6scnCAJmz55d6Jratm2LnJwcLFiwQNWmUCgwd+5crfdRvXp1BAQEIC4uDnFxcfDw8FALoMraXz/jMXfu3HzPLuijZk2fFwDExsbm2ady/hptAlnbtm1x+PBhtSHYWVlZWLRoEXx9fVGjRg1tD6VA2n5mHTt2xMmTJzUOF1e+vmPHjkhLS9N45kS5jY+PD+RyOXbv3q32vC7//Wj7O+rq6ormzZtjyZIlSEpK0liPkouLC9q0aYOVK1di1apVaN26tWqkHpkWngEiykeTJk1QpkwZREZGYujQoZDJZFixYoXeLkHpw6RJk/D333+jadOmGDhwIBQKBebNm4datWrhxIkTBb62WrVq8Pf3x6hRo5CcnAxHR0f8/vvvWvVPyk9YWBiaNm2KMWPG4Pr166hRowbi4+N17h8TERGBCRMmwNraGv369ctzaeTDDz/EihUr4OTkhBo1auDAgQPYsWOHanqA4qjZ0dERzZs3x/Tp0/HixQt4eXnh77//1nhGsEGDBgCAcePGoWvXrrCwsEBYWJjGif3GjBmDNWvWoE2bNhg6dCjKli2L5cuXIzExEb///rveZo3W9jP74osvsH79enTu3Bl9+/ZFgwYN8ODBA2zatAkLFy5EnTp10KtXL/z666+IiorC4cOHERwcjKysLOzYsQOff/452rVrBycnJ3Tu3Blz586FTCaDv78//vzzT9y9e1frmnX5HZ0zZw6aNWuG+vXrY8CAAfDz88P169exefPmPP8t9OrVC506dQIATJkyRfcPk0qHEh93RiSh/IbB16xZU+P2+/btE95++23BxsZG8PT0FL788kth27ZtbxxarRzq+/333+fZJwC1Ibf5DYMfNGhQnte+PoRaEAQhISFBqFevnmBpaSn4+/sLv/zyizBy5EjB2to6n0/hpXPnzgkhISGCvb294OLiIvTv31813P71Ycp2dnZ5Xq+p9vv37ws9e/YUHB0dBScnJ6Fnz57C8ePHtRoGr3T58mUBgABA2Lt3b57nHz58KPTp00dwcXER7O3thdDQUOHChQt5Ph9thsHrUvOtW7eE9u3bC87OzoKTk5PQuXNn4fbt23l+poIgCFOmTBG8vLwEMzMztSHxmn6GV69eFTp16iQ4OzsL1tbWQuPGjYU///xTbRvlsaxbt06tXdOwck20/cyUn8fgwYMFLy8vwdLSUqhQoYIQGRkppKWlqbZ58uSJMG7cOMHPz0+wsLAQ3N3dhU6dOglXr15VbXPv3j2hY8eOgq2trVCmTBnh008/Fc6cOaP175cgaP87KgiCcObMGdXPx9raWqhataoQHR2dZ5/Z2dlCmTJlBCcnJ+Hp06cFfm5UeskEwYD+nCUivQgPD8fZs2c19k8hMnU5OTnw9PREWFgYFi9eLHU5JBH2ASIycq8vCXD58mVs2bIFLVu2lKYgIgO3ceNG3Lt3T61jNZkengEiMnIeHh6q9alu3LiBBQsWIDs7G8ePH0eVKlWkLo/IYBw6dAinTp3ClClT4OLiUujJK6l0YCdoIiPXunVrrFmzBikpKbCyskJQUBCmTZvG8EP0mgULFmDlypWoW7eu2mKsZJp4BoiIiIhMDvsAERERkclhACIiIiKTwz5AGuTm5uL27dtwcHAo0srGREREVHIEQcDjx4/h6en5xklEGYA0uH37Nry9vaUug4iIiArh5s2bqFChQoHbMABp4ODgAED8AB0dHSWuhoiIiLSRkZEBb29v1fd4QRiANFBe9nJ0dGQAIiIiMjLadF9hJ2giIiIyOQxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiIiITA4DEBEREZkcBiAiIiIyOZwJmoiIiEqEQgHs2QPcuQN4eADBwYBcLk0tDEBERERU7OLjgWHDgFu3XrZVqADMng106FDy9fASGBERERWr+HigUyf18AMAyclie3x8ydfEAERERETFRqEQz/wIQt7nlG3Dh4vblSQGICIiIio2e/bkPfPzKkEAbt4UtytJDEBERERUbO7c0e92+sIARERERMXGw0O/2+kLR4EREREZOEMaPq6r4GBxtFdysuZ+QDKZ+HxwcMnWxTNAREREBiw+HvD1Bd55B/j4Y/FfX19pRk4VhlwuDnUHxLDzKuXj2NiSD3QMQERERAbKEIePF0aHDsD69YCXl3p7hQpiuxTzAMkEQdMJKdOWkZEBJycnpKenw9HRUepyiIjIBCkU4pme/EZQKS8dJSYaz+Ww4r6Up8v3N/sAERERGSBdho+3bFliZRWJXG44tfISGBERkQEy1OHjpQUDEBERkQEy1OHjpYXkAWj+/Pnw9fWFtbU1AgMDcfjw4Xy3ffHiBb7++mv4+/vD2toaderUwdatW4u0TyIiIkOkHD7++sgpJZkM8PYu+eHjpYWkASguLg5RUVGYOHEijh07hjp16iA0NBR3797VuP348ePx008/Ye7cuTh37hw+++wztG/fHsePHy/0PomIqPRSKIBdu4A1a8R/S3q9qaIw1OHjpYYgocaNGwuDBg1SPVYoFIKnp6cQExOjcXsPDw9h3rx5am0dOnQQunfvXuh9apKeni4AENLT07V+DRERGZbffxeEChUEQewuLN4qVBDbjYmm4/D2Nr7jKAm6fH9Ldgbo+fPnOHr0KEJCQlRtZmZmCAkJwYEDBzS+Jjs7G9bW1mptNjY22Lt3b6H3SUREpU9pmT8HEOfIuX4d2LkTWL1a/DcxUZq5c0oTyYbBp6WlQaFQwM3NTa3dzc0NFy5c0Pia0NBQzJw5E82bN4e/vz8SEhIQHx8Pxf+f0yzMPgExWGVnZ6seZ2RkFPawiIhIYgoFMGyY5mUXBEG8fDR8ONCunfFcPjKk4eOlheSdoHUxe/ZsVKlSBdWqVYOlpSUGDx6MPn36wMysaIcRExMDJycn1c3b21tPFRMRUUnTZf4cMl2SBSAXFxfI5XKkpqaqtaempsLd3V3ja1xdXbFx40ZkZWXhxo0buHDhAuzt7VGpUqVC7xMAxo4di/T0dNXt5s2bRTw6IiKSCufPIW1IFoAsLS3RoEEDJCQkqNpyc3ORkJCAoKCgAl9rbW0NLy8v5OTk4Pfff0e7du2KtE8rKys4Ojqq3YiIyDhx/hzShqRLYURFRSEyMhINGzZE48aNERsbi6ysLPTp0wcA0KtXL3h5eSEmJgYAcOjQISQnJ6Nu3bpITk7GpEmTkJubiy+//FLrfRIRUemmnD8nOVlzPyDlGlqcP8e0SRqAIiIicO/ePUyYMAEpKSmoW7cutm7dqurEnJSUpNa/59mzZxg/fjyuXbsGe3t7tG3bFitWrICzs7PW+yQiojcr7kUri5Ny/pxOncSw82oI4vw5pMTV4DXgavBEZMri48VRVK92JK5QQQwVxjT0WtNxeHuL4ceYjoO0p8v3NwOQBgxARGSqlPPnvP7NoDxzsn69cYUHYz6TRbpjACoiBiAiMkUKBeDrm/8QcmXfmcREhggyTLp8fxvVPEBERFR8OH8OmRIGICIiAsD5c8i0MAAREREAzp9DpoUBiIiIALycP0fZ4fl1Mpk4iorz51BpwABEREQAXs6fA+QNQZw/h0obBiAiIlLp0EEc6u7lpd5eoYLxDYEnKoikM0ETEZHh6dABaNeO8+dQ6cYAREREecjlQMuWUldBVHx4CYyIiIhMDs8AERHpEZdeIDIODEBERHpSWhYRJTIFvARGRKQHykVEX19KIjlZbI+Pl6YuItKMAYiIqIgUCvHMj6alpZVtw4eL2xGRYWAAIiIqIi4iSmR8GICIiIqIi4gSGR8GICKiIuIiokTGhwGIiKiIuIgokfFhACIiKiIuIkpkfBiAiIj0gIuIEhkXToRIRKQnXESU9CUnB0hLA+7efXm7dw94+BAwMwPMzcWbXK7+75vaivr8621mZvlf+jV0DEBERHrERURJk9xcMby8GmZeDTevtz14IHXF2itsEOvdGxg8WMK6pXtrIiIi4yQIwOPH2oWZu3fFszm6ToRpZga4uADly4s3V1egTBnxvRUK8SxRTs7L+5ratLmvzbaaJvlUUm6Tna3b8YWG6ra9vjEAEZFB4CKiJLWnT7ULM8rHun7hA2KAUYYZZbB5NeC8+rhMGcP5byA3V/xvVJ8hq1IlaY+JAYiIJMdFRKm4PH0K3LghzsRdUJi5exfIzNR9//b22oUZV1fxbI6lpf6PsSSYmYk3CwupK9EfBiAikpRyEdHXT7ErFxHlCCoqyIsXQFIScP06kJgo3l69n5Ki2/4sLbULM8p/bW2L46ioJMgEoaAre6YpIyMDTk5OSE9Ph6Ojo9TlEJVaCgXg65v/OloymXgmKDHRcC4FUMlSKMQwrAw1rwedW7fEyzMFcXAAKlYE3N3zDzPKm4OD8Y5qIt2+v3kGiIgko8siohxZVToJApCaqvnszfXr4tmdFy8K3oe1NeDnJ4bpV/9V3i9blqGG8mIAIiLJcBHR0k8QxCHdmsKN8t9nzwreh4WFeAbn9XCjfOzmxoBDumMAIiLJcBHR0iEjI/8+ONevi8PFC2JmJl7qfP3MjfK+pycvgZL+MQARkWSUi4gmJ2ueZ0TZB4iLiErryRNxJFV+AUebSfvc3TWHG19fcaFYYx0dRcaLAYiIJKNcRLRTJzHsvBqCSuMiogqF2J/l1VtOTt42Kdtff+7RI7GPzpuUK6c53Pj5AT4+gI1NMX+4RDpiACIiSSkXEdU0D1BsrOEPgX/8GLh4EbhwQf2Wmpo3VBjzmFsHh7x9b1697+AgdYVEumEAIiLJGfoiooIghrPXQ87Fi+Llu6IwNxc7+Spvrz8uqfb8nrO3FwNOmTLsaEylCwMQERkEQ1hE9Nkz4PJlzUEnKyv/17m7A9Wqqd+8vMR+LQUFEXNzhgoiqTAAEZFJEQRx+YPXQ86FC2KH3vwuU5mbA1WqiOGmatWXQadqVcDZuSSPgIj0gQGIiEqlFy+Aa9c09895+DD/1zk7A9Wr5z2j4+dXutZBIjJ1DEBEZNQePdIccq5cETseayKTiYHm9bM51aqJSyPwshRR6ccAREQGLzdXXBJBU9+cgha7tLXNeyanWjWgcmUOyyYydQxARGQwnjwBLl3KG3QuXQKePs3/dV5eec/kKDsim5mVXP1EZDwYgIioxAmC2OH41Cn12+XL+XdCtrR82Qn51dtbbwFvWPSZiCgPBiAiI6dQGO78OYC4TtTp0+pB5/Tp/NeHKltWcydkX19xJBYRkT7wfydERiw+XvMMyrNnl/wMygqF2PH49bM6169r3t7SEqhRA6hdW/3m5laiZRORiWIAIjJS8fHiGlqvXzJKThbb168vvhCUlpb3rM6ZM+JEgppUqJA36Lz1FoeVE5F0ZIJgzKvTFI+MjAw4OTkhPT0djuxcQAZIoRAvCb165udVylXUExOLdjns+XNxpNXrZ3Vu39a8va0tUKuWetAJCBAvaxERFTddvr95BojICO3Zk3/4AcSzQjdvittps7yEIIjDyV8POufPixMKalKpUt6zOpUqGVb/IyKi/DAAERmhO3cKv93Tp8DZs+odkk+dEi9raeLk9PJMjjLo1KrF1b+JyLgxABEZIQ8P7baTyYBNm/IONc/NzbutmZk4l87rZ3W8vTkzMhGVPpJPETZ//nz4+vrC2toagYGBOHz4cIHbx8bGomrVqrCxsYG3tzdGjBiBZ6/0vJw0aRJkMpnarVq1asV9GEQlKjhY7ONTUDCRyYBu3YB27YDoaGDdOrE/T24u4OICtGoFjBgBLF0KHD0KZGYC584Ba9cCX30FfPghULEiww8RlU6SngGKi4tDVFQUFi5ciMDAQMTGxiI0NBQXL15E+fLl82y/evVqjBkzBkuWLEGTJk1w6dIl9O7dGzKZDDNnzlRtV7NmTezYsUP12JyTh1ApI5cDs2YBnTvnv40giKOs8htqzmBDRKZM0mQwc+ZM9O/fH3369AEALFy4EJs3b8aSJUswZsyYPNvv378fTZs2xccffwwA8PX1Rbdu3XDo0CG17czNzeHu7l78B0AkgeRkYNkyYMkSzc87OAD9+om3qlU51JyISBPJAtDz589x9OhRjB07VtVmZmaGkJAQHDhwQONrmjRpgpUrV+Lw4cNo3Lgxrl27hi1btqBnz55q212+fBmenp6wtrZGUFAQYmJiULFixXxryc7ORnZ2tupxRkZGEY+OSL+ePwf+/BNYvBjYuvVlHx4HB6BrV6B+fXE5CE9Pw5sJmojIEEkWgNLS0qBQKOD22rSvbm5uuHDhgsbXfPzxx0hLS0OzZs0gCAJycnLw2Wef4auvvlJtExgYiGXLlqFq1aq4c+cOJk+ejODgYJw5cwYO+QxbiYmJweTJk/V3cER6cuGCGHqWLwfu3XvZHhwsnuHp1Amws5OuPiIiYyV5J2hd7Nq1C9OmTcOPP/6IY8eOIT4+Hps3b8aUKVNU27Rp0wadO3dG7dq1ERoaii1btuDRo0f47bff8t3v2LFjkZ6errrdvHmzJA6HSKPMTPHyVtOm4ppYM2aI4cfNDRg9WuzIvHs3EBnJ8ENEVFiSnQFycXGBXC5HamqqWntqamq+/Xeio6PRs2dPfPLJJwCAgIAAZGVlYcCAARg3bhzMzPLmOWdnZ7z11lu4cuVKvrVYWVnBysqqCEdDVDSCABw6JJ7tWbtWDEGAeCmrbVvxbE/btuzPQ0SkL5KdAbK0tESDBg2QkJCgasvNzUVCQgKCgoI0vubJkyd5Qo78/zs75LeiR2ZmJq5evQoPbSdOISpB9+4BM2eKEwsGBQG//CKGn8qVgZgYIClJnMenXTuGHyIifZJ0FFhUVBQiIyPRsGFDNG7cGLGxscjKylKNCuvVqxe8vLwQExMDAAgLC8PMmTNRr149BAYG4sqVK4iOjkZYWJgqCI0aNQphYWHw8fHB7du3MXHiRMjlcnTr1k2y4yR6lUIBbN8unu3544+XS03Y2Ih9evr1A5o35zB1IqLiJGkAioiIwL179zBhwgSkpKSgbt262Lp1q6pjdFJSktoZn/Hjx0Mmk2H8+PFITk6Gq6srwsLCMHXqVNU2t27dQrdu3XD//n24urqiWbNmOHjwIFxdXUv8+Ihedf262Ldn2TJxnS6lBg2ATz4RJy10cpKqOiIi08LV4DXgavCkL8+eARs3imd7EhLEvj4AUKYM0KOHeLanTh1JSyQiKjW4GjyRxE6eFEPPypXAw4cv21u1Es/2hIcD1taSlUdEZPIYgIj0JD0dWLNGDD7//feyvUIFoE8f8ebnJ119RET0EgMQUREIgjgnz+LFwPr1wNOnYruFhThyq18/4L33ODMzEZGhYQAiKoQ7d8TZmZcsAS5fftleo4YYenr2BNjvnojIcDEAEWkpJwfYskWcq2fLFnE4OwDY2wMREWLfnsBADl8nIjIGDEBEb3DpknimZ/lyICXlZXuTJuLZni5dxBBERETGgwGISIOsLLFPz+LFwJ49L9tdXcU1uPr2FdfpIiIi48QARPT/BEEcvbV4sTiaKyNDbDczA1q3Fs/2fPghYGkpbZ1ERFR0DEBkshQK8ezOpUvAqVPiaK7Tp18+7+cnhp7ISHEoOxERlR4MQGSS4uOBIUOA27fV2y0sgM6dxeDTsqV49oeIiEofBiAyOfHxQMeOmp978UJ87t13S7YmIiIqWfz7lkyKQgEMGpT/8zIZMHz4yyHuRERUOjEAkUlZvVp9KPvrBEFcqf3VkV9ERFT6MACRyTh1Chg8WLtt79wp3lqIiEhaDEBkEg4eBFq0eDm0/U08PIq3HiIikhYDEJV6O3YAISHAo0dAUBDg6Zn/chUyGeDtDQQHl2iJRERUwhiAqFT74w/ggw/EmZ3ffx/Yvh2YO1d87vUQpHwcG8vV24mISjsGICq1Vq4Uh7Q/fw506ABs2gTY2Yn3168HvLzUt69QQWzv0EGaeomIqORwHiAqlX788eVw98hIcQV381d+2zt0ANq1E0d73bkj9vkJDuaZHyIiU8EARKVOTAzw1Vfi/SFDxEtammZ0lsvF2Z6JiMj08BIYlRqCAIwZ8zL8REcDs2dzOQsiIsqLZ4CoVMjNFS95LVwoPp4xAxg5UtqaiIjIcDEAkdF78QLo3Vuc5VkmA376CejfX+qqiIjIkDEAkVF79gyIiBBHeJmbAytWAF27Sl0VEREZOgYgMlqPH4sjuXbuBKytxSHsH3wgdVVERGQMGIDIKD14ALRtCxw6BNjbA3/+KS51QUREpA0GIDI6KSnirM6nTwNlywJbtwKNGkldFRERGRMGIDIqN26I63pduSJOXvj330CtWlJXRURExoYBiIzGhQvAe+8Bt24Bvr7iIqf+/lJXRURExohTxJFROH4caN5cDD/VqwN79zL8EBFR4TEAkcHbtw945x3g3j2gQQNg9+68C5kSERHpggGIDNrff4sdntPTxcVKExIAFxepqyIiImPHAEQG6/ffgQ8/BJ48AVq3Fkd7OTlJXRUREZUGDEBkkJYvB7p0EZe56NwZ+OMPwNZW6qqIiKi0YAAigzN3rri2V24u0K8fsGYNYGkpdVVERFSaMACRwRAE4JtvgKFDxccjRgA//wzI5dLWRUREpQ8DEBkEQQC++AKIjhYfT5oE/PCDuLo7ERGRvnEiRJKcQgEMHCie7QGAWbOA4cMlLYmIiEo5BiCS1PPnQK9eQFwcYGYmhqC+faWuioiISjsGIJLM06dAp07Ali2AhQWwapU44ouIiKi4MQCRJDIygI8+Av79F7CxAeLjxbl+iIiISgIDEJW4+/fFsPPff4CDA7B5szjLMxERUUlhAKISdfu2uKL7uXPikhZbt4rrexEREZUkBiAqMdeuASEhQGIi4OkJ7NghruxORERU0jgPEJWIc+fEy1yJiUClSsDevQw/REQkHQYgKnZHjwLNm4uXv2rWBPbsAfz8pK6KiIhMGQMQFavdu4F33hE7PjdqJI768vSUuioiIjJ1DEBUbLZsAUJDgcePgRYtgIQEoFw5qasiIiIygAA0f/58+Pr6wtraGoGBgTh8+HCB28fGxqJq1aqwsbGBt7c3RowYgWfPnhVpn6R/v/0GtGsHPHsGfPAB8Ndf4pB3IiIiQyBpAIqLi0NUVBQmTpyIY8eOoU6dOggNDcXdu3c1br969WqMGTMGEydOxPnz57F48WLExcXhq6++KvQ+Sf8WLwa6dQNycoCuXYENG8TJDomIiAyFTBAEQao3DwwMRKNGjTBv3jwAQG5uLry9vTFkyBCMGTMmz/aDBw/G+fPnkZCQoGobOXIkDh06hL179xZqn5pkZGTAyckJ6enpcHR0LOphmpRZs4CoKPF+//7AggWAXC5tTUREZBp0+f6W7AzQ8+fPcfToUYSEhLwsxswMISEhOHDggMbXNGnSBEePHlVd0rp27Rq2bNmCtm3bFnqfAJCdnY2MjAy1G+lGEICJE1+Gn1GjgJ9+YvghIiLDJNlEiGlpaVAoFHBzc1Nrd3Nzw4ULFzS+5uOPP0ZaWhqaNWsGQRCQk5ODzz77THUJrDD7BICYmBhMnjy5iEdkunJzxeAze7b4+JtvgK++AmQyaesiIiLKj+SdoHWxa9cuTJs2DT/++COOHTuG+Ph4bN68GVOmTCnSfseOHYv09HTV7ebNm3qquPRTKIBPPnkZfubMAcaNY/ghIiLDJtkZIBcXF8jlcqSmpqq1p6amwt3dXeNroqOj0bNnT3zyyScAgICAAGRlZWHAgAEYN25cofYJAFZWVrCysiriEZme7GygRw9g/XrAzAxYsgSIjJS6KiIiojeT7AyQpaUlGjRooNahOTc3FwkJCQgKCtL4midPnsDMTL1k+f93MhEEoVD7pMJ58kQc5r5+PWBpCaxbx/BDRETGQ9LFUKOiohAZGYmGDRuicePGiI2NRVZWFvr06QMA6NWrF7y8vBATEwMACAsLw8yZM1GvXj0EBgbiypUriI6ORlhYmCoIvWmfVHRZWUDr1uJ6Xra2wMaN4grvRERExkLSABQREYF79+5hwoQJSElJQd26dbF161ZVJ+akpCS1Mz7jx4+HTCbD+PHjkZycDFdXV4SFhWHq1Kla75OKbtYsMfw4OQGbNwNNm0pdERERkW4knQfIUHEeoPw9ewb4+AB37wIrVwLdu0tdERERkahY5wHy9fXF119/jaSkpEIXSMZr5Uox/Hh7A126SF0NERFR4egcgIYPH474+HhUqlQJ7733HtauXYvs7OziqI0MTG4u8MMP4v3hwwELC0nLISIiKrRCBaATJ07g8OHDqF69OoYMGQIPDw8MHjwYx44dK44ayUBs2QJcuAA4Oopz/xARERmrQg+Dr1+/PubMmYPbt29j4sSJ+OWXX9CoUSPUrVsXS5YsAbsWlT4zZoj/fvqpGIKIiIiMVaFHgb148QIbNmzA0qVLsX37drz99tvo168fbt26ha+++go7duzA6tWr9VkrSejIEeDffwFzc2DoUKmrISIiKhqdA9CxY8ewdOlSrFmzBmZmZujVqxdmzZqFatWqqbZp3749GjVqpNdCSVrKvj/dugEVKkhbCxERUVHpHIAaNWqE9957DwsWLEB4eDgsNPSE9fPzQ9euXfVSIEnv+nVxpmcAGDlS0lKIiIj0QucAdO3aNfj4+BS4jZ2dHZYuXVroosiwxMaKI8Deew+oU0fqaoiIiIpO507Qd+/exaFDh/K0Hzp0CP/9959eiiLD8fAh8Msv4v1Ro6SthYiISF90DkCDBg3CzZs387QnJydj0KBBeimKDMdPP4lrfwUEcL0vIiIqPXQOQOfOnUP9+vXztNerVw/nzp3TS1FkGLKzgTlzxPujRgEymbT1EBER6YvOAcjKygqpqal52u/cuQNzc0nXViU9W7MGuHMH8PQE2KediIhKE50D0Pvvv4+xY8ciPT1d1fbo0SN89dVXeI/XSEoNQXg58eGwYYClpbT1EBER6ZPOp2xmzJiB5s2bw8fHB/Xq1QMAnDhxAm5ublixYoXeCyRpbNsGnD0L2NsDAwZIXQ0REZF+6RyAvLy8cOrUKaxatQonT56EjY0N+vTpg27dummcE4iMk/LsT//+gLOzpKUQERHpnUzgol15ZGRkwMnJCenp6XA0wUWvjh8H6tcH5HLg6lXgDdM+ERERGQRdvr8L3Wv53LlzSEpKwvPnz9XaP/roo8LukgyEctmLLl00hx+FAtizR+wg7eEBBAeLYYmIiMhYFGom6Pbt2+P06dOQyWSqVd9l/z9GWqFQ6LdCKlE3bwJr14r3NS17ER8vdoq+detlW4UKwOzZQIcOJVMjERFRUek8CmzYsGHw8/PD3bt3YWtri7Nnz2L37t1o2LAhdu3aVQwlUkmaPVs8w/POO0CDBurPxccDnTqphx8ASE4W2+PjS65OIiKiotA5AB04cABff/01XFxcYGZmBjMzMzRr1gwxMTEYOnRocdRIJSQ9HVi0SLz/+rIXCoV45kdTjzFl2/Dh4nZERESGTucApFAo4ODgAABwcXHB7du3AQA+Pj64ePGifqujEvXzz8Djx0CNGkDr1urP7dmT98zPqwRBvHy2Z0/x1khERKQPOvcBqlWrFk6ePAk/Pz8EBgZi+vTpsLS0xKJFi1CpUqXiqJFKwPPn4qrvgNj3x+y1aHznjnb70XY7IiIiKekcgMaPH4+srCwAwNdff40PP/wQwcHBKFeuHOLi4vReIJWM334T+/K4uQHdu+d93sNDu/1oux0REZGU9DIP0IMHD1CmTBnVSDBjZ2rzAAkCUK8ecPIkMHUq8NVXebdRKABfXzEkafqNkcnE0WCJiRwST0RE0tDl+1unPkAvXryAubk5zpw5o9ZetmzZUhN+TFFCghh+bG2Bzz7TvI1cLo4QA/KuCq98HBvL8ENERMZBpwBkYWGBihUrcq6fUka57EW/fkDZsvlv16EDsH494OWl3l6hgtjOeYCIiMhY6HwJbPHixYiPj8eKFStQtqBvSyNmSpfATp0C6tQROz1fvgxo04+dM0ETEZEhKtalMObNm4crV67A09MTPj4+sLOzU3v+2LFjuu6SJDRzpvhvx47ahR9ADDstWxZbSURERMVO5wAUHh5eDGWQFJKTgdWrxfualr0gIiIqrXQOQBMnTiyOOkgCc+cCL16Il7ACA6WuhoiIqOToPBM0lQ6PHwMLF4r3X1/2goiIqLTT+QyQmZlZgUPeOULMOCxeLK799dZbwIcfSl0NERFRydI5AG3YsEHt8YsXL3D8+HEsX74ckydP1lthVHxycoBZs8T7mpa9ICIiKu30MhM0AKxevRpxcXH4448/9LE7SZX2YfBr1wLdugGursCNG4CNjdQVERERFV2xzQRdkLfffhsJCQn62h0VE0EAvv9evD94MMMPERGZJr0EoKdPn2LOnDnwen2KYDI4//4LHDsGWFsDn38udTVERETS0LkP0OuLngqCgMePH8PW1hYrV67Ua3Gkf8plL/r0AVxcpK2FiIhIKjoHoFmzZqkFIDMzM7i6uiIwMBBlypTRa3GkX+fOAZs3i4uXjhghdTVERETS0TkA9e7duxjKoJKgXPYiPByoUkXSUoiIiCSlcx+gpUuXYt26dXna161bh+XLl+ulKNK/lBRgxQrxPic+JCIiU6dzAIqJiYGLhs4j5cuXx7Rp0/RSFOnfvHnA8+dAUBDQpInU1RAREUlL5wCUlJQEPz+/PO0+Pj5ISkrSS1GkX1lZwI8/ivd59oeIiKgQAah8+fI4depUnvaTJ0+iXLlyeimK9GvpUuDhQ8DfH2jXTupqiIiIpKdzAOrWrRuGDh2KnTt3QqFQQKFQ4J9//sGwYcPQtWvX4qiRikCheNn5OSoKkMulrYeIiMgQ6DwKbMqUKbh+/TpatWoFc3Px5bm5uejVqxf7ABmgDRuAxESgXDmAA/iIiIhEOgcgS0tLxMXF4ZtvvsGJEydgY2ODgIAA+Pj4FEd9VASvLnvx+eeAra209RARERkKnQOQUpUqVVCFk8kYtH37gMOHASsrYNAgqashIiIyHDr3AerYsSO+++67PO3Tp09H586d9VIU6Ydy2YtevQA3N2lrISIiMiQ6B6Ddu3ejbdu2edrbtGmD3bt3F6qI+fPnw9fXF9bW1ggMDMThw4fz3bZly5aQyWR5bh988IFqm969e+d5vnXr1oWqzVhdvAhs2iTej4qSthYiIiJDo/MlsMzMTFhaWuZpt7CwQEZGhs4FxMXFISoqCgsXLkRgYCBiY2MRGhqKixcvonz58nm2j4+Px/Pnz1WP79+/jzp16uQ5+9S6dWssXbpU9djKykrn2ozZrFliH6CwMKBaNamrISIiMiw6nwEKCAhAXFxcnva1a9eiRo0aOhcwc+ZM9O/fH3369EGNGjWwcOFC2NraYsmSJRq3L1u2LNzd3VW37du3w9bWNk8AsrKyUtvOlBZqvXsXUK5KwokPiYiI8tL5DFB0dDQ6dOiAq1ev4t133wUAJCQkYPXq1Vi/fr1O+3r+/DmOHj2KsWPHqtrMzMwQEhKCAwcOaLWPxYsXo2vXrrCzs1Nr37VrF8qXL48yZcrg3XffxTfffJPvRI3Z2dnIzs5WPS7MmSxD8uOPwLNnQKNGQHCw1NUQEREZHp3PAIWFhWHjxo24cuUKPv/8c4wcORLJycn4559/ULlyZZ32lZaWBoVCAbfXeui6ubkhJSXlja8/fPgwzpw5g08++UStvXXr1vj111+RkJCA7777Dv/++y/atGkDhUKhcT8xMTFwcnJS3by9vXU6DkPy5Akwf754f9QoQCaTth4iIiJDVKhh8B988IGq03FGRgbWrFmDUaNG4ejRo/mGjOKwePFiBAQEoHHjxmrtr85IHRAQgNq1a8Pf3x+7du1Cq1at8uxn7NixiHqlp3BGRobRhqBffwXS0gBfX6BDB6mrISIiMkw6nwFS2r17NyIjI+Hp6YkffvgB7777Lg4ePKjTPlxcXCCXy5GamqrWnpqaCnd39wJfm5WVhbVr16Jfv35vfJ9KlSrBxcUFV65c0fi8lZUVHB0d1W7G6NVlL0aMAMwLPcsTERFR6aZTAEpJScG3336LKlWqoHPnznB0dER2djY2btyIb7/9Fo0aNdLpzS0tLdGgQQMkJCSo2nJzc5GQkICgoKACX7tu3TpkZ2ejR48eb3yfW7du4f79+/Dw8NCpPmPzv/8Bly8DZcoAfftKXQ0REZHh0joAhYWFoWrVqjh16hRiY2Nx+/ZtzJ07t8gFREVF4eeff8by5ctx/vx5DBw4EFlZWejTpw8AoFevXmqdpJUWL16M8PDwPB2bMzMz8cUXX+DgwYO4fv06EhIS0K5dO1SuXBmhoaFFrteQKSc+HDgQsLeXthYiIiJDpvVFkr/++gtDhw7FwIED9boERkREBO7du4cJEyYgJSUFdevWxdatW1Udo5OSkmBmpp7TLl68iL179+Lvv//Osz+5XI5Tp05h+fLlePToETw9PfH+++9jypQppXouoAMHxKUvLC2BwYOlroaIiMiwyQRBELTZ8ODBg1i8eDHi4uJQvXp19OzZE127doWHhwdOnjxZqDmADFVGRgacnJyQnp5uNP2BOnUCfv9dvPS1eLHU1RAREZU8Xb6/tb4E9vbbb+Pnn3/GnTt38Omnn2Lt2rXw9PREbm4utm/fjsePHxe5cCqcq1eB+HjxPpe9ICIiejOdR4HZ2dmhb9++2Lt3L06fPo2RI0fi22+/Rfny5fHRRx8VR430BsplL9q2BWrWlLoaIiIiw1foYfAAULVqVUyfPh23bt3CmjVr9FUT6eD+fUC5agiXvSAiItJOkQKQklwuR3h4ODYplx+nErNgAfD0KVC/PtCypdTVEBERGQe9BCCSxrNngHImAi57QUREpD0GICO2cqW48nvFiuIoMCIiItIOA5CRys0FfvhBvD98OGBhIWk5RERERoUByEht2QJcuAA4OQGffCJ1NURERMaFAchIKZe9+PRTwMFB2lqIiIiMDQOQETpyBPj3X3G196FDpa6GiIjI+DAAGSFl35+PPwa8vKSthYiIyBgxABmZ69eBdevE+yNHSloKERGR0WIAMjKxseIIsPffB2rXlroaIiIi48QAZEQePgR++UW8z2UviIiICo8ByIj89BOQlSWe+QkJkboaIiIi48UAZCSys4E5c8T7XPaCiIioaBiAjMSaNcCdO+Kor4gIqashIiIybgxARkAQXk58OGwYYGkpbT1ERETGjgHICGzbBpw9K874PGCA1NUQEREZPwYgI6A8+9O/v7j2FxERERUNA5CBO34cSEgA5HLx8hcREREVHQOQgVMuexERAVSsKG0tREREpQUDkAG7eRNYu1a8z2UviIiI9IcByIDNng0oFMC77wL160tdDRERUenBAGSg0tOBRYvE+1z2goiISL8YgAzUzz8Djx8DNWoArVtLXQ0REVHpwgBkgJ4/F1d9B7jsBRERUXFgADJAv/0GJCcD7u7Axx9LXQ0REVHpwwBkYF5d9mLoUMDKStp6iIiISiMGIAOTkACcPAnY2QGffip1NURERKUTA5CBUZ796dcPKFtW2lqIiIhKKwYgA3LqlLjwqZkZMHy41NUQERGVXgxABkS57EWnToCfn7S1EBERlWYMQAbi1i1g9WrxPic+JCIiKl4MQAZi7lwgJwdo3hxo1EjqaoiIiEo3BiADkJEBLFwo3ufZHyIiouLHAGQAFi8WQ1DVqsAHH0hdDRERUenHACSxFy9eLnsxcqQ4AoyIiIiKF79uJbZ+PZCUBJQvD/TsKXU1REREpoEBSEKvLnsxeDBgbS1tPURERKaCAUhCu3YBx44BNjbAwIFSV0NERGQ6GIAkpDz706cP4OIibS1ERESmhAFIImfPAlu2ADIZMGKE1NUQERGZFgYgicycKf7bvj1QubK0tRAREZkaBiAJ3LkDrFwp3ufEh0RERCWPAUgC8+YBz58DTZoAQUFSV0NERGR6GIBKWGYmsGCBeJ9nf4iIiKTBAFTCli4FHj4U+/189JHU1RAREZkmgwhA8+fPh6+vL6ytrREYGIjDhw/nu23Lli0hk8ny3D54ZREtQRAwYcIEeHh4wMbGBiEhIbh8+XJJHEqBcnKAWbPE+1FRgFwubT1ERESmSvIAFBcXh6ioKEycOBHHjh1DnTp1EBoairt372rcPj4+Hnfu3FHdzpw5A7lcjs6dO6u2mT59OubMmYOFCxfi0KFDsLOzQ2hoKJ49e1ZSh6XRhg1AYiJQrhwQGSlpKURERCZN8gA0c+ZM9O/fH3369EGNGjWwcOFC2NraYsmSJRq3L1u2LNzd3VW37du3w9bWVhWABEFAbGwsxo8fj3bt2qF27dr49ddfcfv2bWzcuLEEjyyv48fFeX8GDQJsbSUthYiIyKRJGoCeP3+Oo0ePIiQkRNVmZmaGkJAQHDhwQKt9LF68GF27doWdnR0AIDExESkpKWr7dHJyQmBgYL77zM7ORkZGhtqtOEybBpw/DwwdWiy7JyIiIi1JGoDS0tKgUCjg5uam1u7m5oaUlJQ3vv7w4cM4c+YMPvnkE1Wb8nW67DMmJgZOTk6qm7e3t66HorWqVcVLYERERCQdyS+BFcXixYsREBCAxo0bF2k/Y8eORXp6uup28+ZNPVVIREREhkjSAOTi4gK5XI7U1FS19tTUVLi7uxf42qysLKxduxb9+vVTa1e+Tpd9WllZwdHRUe1GREREpZekAcjS0hINGjRAQkKCqi03NxcJCQkIesMUyevWrUN2djZ69Oih1u7n5wd3d3e1fWZkZODQoUNv3CcRERGZBnOpC4iKikJkZCQaNmyIxo0bIzY2FllZWejTpw8AoFevXvDy8kJMTIza6xYvXozw8HCUe61DjUwmw/Dhw/HNN9+gSpUq8PPzQ3R0NDw9PREeHl5Sh0VEREQGTPIAFBERgXv37mHChAlISUlB3bp1sXXrVlUn5qSkJJiZqZ+ounjxIvbu3Yu///5b4z6//PJLZGVlYcCAAXj06BGaNWuGrVu3wtrautiPh4iIiAyfTBAEQeoiDE1GRgacnJyQnp7O/kBERERGQpfvb6MeBUZERERUGAxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiIiITA4DEBEREZkcBiAiIiIyOQxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiIiITA4DEBEREZkcBiAiIiIyOQxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiIiITA4DEBEREZkcBiAiIiIyOQxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiIiITA4DEBEREZkcBiAiIiIyOQxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiIiITA4DEBEREZkcBiAiIiIyOQxAREREZHLMpS6AiIhMi0KhwIsXL6Qug4yQhYUF5HK5XvbFAERERCVCEASkpKTg0aNHUpdCRszZ2Rnu7u6QyWRF2g8DEBERlQhl+ClfvjxsbW2L/AVGpkUQBDx58gR3794FAHh4eBRpfwxARERU7BQKhSr8lCtXTupyyEjZ2NgAAO7evYvy5csX6XIYO0ETEVGxU/b5sbW1lbgSMnbK36Gi9iNjACIiohLDy15UVPr6HZI8AM2fPx++vr6wtrZGYGAgDh8+XOD2jx49wqBBg+Dh4QErKyu89dZb2LJli+r5SZMmQSaTqd2qVatW3IdBRESkFV9fX8TGxmq9/a5duyCTydh5XM8k7QMUFxeHqKgoLFy4EIGBgYiNjUVoaCguXryI8uXL59n++fPneO+991C+fHmsX78eXl5euHHjBpydndW2q1mzJnbs2KF6bG7Ork5ERKWBQgHs2QPcuQN4eADBwYCeRkXn8aYzDRMnTsSkSZN03u+RI0dgZ2en9fZNmjTBnTt34OTkpPN7Uf4kTQYzZ85E//790adPHwDAwoULsXnzZixZsgRjxozJs/2SJUvw4MED7N+/HxYWFgDEJP06c3NzuLu7F2vtRERUsuLjgWHDgFu3XrZVqADMng106KD/97tz547qflxcHCZMmICLFy+q2uzt7VX3BUGAQqHQ6g9uV1dXneqwtLTkd1oxkOwS2PPnz3H06FGEhIS8LMbMDCEhIThw4IDG12zatAlBQUEYNGgQ3NzcUKtWLUybNg0KhUJtu8uXL8PT0xOVKlVC9+7dkZSUVKzHQkRExSs+HujUST38AEBystgeH6//93R3d1fdnJycIJPJVI8vXLgABwcH/PXXX2jQoAGsrKywd+9eXL16Fe3atYObmxvs7e3RqFEjtSsSQN5LYDKZDL/88gvat28PW1tbVKlSBZs2bVI9//olsGXLlsHZ2Rnbtm1D9erVYW9vj9atW6sFtpycHAwdOhTOzs4oV64cRo8ejcjISISHh+d7vPfv30e3bt3g5eUFW1tbBAQEYM2aNWrb5ObmYvr06ahcuTKsrKxQsWJFTJ06VfX8rVu30K1bN5QtWxZ2dnZo2LAhDh06VIhPv/hJFoDS0tKgUCjg5uam1u7m5oaUlBSNr7l27RrWr18PhUKBLVu2IDo6Gj/88AO++eYb1TaBgYFYtmwZtm7digULFiAxMRHBwcF4/PhxvrVkZ2cjIyND7UZERIZBoRDP/AhC3ueUbcOHi9uVtDFjxuDbb7/F+fPnUbt2bWRmZqJt27ZISEjA8ePH0bp1a4SFhb3xD/HJkyejS5cuOHXqFNq2bYvu3bvjwYMH+W7/5MkTzJgxAytWrMDu3buRlJSEUaNGqZ7/7rvvsGrVKixduhT79u1DRkYGNm7cWGANz549Q4MGDbB582acOXMGAwYMQM+ePdX65o4dOxbffvstoqOjce7cOaxevVr1PZ6ZmYkWLVogOTkZmzZtwsmTJ/Hll18iNzdXi09SAoJEkpOTBQDC/v371dq/+OILoXHjxhpfU6VKFcHb21vIyclRtf3www+Cu7t7vu/z8OFDwdHRUfjll1/y3WbixIkCgDy39PR0HY+KiIg0efr0qXDu3Dnh6dOnOr92505BEKNOwbedO/VetsrSpUsFJyenV2raKQAQNm7c+MbX1qxZU5g7d67qsY+PjzBr1izVYwDC+PHjVY8zMzMFAMJff/2l9l4PHz5U1QJAuHLliuo18+fPF9zc3FSP3dzchO+//171OCcnR6hYsaLQrl07bQ9ZEARB+OCDD4SRI0cKgiAIGRkZgpWVlfDzzz9r3Pann34SHBwchPv37+v0Hroq6HcpPT1d6+9vyc4Aubi4QC6XIzU1Va09NTU132udHh4eeOutt9QmPqpevTpSUlLw/Plzja9xdnbGW2+9hStXruRby9ixY5Genq663bx5sxBHRERExeGVKzt62U6fGjZsqPY4MzMTo0aNQvXq1eHs7Ax7e3ucP3/+jWeAateurbpvZ2cHR0dH1YzHmtja2sLf31/12MPDQ7V9eno6UlNT0bhxY9XzcrkcDRo0KLAGhUKBKVOmICAgAGXLloW9vT22bdumqv38+fPIzs5Gq1atNL7+xIkTqFevHsqWLVvg+xgKyQKQpaUlGjRogISEBFVbbm4uEhISEBQUpPE1TZs2xZUrV9ROp126dAkeHh6wtLTU+JrMzExcvXq1wCmzrays4OjoqHYjIiLDoO2KB0VcGaFQXh/NNWrUKGzYsAHTpk3Dnj17cOLECQQEBOT7R7qScmCPkkwmK/DSkabtBU3XCHXw/fffY/bs2Rg9ejR27tyJEydOIDQ0VFW7chbm/LzpeUMj6TxAUVFR+Pnnn7F8+XKcP38eAwcORFZWlmpUWK9evTB27FjV9gMHDsSDBw8wbNgwXLp0CZs3b8a0adMwaNAg1TajRo3Cv//+i+vXr2P//v1o37495HI5unXrVuLHR0RERRccLI72ym9UukwGeHuL20lt37596N27N9q3b4+AgAC4u7vj+vXrJVqDk5MT3NzccOTIEVWbQqHAsWPHCnzdvn370K5dO/To0QN16tRBpUqVcOnSJdXzVapUgY2NjdqJi1fVrl0bJ06cKLDvkiGRdBh8REQE7t27hwkTJiAlJQV169bF1q1bVR2qkpKSYGb2MqN5e3tj27ZtGDFiBGrXrg0vLy8MGzYMo0ePVm2j7IF+//59uLq6olmzZjh48KDOww6JiMgwyOXiUPdOncSw8+qJDmUoio0tvvmAdFGlShXEx8cjLCwMMpkM0dHRknQCHjJkCGJiYlC5cmVUq1YNc+fOxcOHDwuc26hKlSpYv3499u/fjzJlymDmzJlITU1FjRo1AADW1tYYPXo0vvzyS1haWqJp06a4d+8ezp49i379+qFbt26YNm0awsPDERMTAw8PDxw/fhyenp75XtmRkuQzBA4ePBiDBw/W+NyuXbvytAUFBeHgwYP57m/t2rX6Ko2IiAxEhw7A+vWa5wGKjS2eeYAKY+bMmejbty+aNGkCFxcXjB49WpKRxaNHj0ZKSgp69eoFuVyOAQMGIDQ0tMDFQ8ePH49r164hNDQUtra2GDBgAMLDw5Genq7aJjo6Gubm5pgwYQJu374NDw8PfPbZZwDEri1///03Ro4cibZt2yInJwc1atTA/Pnzi/14C0MmFPWiYSmUkZEBJycnpKensz8QEZEePHv2DImJifDz84O1tXWh91OSM0GXJrm5uahevTq6dOmCKVOmSF1OkRT0u6TL97fkZ4CIiIi0JZcDLVtKXYXhu3HjBv7++2+0aNEC2dnZmDdvHhITE/Hxxx9LXZrBkHwxVCIiItIvMzMzLFu2DI0aNULTpk1x+vRp7NixA9WrV5e6NIPBM0BERESljLe3N/bt2yd1GQaNZ4CIiIjI5DAAERERkclhACIiIiKTwwBEREREJocBiIiIiEwOAxARERGZHAYgIiKiYtSyZUsMHz5c9djX1xexsbEFvkYmk2Hjxo1Ffm997ac0YgAiIiLSICwsDK1bt9b43J49eyCTyXDq1Cmd93vkyBEMGDCgqOWpmTRpEurWrZun/c6dO2jTpo1e36u0YAAiIiLSoF+/fti+fTtuvbr66v9bunQpGjZsiNq1a+u8X1dXV9ja2uqjxDdyd3eHlZVVibyXsWEAIiIi0uDDDz+Eq6srli1bptaemZmJdevWoV+/frh//z66desGLy8v2NraIiAgAGvWrClwv69fArt8+TKaN28Oa2tr1KhRA9u3b8/zmtGjR+Ott96Cra0tKlWqhOjoaLx48QIAsGzZMkyePBknT56ETCaDTCZT1fz6JbDTp0/j3XffhY2NDcqVK4cBAwYgMzNT9Xzv3r0RHh6OGTNmwMPDA+XKlcOgQYNU76XJ1atX0a5dO7i5ucHe3h6NGjXCjh071LbJzs7G6NGj4e3tDSsrK1SuXBmLFy9WPX/27Fl8+OGHcHR0hIODA4KDg3H16tUCP8ei4lIYREQkCUEAnjwp+fe1tQVksjdvZ25ujl69emHZsmUYN24cZP//onXr1kGhUKBbt27IzMxEgwYNMHr0aDg6OmLz5s3o2bMn/P390bhx4ze+R25uLjp06AA3NzccOnQI6enpav2FlBwcHLBs2TJ4enri9OnT6N+/PxwcHPDll18iIiICZ86cwdatW1XBw8nJKc8+srKyEBoaiqCgIBw5cgR3797FJ598gsGDB6uFvJ07d8LDwwM7d+7ElStXEBERgbp166J///4ajyEzMxNt27bF1KlTYWVlhV9//RVhYWG4ePEiKlasCADo1asXDhw4gDlz5qBOnTpITExEWloaACA5ORnNmzdHy5Yt8c8//8DR0RH79u1DTk7OGz+/IhEoj/T0dAGAkJ6ertf95uQIws6dgrB6tfhvTo5ed09EZLCePn0qnDt3Tnj69KmqLTNTEMQYVLK3zEzt6z5//rwAQNi5c6eqLTg4WOjRo0e+r/nggw+EkSNHqh63aNFCGDZsmOqxj4+PMGvWLEEQBGHbtm2Cubm5kJycrHr+r7/+EgAIGzZsyPc9vv/+e6FBgwaqxxMnThTq1KmTZ7tX97No0SKhTJkyQuYrH8DmzZsFMzMzISUlRRAEQYiMjBR8fHyEnFe+oDp37ixERETkW4smNWvWFObOnSsIgiBcvHhRACBs375d47Zjx44V/Pz8hOfPn2u1b02/S0q6fH/zElgJiY8HfH2Bd94BPv5Y/NfXV2wnIiLDVK1aNTRp0gRLliwBAFy5cgV79uxBv379AAAKhQJTpkxBQEAAypYtC3t7e2zbtg1JSUla7f/8+fPw9vaGp6enqi0oKCjPdnFxcWjatCnc3d1hb2+P8ePHa/0er75XnTp1YGdnp2pr2rQpcnNzcfHiRVVbzZo1IZfLVY89PDxw9+7dfPebmZmJUaNGoXr16nB2doa9vT3Onz+vqu/EiROQy+Vo0aKFxtefOHECwcHBsLCw0Ol4ioqXwEpAfDzQqZP4t8erkpPF9vXrgQ4dpKmNiEgqtrbAK91PSvR9ddGvXz8MGTIE8+fPx9KlS+Hv76/6Mv/+++8xe/ZsxMbGIiAgAHZ2dhg+fDieP3+ut3oPHDiA7t27Y/LkyQgNDYWTkxPWrl2LH374QW/v8arXg4hMJkNubm6+248aNQrbt2/HjBkzULlyZdjY2KBTp06qz8DGxqbA93vT88WFAaiYKRTAsGF5ww8gtslkwPDhQLt2wCuBm4io1JPJgFdORhisLl26YNiwYVi9ejV+/fVXDBw4UNUfaN++fWjXrh169OgBQOzTc+nSJdSoUUOrfVevXh03b97EnTt34OHhAQA4ePCg2jb79++Hj48Pxo0bp2q7ceOG2jaWlpZQKBRvfK9ly5YhKytLdRZo3759MDMzQ9WqVbWqV5N9+/ahd+/eaN++PQDxjND169dVzwcEBCA3Nxf//vsvQkJC8ry+du3aWL58OV68eFGiZ4F4CayY7dkDaBhBqSIIwM2b4nZERGR47O3tERERgbFjx+LOnTvo3bu36rkqVapg+/bt2L9/P86fP49PP/0UqampWu87JCQEb731FiIjI3Hy5Ens2bNHLego3yMpKQlr167F1atXMWfOHGzYsEFtG19fXyQmJuLEiRNIS0tDdnZ2nvfq3r07rK2tERkZiTNnzmDnzp0YMmQIevbsCTc3N90+lNfqi4+Px4kTJ3Dy5El8/PHHameMfH19ERkZib59+2Ljxo1ITEzErl278NtvvwEABg8ejIyMDHTt2hX//fcfLl++jBUrVqhdlisODEDF7M4d/W5HREQlr1+/fnj48CFCQ0PV+uuMHz8e9evXR2hoKFq2bAl3d3eEh4drvV8zMzNs2LABT58+RePGjfHJJ59g6tSpatt89NFHGDFiBAYPHoy6deti//79iI6OVtumY8eOaN26Nd555x24urpqHIpva2uLbdu24cGDB2jUqBE6deqEVq1aYd68ebp9GK+ZOXMmypQpgyZNmiAsLAyhoaGoX7++2jYLFixAp06d8Pnnn6NatWro378/srKyAADlypXDP//8g8zMTLRo0QINGjTAzz//XOxng2SCoOnijGnLyMiAk5MT0tPT4ejoWKR97doldnh+k507gZYti/RWREQG69mzZ0hMTISfnx+sra2lLoeMWEG/S7p8f/MMUDELDgYqVMh/zgmZDPD2FrcjIiKiksEAVMzkcmD2bPH+6yFI+Tg2lh2giYiIShIDUAno0EEc6u7lpd5eoQKHwBMREUmBw+BLSIcO4lD3PXvEDs8eHuJlL575ISIiKnkMQCVILmdHZyIiIkPAS2BERFRiOPCYikpfv0MMQEREVOyUc7o8kWL5dypVlL9DRZ0niJfAiIio2Mnlcjg7O6sW1bS1tVUtJ0GkDUEQ8OTJE9y9exfOzs5qC7YWBgMQERGVCHd3dwAocGVxojdxdnZW/S4VBQMQERGVCJlMBg8PD5QvXx4vXryQuhwyQhYWFkU+86PEAERERCVKLpfr7UuMqLDYCZqIiIhMDgMQERERmRwGICIiIjI57AOkgXKSpYyMDIkrISIiIm0pv7e1mSyRAUiDx48fAwC8vb0lroSIiIh09fjxYzg5ORW4jUzgvOR55Obm4vbt23BwcOBEXfnIyMiAt7c3bt68CUdHR6nLMXn8eRgW/jwMC38ehqU4fx6CIODx48fw9PSEmVnBvXx4BkgDMzMzVKhQQeoyjIKjoyP/h2JA+PMwLPx5GBb+PAxLcf083nTmR4mdoImIiMjkMAARERGRyWEAokKxsrLCxIkTYWVlJXUpBP48DA1/HoaFPw/DYig/D3aCJiIiIpPDM0BERERkchiAiIiIyOQwABEREZHJYQAiIiIik8MARFqLiYlBo0aN4ODggPLlyyM8PBwXL16Uuiz6f99++y1kMhmGDx8udSkmLTk5GT169EC5cuVgY2ODgIAA/Pfff1KXZZIUCgWio6Ph5+cHGxsb+Pv7Y8qUKVqtE0VFt3v3boSFhcHT0xMymQwbN25Ue14QBEyYMAEeHh6wsbFBSEgILl++XGL1MQCR1v79918MGjQIBw8exPbt2/HixQu8//77yMrKkro0k3fkyBH89NNPqF27ttSlmLSHDx+iadOmsLCwwF9//YVz587hhx9+QJkyZaQuzSR99913WLBgAebNm4fz58/ju+++w/Tp0zF37lypSzMJWVlZqFOnDubPn6/x+enTp2POnDlYuHAhDh06BDs7O4SGhuLZs2clUh+HwVOh3bt3D+XLl8e///6L5s2bS12OycrMzET9+vXx448/4ptvvkHdunURGxsrdVkmacyYMdi3bx/27NkjdSkE4MMPP4SbmxsWL16sauvYsSNsbGywcuVKCSszPTKZDBs2bEB4eDgA8eyPp6cnRo4ciVGjRgEA0tPT4ebmhmXLlqFr167FXhPPAFGhpaenAwDKli0rcSWmbdCgQfjggw8QEhIidSkmb9OmTWjYsCE6d+6M8uXLo169evj555+lLstkNWnSBAkJCbh06RIA4OTJk9i7dy/atGkjcWWUmJiIlJQUtf9vOTk5ITAwEAcOHCiRGrgYKhVKbm4uhg8fjqZNm6JWrVpSl2Oy1q5di2PHjuHIkSNSl0IArl27hgULFiAqKgpfffUVjhw5gqFDh8LS0hKRkZFSl2dyxowZg4yMDFSrVg1yuRwKhQJTp05F9+7dpS7N5KWkpAAA3Nzc1Nrd3NxUzxU3BiAqlEGDBuHMmTPYu3ev1KWYrJs3b2LYsGHYvn07rK2tpS6HIP5h0LBhQ0ybNg0AUK9ePZw5cwYLFy5kAJLAb7/9hlWrVmH16tWoWbMmTpw4geHDh8PT05M/D+IlMNLd4MGD8eeff2Lnzp2oUKGC1OWYrKNHj+Lu3buoX78+zM3NYW5ujn///Rdz5syBubk5FAqF1CWaHA8PD9SoUUOtrXr16khKSpKoItP2xRdfYMyYMejatSsCAgLQs2dPjBgxAjExMVKXZvLc3d0BAKmpqWrtqampqueKGwMQaU0QBAwePBgbNmzAP//8Az8/P6lLMmmtWrXC6dOnceLECdWtYcOG6N69O06cOAG5XC51iSanadOmeaaGuHTpEnx8fCSqyLQ9efIEZmbqX3NyuRy5ubkSVURKfn5+cHd3R0JCgqotIyMDhw4dQlBQUInUwEtgpLVBgwZh9erV+OOPP+Dg4KC6Tuvk5AQbGxuJqzM9Dg4Oefpf2dnZoVy5cuyXJZERI0agSZMmmDZtGrp06YLDhw9j0aJFWLRokdSlmaSwsDBMnToVFStWRM2aNXH8+HHMnDkTffv2lbo0k5CZmYkrV66oHicmJuLEiRMoW7YsKlasiOHDh+Obb75BlSpV4Ofnh+joaHh6eqpGihU7gUhLADTeli5dKnVp9P9atGghDBs2TOoyTNr//vc/oVatWoKVlZVQrVo1YdGiRVKXZLIyMjKEYcOGCRUrVhSsra2FSpUqCePGjROys7OlLs0k7Ny5U+N3RmRkpCAIgpCbmytER0cLbm5ugpWVldCqVSvh4sWLJVYf5wEiIiIik8M+QERERGRyGICIiIjI5DAAERERkclhACIiIiKTwwBEREREJocBiIiIiEwOAxARERGZHAYgIqJ8yGQybNy4UeoyiKgYMAARkUHq3bs3ZDJZnlvr1q2lLo2ISgGuBUZEBqt169ZYunSpWpuVlZVE1RBRacIzQERksKysrODu7q52K1OmDADx8tSCBQvQpk0b2NjYoFKlSli/fr3a60+fPo13330XNjY2KFeuHAYMGIDMzEy1bZYsWYKaNWvCysoKHh4eGDx4sNrzaWlpaN++PWxtbVGlShVs2rRJ9dzDhw/RvXt3uLq6wsbGBlWqVMkT2IjIMDEAEZHRio6ORseOHXHy5El0794dXbt2xfnz5wEAWVlZCA0NRZkyZXDkyBGsW7cOO3bsUAs4CxYswKBBgzBgwACcPn0amzZtQuXKldXeY/LkyejSpQtOnTqFtm3bonv37njw4IHq/c+dO4e//voL58+fx4IFC+Di4lJyHwARFV6JLbtKRKSDyMhIQS6XC3Z2dmq3qVOnCoIgCACEzz77TO01gYGBwsCBAwVBEIRFixYJZcqUETIzM1XPb968WTAzMxNSUlIEQRAET09PYdy4cfnWAEAYP3686nFmZqYAQPjrr78EQRCEsLAwoU+fPvo5YCIqUewDREQG65133sGCBQvU2sqWLau6HxQUpPZcUFAQTpw4AQA4f/486tSpAzs7O9XzTZs2RW5uLi5evAiZTIbbt2+jVatWBdZQu3Zt1X07Ozs4Ojri7t27AICBAweiY8eOOHbsGN5//32Eh4ejSZMmhTpWIipZDEBEZLDs7OzyXJLSFxsbG622s7CwUHssk8mQm5sLAGjTpg1u3LiBLVu2YPv27WjVqhUGDRqEGTNm6L1eItIv9gEiIqN18ODBPI+rV68OAKhevTpOnjyJrKws1fP79u2DmZkZqlatCgcHB/j6+iIhIaFINbi6uiIyMhIrV65EbGwsFi1aVKT9EVHJ4BkgIjJY2dnZSElJUWszNzdXdTRet24dGjZsiGbNmmHVqlU4fPgwFi9eDADo3r07Jk6ciMjISEyaNAn37t3DkCFD0LNnT7i5uQEAJk2ahM8++wzly5dHmzZt8PjxY+zbtw9DhgzRqr4JEyagQYMGqFmzJrKzs/Hnn3+qAhgRGTYGICIyWFu3boWHh4daW9WqVXHhwgUA4gittWvX4vPPP4eHhwfWrFmDGjVqAABsbW2xbds2DBs2DI0aNYKtrS06duyImTNnqvYVGRmJZ8+eYdasWRg1ahRcXFzQqVMnreuztLTE2LFjcf36ddjY2CA4OBhr167Vw5ETUXGTCYIgSF0EEZGuZDIZNmzYgPDwcKlLISIjxD5AREREZHIYgIiIiMjksA8QERklXr0noqLgGSAiIiIyOQxAREREZHIYgIiIiMjkMAARERGRyWEAIiIiIpPDAEREREQmhwGIiIiITA4DEBEREZkcBiAiIiIyOf8Hw4fJoTNf+rYAAAAASUVORK5CYII=", "text/plain": [ "
" ] @@ -871,7 +887,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m5s\u001b[0m 6ms/step - accuracy: 0.4957 - binary_accuracy: 0.0000e+00 - loss: 0.0000e+00\n", + "\u001b[1m782/782\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 2ms/step - accuracy: 0.4935 - binary_accuracy: 0.0000e+00 - loss: 0.0000e+00\n", "{'accuracy': 0.5, 'binary_accuracy': 0.0, 'loss': 0.0}\n" ] } @@ -910,15 +926,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 54ms/step\n" + "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 61ms/step\n" ] }, { "data": { "text/plain": [ - "array([[0.6689762 ],\n", - " [0.62865674],\n", - " [0.6049684 ]], dtype=float32)" + "array([[0.67346764],\n", + " [0.634105 ],\n", + " [0.61044645]], dtype=float32)" ] }, "execution_count": 27, @@ -1089,15 +1105,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 51ms/step\n" + "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 59ms/step\n" ] }, { "data": { "text/plain": [ - "array([[0.6689762 ],\n", - " [0.62865674],\n", - " [0.6049684 ]], dtype=float32)" + "array([[0.67346764],\n", + " [0.634105 ],\n", + " [0.61044645]], dtype=float32)" ] }, "execution_count": 31, @@ -1143,7 +1159,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 33, "id": "31de0c5f", "metadata": {}, "outputs": [], @@ -1166,7 +1182,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 34, "id": "6b653c43", "metadata": {}, "outputs": [ @@ -1174,11 +1190,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/06 22:02:26 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", - "25/01/06 22:02:26 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "25/02/04 14:05:31 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/02/04 14:05:31 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "25/01/06 22:02:26 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" + "25/02/04 14:05:31 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" ] } ], @@ -1232,7 +1248,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 35, "id": "ef3309eb", "metadata": {}, "outputs": [], @@ -1253,7 +1269,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 36, "id": "bb05466f", "metadata": {}, "outputs": [ @@ -1263,7 +1279,7 @@ "StructType([StructField('text', StringType(), True)])" ] }, - "execution_count": 39, + "execution_count": 36, "metadata": {}, "output_type": "execute_result" } @@ -1275,7 +1291,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 37, "id": "3f0a594b", "metadata": {}, "outputs": [ @@ -1283,7 +1299,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/06 22:02:35 WARN TaskSetManager: Stage 0 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n", + "25/02/04 14:05:36 WARN TaskSetManager: Stage 0 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n", " \r" ] }, @@ -1293,7 +1309,7 @@ "[Row(text=\"Anyone remember the first CKY, CKY2K etc..? Back when it was about making crazy cool stuff, rather than watching Bam Margera act like a douchebag, spoiled 5 year old, super/rock-star wannabe.

The show used to be awesome, however, Bam's fame and wealth has led him to believe, that we now enjoy him acting childish and idiotic, more than actual cool stuff, that used to be in ex. CKY2K.

The acts are so repetitive, there's like nothing new, except annoying stupidity and rehearsed comments... The only things we see is Bam Margera, so busy showing us how much he doesn't care, how much money he got or whatsoever.

I really got nothing much left to say except, give us back CKY2K, cause Bam suck..

I enjoy watching Steve-o, Knoxville etc. a thousand times more.\")]" ] }, - "execution_count": 40, + "execution_count": 37, "metadata": {}, "output_type": "execute_result" } @@ -1304,7 +1320,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 38, "id": "9d9db063", "metadata": {}, "outputs": [ @@ -1312,7 +1328,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/06 22:02:36 WARN TaskSetManager: Stage 3 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" + "25/02/04 14:05:37 WARN TaskSetManager: Stage 3 contains a task of very large size (4021 KiB). The maximum recommended task size is 1000 KiB.\n" ] } ], @@ -1336,7 +1352,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 39, "id": "1c081557", "metadata": {}, "outputs": [], @@ -1348,7 +1364,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 40, "id": "60af570a", "metadata": {}, "outputs": [], @@ -1359,7 +1375,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 41, "id": "a690f6df", "metadata": {}, "outputs": [], @@ -1382,7 +1398,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 42, "id": "7b7a8395-e2ae-4c3c-bf57-763dfde600ad", "metadata": {}, "outputs": [], @@ -1406,7 +1422,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 43, "id": "8c0524cf-3a75-4fb8-8025-f0654acce13e", "metadata": {}, "outputs": [], @@ -1457,7 +1473,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 44, "id": "0d603644-d938-4c87-aa8a-2512251638d5", "metadata": {}, "outputs": [], @@ -1469,7 +1485,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 45, "id": "0b480622-8dc1-4879-933e-c43112768630", "metadata": {}, "outputs": [ @@ -1484,8 +1500,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 5.29 ms, sys: 4.43 ms, total: 9.73 ms\n", - "Wall time: 4.3 s\n" + "CPU times: user 6.81 ms, sys: 3.75 ms, total: 10.6 ms\n", + "Wall time: 4.62 s\n" ] }, { @@ -1504,7 +1520,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 46, "id": "31b0a262-387e-4a5e-a60e-b9b8ee456199", "metadata": {}, "outputs": [ @@ -1512,8 +1528,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 4.94 ms, sys: 0 ns, total: 4.94 ms\n", - "Wall time: 150 ms\n" + "CPU times: user 4.58 ms, sys: 0 ns, total: 4.58 ms\n", + "Wall time: 142 ms\n" ] } ], @@ -1525,7 +1541,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 47, "id": "7ef9e431-59f5-4b29-9f79-ae16a9cfb0b9", "metadata": {}, "outputs": [ @@ -1533,8 +1549,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.38 ms, sys: 2.54 ms, total: 4.92 ms\n", - "Wall time: 206 ms\n" + "CPU times: user 903 μs, sys: 4.09 ms, total: 5 ms\n", + "Wall time: 222 ms\n" ] } ], @@ -1546,7 +1562,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 48, "id": "9a325ee2-3268-414a-bb75-a5fcf794f512", "metadata": { "scrolled": true @@ -1559,26 +1575,26 @@ "+--------------------------------------------------------------------------------+----------+\n", "| lines| preds|\n", "+--------------------------------------------------------------------------------+----------+\n", - "|The only reason I'm even giving this movie a 4 is because it was made in to a...|0.52321863|\n", - "|Awkward disaster mishmash has a team of scavengers coming across the overturn...|0.55067354|\n", - "|Here is a fantastic concept for a film - a series of meteors crash into a sma...| 0.6197893|\n", - "| I walked out of the cinema having suffered this film after 30 mins| 0.5503541|\n", - "|A wildly uneven film where the major problem is the uneasy mix of comedy and ...| 0.5540192|\n", - "|Leonard Rossiter and Frances de la Tour carry this film, not without a strugg...| 0.5467422|\n", - "| A good cast| 0.5688838|\n", - "|Yet again, I appear to be the only person on planet Earth who is capable of c...|0.55650306|\n", - "|As a serious horror fan, I get that certain marketing ploys are used to sell ...| 0.5629433|\n", - "|Upon writing this review I have difficulty trying to think of what to write a...| 0.5383269|\n", - "| Simply awful| 0.5275883|\n", - "|I am a fan of Ed Harris' work and I really had high expectations about this film|0.55910736|\n", - "| Well|0.56994545|\n", - "| This is a new approach to comedy| 0.5674365|\n", - "| It's been mentioned by others the inane dialogue in this series and I agree|0.55741817|\n", - "|One of the most boring movies I've ever had to sit through, it's completely f...| 0.5303776|\n", - "|This movie was playing on Lifetime Movie Network last month and I decided to ...| 0.5663204|\n", - "| 1983's \"Frightmare\" is an odd little film| 0.560836|\n", - "| 'Felony' is a B-movie| 0.5602156|\n", - "| This movie defines the word \"confused\"| 0.5535761|\n", + "|The only reason I'm even giving this movie a 4 is because it was made in to a...| 0.571606|\n", + "|Awkward disaster mishmash has a team of scavengers coming across the overturn...| 0.6264358|\n", + "|Here is a fantastic concept for a film - a series of meteors crash into a sma...| 0.6764294|\n", + "| I walked out of the cinema having suffered this film after 30 mins| 0.6258814|\n", + "|A wildly uneven film where the major problem is the uneasy mix of comedy and ...|0.63658905|\n", + "|Leonard Rossiter and Frances de la Tour carry this film, not without a strugg...| 0.633625|\n", + "| A good cast|0.65998995|\n", + "|Yet again, I appear to be the only person on planet Earth who is capable of c...| 0.6435825|\n", + "|As a serious horror fan, I get that certain marketing ploys are used to sell ...| 0.6453945|\n", + "|Upon writing this review I have difficulty trying to think of what to write a...|0.61587423|\n", + "| Simply awful| 0.594154|\n", + "|I am a fan of Ed Harris' work and I really had high expectations about this film| 0.6366444|\n", + "| Well|0.65976477|\n", + "| This is a new approach to comedy| 0.6555772|\n", + "| It's been mentioned by others the inane dialogue in this series and I agree| 0.6534178|\n", + "|One of the most boring movies I've ever had to sit through, it's completely f...| 0.5919746|\n", + "|This movie was playing on Lifetime Movie Network last month and I decided to ...| 0.6527056|\n", + "| 1983's \"Frightmare\" is an odd little film|0.64622015|\n", + "| 'Felony' is a B-movie|0.64882356|\n", + "| This movie defines the word \"confused\"|0.63689107|\n", "+--------------------------------------------------------------------------------+----------+\n", "only showing top 20 rows\n", "\n" @@ -1617,7 +1633,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 49, "id": "f4f14c8f", "metadata": {}, "outputs": [], @@ -1645,7 +1661,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 50, "id": "9614a192", "metadata": {}, "outputs": [], @@ -1671,7 +1687,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 51, "id": "32d0142a", "metadata": {}, "outputs": [], @@ -1684,24 +1700,19 @@ "id": "edddffb9", "metadata": {}, "source": [ - "Import the utility functions from pytriton_utils.py:" + "Import the helper class from pytriton_utils.py:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 52, "id": "444bad3f", "metadata": {}, "outputs": [], "source": [ "sc.addPyFile(\"pytriton_utils.py\")\n", "\n", - "from pytriton_utils import (\n", - " use_stage_level_scheduling,\n", - " find_ports,\n", - " start_triton,\n", - " stop_triton\n", - ")" + "from pytriton_utils import TritonServerManager" ] }, { @@ -1714,7 +1725,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 53, "id": "a4d37d33", "metadata": {}, "outputs": [], @@ -1793,6 +1804,7 @@ " )\n", "\n", " def _stop_triton(signum, frame):\n", + " # The server manager sends SIGTERM to stop the server; this function ensures graceful cleanup.\n", " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", " triton.stop()\n", "\n", @@ -1821,7 +1833,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 54, "id": "a01c6198", "metadata": {}, "outputs": [], @@ -1832,82 +1844,38 @@ }, { "cell_type": "markdown", - "id": "e4eaf6da", + "id": "fcdb7c5a", "metadata": {}, "source": [ - "To ensure that only one Triton inference server is started per node, we use stage-level scheduling to delegate each task to a separate GPU. " + "The `TritonClusterManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n", + "- Find available ports for HTTP/gRPC/metrics\n", + "- Deploy a server on each node via stage-level scheduling\n", + "- Gracefully shutdown servers across nodes" ] }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 55, "id": "4d5dc419", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - } - ], - "source": [ - "sc = spark.sparkContext\n", - "nodeRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "nodeRDD = use_stage_level_scheduling(spark, nodeRDD)" - ] - }, - { - "cell_type": "markdown", - "id": "0bdba73f", - "metadata": {}, - "source": [ - "Triton occupies ports for HTTP requests, GRPC requests, and the metrics service." - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "id": "7fa58218", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using ports [7000, 7001, 7002]\n" - ] - } - ], + "outputs": [], "source": [ "model_name = \"TextModel\"\n", - "ports = find_ports()\n", - "assert len(ports) == 3\n", - "print(f\"Using ports {ports}\")" + "server_manager = TritonServerManager(num_nodes=num_nodes, model_name=model_name, model_path=triton_model_path)" ] }, { "cell_type": "code", - "execution_count": 61, - "id": "bdcf9187", + "execution_count": null, + "id": "20198644", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 19:> (0 + 1) / 1]\r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Triton Server PIDs:\n", - " {\n", - " \"cb4ae00-lcedt\": 2897388\n", - "}\n" + "2025-02-07 11:03:44,809 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-07 11:03:44,810 - INFO - Starting 1 servers.\n" ] }, { @@ -1916,14 +1884,20 @@ "text": [ " \r" ] + }, + { + "data": { + "text/plain": [ + "{'cb4ae00-lcedt': (2020631, [7000, 7001, 7002])}" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "pids = nodeRDD.barrier().mapPartitions(lambda _: start_triton(triton_server_fn=triton_server,\n", - " ports=ports,\n", - " model_name=model_name,\n", - " model_path=triton_model_path)).collectAsMap()\n", - "print(\"Triton Server PIDs:\\n\", json.dumps(pids, indent=4))" + "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n", + "server_manager.start_servers(triton_server)" ] }, { @@ -1934,27 +1908,45 @@ "#### Define client function" ] }, + { + "cell_type": "markdown", + "id": "798c2815", + "metadata": {}, + "source": [ + "Get the hostname -> url mapping from the server manager:" + ] + }, { "cell_type": "code", - "execution_count": 62, - "id": "d590cd25", + "execution_count": null, + "id": "813d42cf", "metadata": {}, "outputs": [], "source": [ - "url = f\"http://localhost:{ports[0]}\"" + "host_to_http_url = server_manager.host_to_http_url # or server_manager.host_to_grpc_url" + ] + }, + { + "cell_type": "markdown", + "id": "f16617e3", + "metadata": {}, + "source": [ + "Define the Triton inference function, which returns a predict function for batch inference through the server:" ] }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 58, "id": "0ad47438", "metadata": {}, "outputs": [], "source": [ - "def triton_fn(url, model_name):\n", + "def triton_fn(model_name, host_to_url):\n", + " import socket\n", " import numpy as np\n", " from pytriton.client import ModelClient\n", "\n", + " url = host_to_url[socket.gethostname()]\n", " print(f\"CLIENT: Connecting to {model_name} at {url}\")\n", "\n", " def infer_batch(inputs):\n", @@ -1968,6 +1960,18 @@ " return infer_batch" ] }, + { + "cell_type": "code", + "execution_count": 61, + "id": "8e06d33f-5cef-4a48-afc3-5d468f8ec2b4", + "metadata": {}, + "outputs": [], + "source": [ + "classify = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_http_url),\n", + " return_type=FloatType(),\n", + " batch_size=64)" + ] + }, { "cell_type": "markdown", "id": "91974885", @@ -1978,7 +1982,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 59, "id": "41106a02-236e-4cb3-ac51-76aa64b663c2", "metadata": {}, "outputs": [], @@ -1988,7 +1992,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 60, "id": "e851870b", "metadata": {}, "outputs": [ @@ -1996,7 +2000,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "25/01/06 22:02:44 WARN CacheManager: Asked to cache already cached data.\n" + "25/02/04 14:05:48 WARN CacheManager: Asked to cache already cached data.\n" ] } ], @@ -2006,19 +2010,7 @@ }, { "cell_type": "code", - "execution_count": 66, - "id": "8e06d33f-5cef-4a48-afc3-5d468f8ec2b4", - "metadata": {}, - "outputs": [], - "source": [ - "classify = predict_batch_udf(partial(triton_fn, url=url, model_name=\"TextModel\"),\n", - " return_type=FloatType(),\n", - " batch_size=64)" - ] - }, - { - "cell_type": "code", - "execution_count": 67, + "execution_count": 62, "id": "d89e74ad-e551-4bfa-ad08-98725878630a", "metadata": {}, "outputs": [ @@ -2026,15 +2018,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "[Stage 23:> (0 + 8) / 8]\r" + "[Stage 24:==============> (2 + 6) / 8]\r" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 10.4 ms, sys: 7 ms, total: 17.4 ms\n", - "Wall time: 2.53 s\n" + "CPU times: user 2.92 ms, sys: 4.06 ms, total: 6.97 ms\n", + "Wall time: 1.03 s\n" ] }, { @@ -2053,7 +2045,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 63, "id": "b4fa7fc9-341c-49a6-9af2-e316f2355d67", "metadata": {}, "outputs": [ @@ -2061,8 +2053,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 2.59 ms, sys: 631 μs, total: 3.22 ms\n", - "Wall time: 214 ms\n" + "CPU times: user 1.39 ms, sys: 2.15 ms, total: 3.53 ms\n", + "Wall time: 237 ms\n" ] } ], @@ -2074,7 +2066,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 64, "id": "564f999b", "metadata": {}, "outputs": [ @@ -2082,8 +2074,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 288 μs, sys: 3.66 ms, total: 3.95 ms\n", - "Wall time: 245 ms\n" + "CPU times: user 862 μs, sys: 2.77 ms, total: 3.63 ms\n", + "Wall time: 225 ms\n" ] } ], @@ -2095,7 +2087,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 65, "id": "9222e8a9", "metadata": {}, "outputs": [ @@ -2106,26 +2098,26 @@ "+--------------------------------------------------------------------------------+----------+\n", "| lines| preds|\n", "+--------------------------------------------------------------------------------+----------+\n", - "|The only reason I'm even giving this movie a 4 is because it was made in to a...| 0.5441438|\n", - "|Awkward disaster mishmash has a team of scavengers coming across the overturn...|0.58016133|\n", - "|Here is a fantastic concept for a film - a series of meteors crash into a sma...|0.55131954|\n", - "| I walked out of the cinema having suffered this film after 30 mins| 0.542057|\n", - "|A wildly uneven film where the major problem is the uneasy mix of comedy and ...| 0.5196002|\n", - "|Leonard Rossiter and Frances de la Tour carry this film, not without a strugg...|0.53112733|\n", - "| A good cast| 0.5486873|\n", - "|Yet again, I appear to be the only person on planet Earth who is capable of c...| 0.5343111|\n", - "|As a serious horror fan, I get that certain marketing ploys are used to sell ...| 0.5497148|\n", - "|Upon writing this review I have difficulty trying to think of what to write a...| 0.5581456|\n", - "| Simply awful| 0.5701754|\n", - "|I am a fan of Ed Harris' work and I really had high expectations about this film| 0.5510578|\n", - "| Well|0.55721515|\n", - "| This is a new approach to comedy|0.56038314|\n", - "| It's been mentioned by others the inane dialogue in this series and I agree| 0.5451202|\n", - "|One of the most boring movies I've ever had to sit through, it's completely f...|0.56161135|\n", - "|This movie was playing on Lifetime Movie Network last month and I decided to ...| 0.5555233|\n", - "| 1983's \"Frightmare\" is an odd little film| 0.5363368|\n", - "| 'Felony' is a B-movie|0.55682427|\n", - "| This movie defines the word \"confused\"| 0.5630136|\n", + "|The only reason I'm even giving this movie a 4 is because it was made in to a...|0.67212176|\n", + "|Awkward disaster mishmash has a team of scavengers coming across the overturn...|0.63807774|\n", + "|Here is a fantastic concept for a film - a series of meteors crash into a sma...|0.65471745|\n", + "| I walked out of the cinema having suffered this film after 30 mins| 0.6527998|\n", + "|A wildly uneven film where the major problem is the uneasy mix of comedy and ...| 0.6405446|\n", + "|Leonard Rossiter and Frances de la Tour carry this film, not without a strugg...|0.63534474|\n", + "| A good cast|0.64761806|\n", + "|Yet again, I appear to be the only person on planet Earth who is capable of c...|0.66956663|\n", + "|As a serious horror fan, I get that certain marketing ploys are used to sell ...|0.62346375|\n", + "|Upon writing this review I have difficulty trying to think of what to write a...| 0.681598|\n", + "| Simply awful| 0.6537583|\n", + "|I am a fan of Ed Harris' work and I really had high expectations about this film| 0.6382922|\n", + "| Well|0.65424603|\n", + "| This is a new approach to comedy| 0.6628315|\n", + "| It's been mentioned by others the inane dialogue in this series and I agree|0.63345987|\n", + "|One of the most boring movies I've ever had to sit through, it's completely f...| 0.6459369|\n", + "|This movie was playing on Lifetime Movie Network last month and I decided to ...|0.65335083|\n", + "| 1983's \"Frightmare\" is an odd little film|0.65602964|\n", + "| 'Felony' is a B-movie| 0.6583404|\n", + "| This movie defines the word \"confused\"| 0.6217103|\n", "+--------------------------------------------------------------------------------+----------+\n", "only showing top 20 rows\n", "\n" @@ -2148,22 +2140,16 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 66, "id": "a71ac9b6-47a2-4306-bc40-9ce7b4e968ec", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requesting stage-level resources: (cores=5, gpu=1.0)\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "2025-02-04 14:05:50,166 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-04 14:06:00,351 - INFO - Sucessfully stopped 1 servers. \n" ] }, { @@ -2172,20 +2158,18 @@ "[True]" ] }, - "execution_count": 71, + "execution_count": 66, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "shutdownRDD = sc.parallelize(list(range(num_nodes)), num_nodes)\n", - "shutdownRDD = use_stage_level_scheduling(spark, shutdownRDD)\n", - "shutdownRDD.barrier().mapPartitions(lambda _: stop_triton(pids)).collect()" + "server_manager.stop_servers()" ] }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 67, "id": "54a90574-7cbb-487b-b7a8-dcda0e6e301f", "metadata": {}, "outputs": [], From 1bc43fb06eb7b9027b0e5aea455c417060bc0b6e Mon Sep 17 00:00:00 2001 From: "Rishi C." <77904151+rishic3@users.noreply.github.com> Date: Fri, 14 Feb 2025 13:09:53 -0800 Subject: [PATCH 09/12] Add LLM batch inference examples (#493) Add deepseek-r1 and gemma-7b LLM batch inference notebooks. Updated CSP instructions since these notebooks require >20GB GPU RAM (A10/L4). --------- Signed-off-by: Rishi Chandra --- .../Spark-DL/dl_inference/README.md | 26 +- .../dl_inference/databricks/README.md | 13 +- .../databricks/setup/start_cluster.sh | 11 +- .../Spark-DL/dl_inference/dataproc/README.md | 3 +- .../dataproc/setup/start_cluster.sh | 46 +- .../huggingface/deepseek-r1_torch.ipynb | 878 ++++++++++++++++++ .../huggingface/gemma-7b_torch.ipynb | 850 +++++++++++++++++ .../dl_inference/images/spark-pytriton.png | Bin 112801 -> 42644 bytes .../Spark-DL/dl_inference/pytriton_utils.py | 45 +- 9 files changed, 1811 insertions(+), 61 deletions(-) create mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/deepseek-r1_torch.ipynb create mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/gemma-7b_torch.ipynb diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md b/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md index 8662ea3d..451256c2 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md @@ -43,15 +43,18 @@ Below is a full list of the notebooks with links to the examples they are based | | Framework | Notebook Name | Description | Link | ------------- | ------------- | ------------- | ------------- | ------------- -| 1 | PyTorch | Image Classification | Training a model to predict clothing categories in FashionMNIST, including accelerated inference with Torch-TensorRT. | [Link](https://pytorch.org/tutorials/beginner/basics/quickstart_tutorial.html) -| 2 | PyTorch | Housing Regression | Training a model to predict housing prices in the California Housing Dataset, including accelerated inference with Torch-TensorRT. | [Link](https://github.com/christianversloot/machine-learning-articles/blob/main/how-to-create-a-neural-network-for-regression-with-pytorch.md) -| 3 | Tensorflow | Image Classification | Training a model to predict hand-written digits in MNIST. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/save_and_load.ipynb) -| 4 | Tensorflow | Keras Preprocessing | Training a model with preprocessing layers to predict likelihood of pet adoption in the PetFinder mini dataset. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/preprocessing_layers.ipynb) -| 5 | Tensorflow | Keras Resnet50 | Training ResNet-50 to perform flower recognition from flower images. | [Link](https://docs.databricks.com/en/_extras/notebooks/source/deep-learning/keras-metadata.html) -| 6 | Tensorflow | Text Classification | Training a model to perform sentiment analysis on the IMDB dataset. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/text_classification.ipynb) -| 7+8 | HuggingFace | Conditional Generation | Sentence translation using the T5 text-to-text transformer for both Torch and Tensorflow. | [Link](https://huggingface.co/docs/transformers/model_doc/t5#t5) -| 9+10 | HuggingFace | Pipelines | Sentiment analysis using Huggingface pipelines for both Torch and Tensorflow. | [Link](https://huggingface.co/docs/transformers/quicktour#pipeline-usage) -| 11 | HuggingFace | Sentence Transformers | Sentence embeddings using SentenceTransformers in Torch. | [Link](https://huggingface.co/sentence-transformers) +| 1 | HuggingFace | DeepSeek-R1 | LLM batch inference using the DeepSeek-R1-Distill-Llama reasoning model. | [Link](https://huggingface.co/deepseek-ai/DeepSeek-R1) +| 2 | HuggingFace | Gemma-7b | LLM batch inference using the lightweight Google Gemma-7b model. | [Link](https://huggingface.co/google/gemma-7b-it) +| 3 | HuggingFace | Sentence Transformers | Sentence embeddings using SentenceTransformers in Torch. | [Link](https://huggingface.co/sentence-transformers) +| 4+5 | HuggingFace | Conditional Generation | Sentence translation using the T5 text-to-text transformer for both Torch and Tensorflow. | [Link](https://huggingface.co/docs/transformers/model_doc/t5#t5) +| 6+7 | HuggingFace | Pipelines | Sentiment analysis using Huggingface pipelines for both Torch and Tensorflow. | [Link](https://huggingface.co/docs/transformers/quicktour#pipeline-usage) +| 8 | PyTorch | Image Classification | Training a model to predict clothing categories in FashionMNIST, and deploying with Torch-TensorRT accelerated inference. | [Link](https://pytorch.org/tutorials/beginner/basics/quickstart_tutorial.html) +| 9 | PyTorch | Housing Regression | Training and deploying a model to predict housing prices in the California Housing Dataset, and deploying with Torch-TensorRT accelerated inference. | [Link](https://github.com/christianversloot/machine-learning-articles/blob/main/how-to-create-a-neural-network-for-regression-with-pytorch.md) +| 10 | Tensorflow | Image Classification | Training and deploying a model to predict hand-written digits in MNIST. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/save_and_load.ipynb) +| 11 | Tensorflow | Keras Preprocessing | Training and deploying a model with preprocessing layers to predict likelihood of pet adoption in the PetFinder mini dataset. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/preprocessing_layers.ipynb) +| 12 | Tensorflow | Keras Resnet50 | Deploying ResNet-50 to perform flower recognition from flower images. | [Link](https://docs.databricks.com/en/_extras/notebooks/source/deep-learning/keras-metadata.html) +| 13 | Tensorflow | Text Classification | Training and deploying a model to perform sentiment analysis on the IMDB dataset. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/text_classification.ipynb) + ## Running Locally @@ -130,9 +133,8 @@ The notebooks use [PyTriton](https://github.com/triton-inference-server/pytriton The diagram above shows how Spark distributes inference tasks to run on the Triton Inference Server, with PyTriton handling request/response communication with the server. The process looks like this: -- Distribute a PyTriton task across the Spark cluster, instructing each worker to launch a Triton server process. - - Use stage-level scheduling to ensure there is a 1:1 mapping between worker nodes and servers. -- Define a Triton inference function, which contains a client that binds to the local server on a given worker and sends inference requests. +- Prior to inference, launch a Triton server process on each node. +- Define a Triton predict function, which creates a client that binds to the local server and sends/receives inference requests. - Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark. - Finally, distribute a shutdown signal to terminate the Triton server processes on each worker. diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/README.md b/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/README.md index 58c7d12b..cc760522 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/README.md +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/README.md @@ -34,22 +34,25 @@ databricks workspace import $INIT_DEST --format AUTO --file $INIT_SRC ``` -6. Launch the cluster with the provided script (note that the script specifies **Azure instances** by default; change as needed): +6. Launch the cluster with the provided script. By default the script will create a cluster with 4 A10 worker nodes and 1 A10 driver node. (Note that the script uses **Azure instances** by default; change as needed). ```shell cd setup chmod +x start_cluster.sh ./start_cluster.sh ``` - OR, start the cluster from the Databricks UI: - Go to `Compute > Create compute` and set the desired cluster settings. - Integration with Triton inference server uses stage-level scheduling (Spark>=3.4.0). Make sure to: - - use a cluster with GPU resources + - use a cluster with GPU resources (for LLM examples, make sure the selected GPUs have sufficient RAM) - set a value for `spark.executor.cores` - ensure that `spark.executor.resource.gpu.amount` = 1 - Under `Advanced Options > Init Scripts`, upload the init script from your workspace. - - Under environment variables, set `FRAMEWORK=torch` or `FRAMEWORK=tf` based on the notebook used. - - For Tensorflow notebooks, we recommend setting the environment variable `TF_GPU_ALLOCATOR=cuda_malloc_async` (especially for Huggingface LLM models), which enables the CUDA driver to implicity release unused memory from the pool. + - Under environment variables, set: + - `FRAMEWORK=torch` or `FRAMEWORK=tf` based on the notebook used. + - `HF_HOME=/dbfs/FileStore/hf_home` to cache Huggingface models in DBFS. + - `TF_GPU_ALLOCATOR=cuda_malloc_async` to implicity release unused GPU memory in Tensorflow notebooks. + + 7. Navigate to the notebook in your workspace and attach it to the cluster. The default cluster name is `spark-dl-inference-$FRAMEWORK`. \ No newline at end of file diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/start_cluster.sh b/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/start_cluster.sh index 98c1d5ac..457b080b 100755 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/start_cluster.sh +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/databricks/setup/start_cluster.sh @@ -14,6 +14,9 @@ if [[ -z ${FRAMEWORK} ]]; then exit 1 fi +# Modify the node_type_id and driver_node_type_id below if you don't have this specific instance type. +# Modify executor.cores=(cores per node) and task.resource.gpu.amount=(1/executor cores) accordingly. +# We recommend selecting A10/L4+ instances for these examples. json_config=$(cat < `Clusters` > `(Cluster Name)` > `Web Interfaces` > `Jupyter/Lab` diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/start_cluster.sh b/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/start_cluster.sh index b44df43b..35840bae 100755 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/start_cluster.sh +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/dataproc/setup/start_cluster.sh @@ -77,29 +77,29 @@ else exit 1 fi -# start cluster if not already running if gcloud dataproc clusters list | grep -q "${cluster_name}"; then echo "Cluster ${cluster_name} already exists." -else - gcloud dataproc clusters create ${cluster_name} \ - --image-version=2.2-ubuntu \ - --region ${COMPUTE_REGION} \ - --master-machine-type n1-standard-16 \ - --num-workers 4 \ - --worker-min-cpu-platform="Intel Skylake" \ - --worker-machine-type n1-standard-16 \ - --master-accelerator type=nvidia-tesla-t4,count=1 \ - --worker-accelerator type=nvidia-tesla-t4,count=1 \ - --initialization-actions gs://${SPARK_DL_HOME}/init/spark-rapids.sh,${INIT_PATH} \ - --metadata gpu-driver-provider="NVIDIA" \ - --metadata gcs-bucket=${GCS_BUCKET} \ - --metadata spark-dl-home=${SPARK_DL_HOME} \ - --metadata requirements="${requirements}" \ - --worker-local-ssd-interface=NVME \ - --optional-components=JUPYTER \ - --bucket ${GCS_BUCKET} \ - --enable-component-gateway \ - --max-idle "60m" \ - --subnet=default \ - --no-shielded-secure-boot + exit 0 fi + +CLUSTER_PARAMS=( + --image-version=2.2-ubuntu + --region ${COMPUTE_REGION} + --num-workers 4 + --master-machine-type g2-standard-8 + --worker-machine-type g2-standard-8 + --initialization-actions gs://${SPARK_DL_HOME}/init/spark-rapids.sh,${INIT_PATH} + --metadata gpu-driver-provider="NVIDIA" + --metadata gcs-bucket=${GCS_BUCKET} + --metadata spark-dl-home=${SPARK_DL_HOME} + --metadata requirements="${requirements}" + --worker-local-ssd-interface=NVME + --optional-components=JUPYTER + --bucket ${GCS_BUCKET} + --enable-component-gateway + --max-idle "60m" + --subnet=default + --no-shielded-secure-boot +) + +gcloud dataproc clusters create ${cluster_name} "${CLUSTER_PARAMS[@]}" diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/deepseek-r1_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/deepseek-r1_torch.ipynb new file mode 100644 index 00000000..55f0d879 --- /dev/null +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/deepseek-r1_torch.ipynb @@ -0,0 +1,878 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "# PySpark LLM Inference: DeepSeek-R1\n", + "\n", + "In this notebook, we demonstrate distributed batch inference with [DeepSeek-R1](https://github.com/deepseek-ai/DeepSeek-R1), using open weights on Huggingface.\n", + "\n", + "We use [DeepSeek-R1-Distill-Llama-8B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B) as demonstration. DeepSeek's distilled models are based on open-source LLMs (such as Llama/Qwen), and are fine-tuned using samples generated by DeepSeek-R1 to perform multi-step reasoning tasks.\n", + "\n", + "**Note:** Running this model on GPU with 16-bit precision requires **~18GB** of GPU RAM. Make sure your instances have sufficient GPU capacity." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Manually enable Huggingface tokenizer parallelism to avoid disabling with PySpark parallelism.\n", + "# See (https://github.com/huggingface/transformers/issues/5486) for more info. \n", + "import os\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check the cluster environment to handle any platform-specific configurations." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n", + "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n", + "on_standalone = not (on_databricks or on_dataproc)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For cloud environments, set the huggingface cache dir to DBFS/GCS so that executors can load the model from cache." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "if on_databricks:\n", + " hf_home = \"/dbfs/FileStore/hf_home\"\n", + " dbutils.fs.mkdirs(hf_home)\n", + " os.environ[\"HF_HOME\"] = hf_home\n", + "elif on_dataproc:\n", + " hf_home = \"/mnt/gcs/hf_home\"\n", + " os.mkdir(hf_home) if not os.path.exists(hf_home) else None\n", + " os.environ[\"HF_HOME\"] = hf_home" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Warmup: Running locally\n", + "\n", + "**Note:** If the driver node does not have sufficient GPU capacity, proceed to the PySpark section." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0ab193983c774a948e375407d7df1f83", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading checkpoint shards: 0%| | 0/2 [00:00\n", + "\n", + "To determine how many **r's** are in the word **strawberry**, let's follow these steps:\n", + "\n", + "1. **Write down the word:**\n", + " \n", + " S T R A W B E R R Y\n", + "\n", + "2. **Identify and count each occurrence of the letter R:**\n", + " \n", + " - **1.** S - no R\n", + " - **2.** T - no R\n", + " - **3.** R - **1 R**\n", + " - **4.** A - no R\n", + " - **5.** W - no R\n", + " - **6.** B - no R\n", + " - **7.** E - no R\n", + " - **8.** R - **2 R's**\n", + " - **9.** R - **3 R's**\n", + " - **10.** Y - no R\n", + "\n", + "3. **Total count of R's:**\n", + " \n", + " There are **3 R's** in the word **strawberry**.\n", + "\n", + "\\boxed{3}\n" + ] + } + ], + "source": [ + "res = pipe([\"How many r's are there in strawberry?\"], max_new_tokens=512, temperature=0.1)\n", + "print(\"\\n\", res[0][0]['generated_text'])" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " Which number is bigger: 9.9 or 9.11? Let's see.\n", + "\n", + "First, I need to compare the whole number parts of both numbers. Both 9.9 and 9.11 have the same whole number part, which is 9.\n", + "\n", + "Since the whole numbers are equal, I'll compare the decimal parts. For 9.9, the decimal part is 0.9, and for 9.11, the decimal part is 0.11.\n", + "\n", + "To make it easier, I can express 0.9 as 0.90. Now, comparing 0.90 and 0.11, it's clear that 0.90 is greater than 0.11.\n", + "\n", + "Therefore, 9.9 is bigger than 9.11.\n", + "\n", + "\n", + "To determine which number is larger between **9.9** and **9.11**, let's compare them step by step.\n", + "\n", + "1. **Compare the Whole Numbers:**\n", + " - Both numbers have the same whole number part: **9**.\n", + " \n", + "2. **Compare the Decimal Parts:**\n", + " - **9.9** can be written as **9.90**.\n", + " - **9.11** remains **9.11**.\n", + " \n", + "3. **Analyze the Decimal Comparison:**\n", + " - Compare the tenths place:\n", + " - **9.90** has **9** in the tenths place.\n", + " - **9.11** has **1** in the tenths place.\n", + " - Since **9 > 1**, **9.90** is greater than **9.11**.\n", + "\n", + "4. **Conclusion:**\n", + " - Therefore, **9.9** is larger than **9.11**.\n", + "\n", + "\\boxed{9.9}\n" + ] + } + ], + "source": [ + "res = pipe([\"Which number is bigger: 9.9 or 9.11?\"], max_new_tokens=512, temperature=0.1)\n", + "print(\"\\n\", res[0][0]['generated_text'])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "\n", + "# Unload the model from GPU memory.\n", + "del pipe\n", + "torch.cuda.empty_cache()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## PySpark" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql.types import *\n", + "from pyspark import SparkConf\n", + "from pyspark.sql import SparkSession\n", + "from pyspark.sql.functions import pandas_udf, col, struct, length\n", + "from pyspark.ml.functions import predict_batch_udf" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import datasets\n", + "from datasets import load_dataset\n", + "datasets.disable_progress_bars()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Create Spark Session\n", + "\n", + "For local standalone clusters, we'll connect to the cluster and create the Spark Session. \n", + "For CSP environments, Spark will either be preconfigured (Databricks) or we'll need to create the Spark Session (Dataproc)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/02/10 09:40:01 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/02/10 09:40:01 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", + "25/02/10 09:40:02 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" + ] + } + ], + "source": [ + "conf = SparkConf()\n", + "\n", + "if 'spark' not in globals():\n", + " if on_standalone:\n", + " import socket\n", + " conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + " hostname = socket.gethostname()\n", + " conf.setMaster(f\"spark://{hostname}:7077\")\n", + " conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", + " conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_env}/lib:{conda_env}/lib/python3.11/site-packages/nvidia_pytriton.libs:$LD_LIBRARY_PATH\")\n", + " elif on_dataproc:\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conda_lib_path=\"/opt/conda/miniconda3/lib\"\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_lib_path}:$LD_LIBRARY_PATH\")\n", + " conf.set(\"spark.executor.instances\", \"4\") # dataproc defaults to 2\n", + " conf.set(\"spark.executorEnv.HF_HOME\", hf_home)\n", + "\n", + " conf.set(\"spark.executor.cores\", \"8\")\n", + " conf.set(\"spark.task.maxFailures\", \"1\")\n", + " conf.set(\"spark.task.resource.gpu.amount\", \"0.125\")\n", + " conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n", + " conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", + " conf.set(\"spark.python.worker.reuse\", \"true\")\n", + "\n", + "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", + "sc = spark.sparkContext" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Load DataFrame\n", + "\n", + "Load the Orca Math Word Problems dataset from Huggingface and store in a Spark Dataframe." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = load_dataset(\"microsoft/orca-math-word-problems-200k\", split=\"train[:1%]\")\n", + "dataset = dataset.to_pandas()[\"question\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------+\n", + "| question|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "|Jungkook is the 5th place. Find the number of people who crossed the finish line faster than Jung...|\n", + "|A number divided by 10 is 6. Yoongi got the result by subtracting 15 from a certain number. What ...|\n", + "|Dongju selects a piece of paper with a number written on it, and wants to make a three-digit numb...|\n", + "|You wanted to subtract 46 from a number, but you accidentally subtract 59 and get 43. How much do...|\n", + "|The length of one span of Jinseo is about 12 centimeters (cm). When Jinseo measured the length of...|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "only showing top 5 rows\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "df = spark.createDataFrame(dataset, schema=StringType()).withColumnRenamed(\"value\", \"question\")\n", + "df.show(5, truncate=100)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "data_path = \"spark-dl-datasets/orca_math\"\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " data_path = \"dbfs:/FileStore/\" + data_path\n", + "\n", + "df.write.mode(\"overwrite\").json(data_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Triton Inference Server\n", + "We'll demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. \n", + "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server. \n", + "\n", + "The process looks like this:\n", + "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n", + "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n", + "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n", + "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n", + "\n", + "\"drawing\"" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Import the helper class from pytriton_utils.py:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "sc.addPyFile(\"pytriton_utils.py\")\n", + "\n", + "from pytriton_utils import TritonServerManager" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define the Triton Server function:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def triton_server(ports):\n", + " import time\n", + " import signal\n", + " import numpy as np\n", + " import torch\n", + " from transformers import pipeline\n", + " from pytriton.decorators import batch\n", + " from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n", + " from pytriton.triton import Triton, TritonConfig\n", + " from pyspark import TaskContext\n", + "\n", + " print(f\"SERVER: Initializing model on worker {TaskContext.get().partitionId()}.\")\n", + " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + " pipe = pipeline(\"text-generation\", model=\"deepseek-ai/DeepSeek-R1-Distill-Llama-8B\", torch_dtype=torch.bfloat16, device=device)\n", + " print(f\"SERVER: Using {device} device.\")\n", + "\n", + " @batch\n", + " def _infer_fn(**inputs):\n", + " prompts = np.squeeze(inputs[\"prompts\"]).tolist()\n", + " decoded_prompts = [p.decode(\"utf-8\") for p in prompts]\n", + " # limit responses to 256 tokens, since reasoning tasks can take a while\n", + " responses = pipe(decoded_prompts, max_new_tokens=256, temperature=0.2, return_full_text=False)\n", + " return {\n", + " \"responses\": np.array([r[0]['generated_text'] for r in responses]).reshape(-1, 1)\n", + " }\n", + "\n", + " workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n", + " triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n", + " with Triton(config=triton_conf, workspace=workspace_path) as triton:\n", + " triton.bind(\n", + " model_name=\"deepseek-r1\",\n", + " infer_func=_infer_fn,\n", + " inputs=[\n", + " Tensor(name=\"prompts\", dtype=object, shape=(-1,)),\n", + " ],\n", + " outputs=[\n", + " Tensor(name=\"responses\", dtype=object, shape=(-1,)),\n", + " ],\n", + " config=ModelConfig(\n", + " max_batch_size=16,\n", + " batcher=DynamicBatcher(max_queue_delay_microseconds=5000), # 5ms\n", + " ),\n", + " strict=True,\n", + " )\n", + "\n", + " def _stop_triton(signum, frame):\n", + " # The server manager sends SIGTERM to stop the server; this function ensures graceful cleanup.\n", + " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", + " triton.stop()\n", + "\n", + " signal.signal(signal.SIGTERM, _stop_triton)\n", + "\n", + " print(\"SERVER: Serving inference\")\n", + " triton.serve()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Start Triton servers" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Specify the number of nodes in the cluster.** \n", + "Following the README, the example standalone cluster uses 1 node. The example Databricks/Dataproc cluster scripts use 4 nodes by default. " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "# Change based on cluster setup\n", + "num_nodes = 1 if on_standalone else 4" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `TritonClusterManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n", + "- Find available ports for HTTP/gRPC/metrics\n", + "- Deploy a server on each node via stage-level scheduling\n", + "- Gracefully shutdown servers across nodes" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "model_name = \"deepseek-r1\"\n", + "server_manager = TritonServerManager(num_nodes=num_nodes, model_name=model_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-10 09:40:17,442 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-10 09:40:17,442 - INFO - Starting 1 servers.\n", + " \r" + ] + }, + { + "data": { + "text/plain": [ + "{'cb4ae00-lcedt': (272659, [7000, 7001, 7002])}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n", + "server_manager.start_servers(triton_server, wait_retries=24) # allow up to 2 minutes for model loading" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Define client function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Get the hostname -> url mapping from the server manager:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "host_to_grpc_url = server_manager.host_to_grpc_url # or server_manager.host_to_http_url" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define the Triton inference function, which returns a predict function for batch inference through the server:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "def triton_fn(model_name, host_to_url):\n", + " import socket\n", + " import numpy as np\n", + " from pytriton.client import ModelClient\n", + "\n", + " url = host_to_url[socket.gethostname()]\n", + " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", + "\n", + " def infer_batch(inputs):\n", + " with ModelClient(url, model_name, inference_timeout_s=500) as client:\n", + " flattened = np.squeeze(inputs).tolist()\n", + " # Encode batch\n", + " encoded_batch = [[text.encode(\"utf-8\")] for text in flattened]\n", + " encoded_batch_np = np.array(encoded_batch, dtype=np.bytes_)\n", + " # Run inference\n", + " result_data = client.infer_batch(encoded_batch_np)\n", + " result_data = np.squeeze(result_data[\"responses\"], -1)\n", + " return result_data\n", + " \n", + " return infer_batch" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "generate = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_grpc_url),\n", + " return_type=StringType(),\n", + " input_tensor_shapes=[[1]],\n", + " batch_size=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Load and preprocess DataFrame\n", + "\n", + "We'll select a few of the shorter questions for demonstration, since reasoning tasks can take a while." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.json(data_path)\n", + "df = df.filter(length(col(\"question\")) <= 100).limit(16).repartition(8).cache()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Run Inference" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 6:==============> (2 + 6) / 8]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 18.6 ms, sys: 8.31 ms, total: 26.9 ms\n", + "Wall time: 1min 46s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "%%time\n", + "# first pass caches model/fn\n", + "preds = df.withColumn(\"response\", generate(col(\"question\")))\n", + "results = preds.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 23:==================================================> (7 + 1) / 8]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 9.55 ms, sys: 4.51 ms, total: 14.1 ms\n", + "Wall time: 1min 45s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "%%time\n", + "preds = df.withColumn(\"response\", generate(\"question\"))\n", + "results = preds.collect()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sample output:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Q: There are 9 dogs and 23 cats. How many more cats are there than dogs? \n", + "\n", + "A: Let me think. So, I have 23 cats and 9 dogs. To find out how many more cats there are than dogs, I need to subtract the number of dogs from the number of cats. That would be 23 minus 9. Let me do the subtraction: 23 minus 9 is 14. So, there are 14 more cats than dogs.\n", + "\n", + "Wait, let me double-check that. If I have 9 dogs and 23 cats, subtracting the number of dogs from the number of cats should give me the difference. So, 23 minus 9 is indeed 14. Yeah, that seems right. I don't think I made a mistake there. So, the answer is 14 more cats than dogs.\n", + "\n", + "**Final Answer**\n", + "The number of cats exceeds the number of dogs by \\boxed{14}.\n", + "\\boxed{14}\n", + "\n", + "\n", + "To determine how many more cats there are than dogs, we subtract the number of dogs from the number of cats. \n", + "\n", + "Given:\n", + "- Number of cats = 23\n", + "- Number of dogs = 9\n", + "\n", + "The calculation is:\n", + "\\[ 23 - 9 = 14 \\]\n", + "\n", + "Thus, there are 14 more cats than dogs.\n", + "\n", + "\\[\n", + "\\boxed{14}\n", + "\\] \n", + "\n" + ] + } + ], + "source": [ + "print(f\"Q: {results[2].question} \\n\")\n", + "print(f\"A: {results[2].response} \\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Shut down server on each executor" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-10 09:43:36,499 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-10 09:43:41,701 - INFO - Sucessfully stopped 1 servers. \n" + ] + }, + { + "data": { + "text/plain": [ + "[True]" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "server_manager.stop_servers()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n", + " spark.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "spark-dl-torch", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/gemma-7b_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/gemma-7b_torch.ipynb new file mode 100644 index 00000000..9d6d43a2 --- /dev/null +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/gemma-7b_torch.ipynb @@ -0,0 +1,850 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "# PySpark LLM Inference: Gemma-7b\n", + "\n", + "In this notebook, we demonstrate distributed inference with the Google [Gemma-7b-instruct](https://huggingface.co/google/gemma-7b-it) LLM, using open-weights on Huggingface.\n", + "\n", + "The Gemma-7b-instruct is an instruction-fine-tuned version of the Gemma-7b base model.\n", + "\n", + "**Note:** Running this model on GPU with 16-bit precision requires **~18 GB** of GPU RAM. Make sure your instances have sufficient GPU capacity." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Manually enable Huggingface tokenizer parallelism to avoid disabling with PySpark parallelism.\n", + "# See (https://github.com/huggingface/transformers/issues/5486) for more info. \n", + "import os\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check the cluster environment to handle any platform-specific configurations." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n", + "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n", + "on_standalone = not (on_databricks or on_dataproc)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For cloud environments, set the huggingface cache dir to DBFS/GCS so that executors can load the model from cache." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "if on_databricks:\n", + " hf_home = \"/dbfs/FileStore/hf_home\"\n", + " dbutils.fs.mkdirs(hf_home)\n", + " os.environ[\"HF_HOME\"] = hf_home\n", + "elif on_dataproc:\n", + " hf_home = \"/mnt/gcs/hf_home\"\n", + " os.mkdir(hf_home) if not os.path.exists(hf_home) else None\n", + " os.environ[\"HF_HOME\"] = hf_home" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Warmup: Running locally\n", + "\n", + "**Note**: If the driver node does not have sufficient GPU capacity, proceed to the PySpark section." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First visit the [Gemma Huggingface repository](https://huggingface.co/google/gemma-7b-it) to accept the terms to access the model, then login via huggingface_hub." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from huggingface_hub import login\n", + "\n", + "login()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "58494ca5858c40e39f924ad330a65885", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading checkpoint shards: 0%| | 0/4 [00:00Write me a poem about Apache Spark.\n", + "\n", + "In the realm of big data, a spark ignites,\n", + "A framework born to conquer the night.\n", + "Apache Spark, a lightning-fast tool,\n", + "For processing data, swift and cool.\n", + "\n", + "With its resilient distributed architecture,\n", + "It slices through terabytes with grace.\n", + "No longer bound by memory's plight,\n", + "Spark empowers us to analyze with might.\n", + "\n", + "From Python to Scala, it's a versatile spark,\n", + "Unveiling insights hidden in the dark.\n", + "\n" + ] + } + ], + "source": [ + "input_text = \"Write me a poem about Apache Spark.\"\n", + "inputs = tokenizer(input_text, return_tensors=\"pt\").to(\"cuda\")\n", + "\n", + "outputs = model.generate(**inputs, max_new_tokens=100, temperature=0.1, do_sample=True)\n", + "print(tokenizer.decode(outputs[0]))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "\n", + "# Unload the model from GPU memory.\n", + "del model\n", + "torch.cuda.empty_cache()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## PySpark" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from pyspark.sql.types import *\n", + "from pyspark import SparkConf\n", + "from pyspark.sql import SparkSession\n", + "from pyspark.sql.functions import *\n", + "from pyspark.ml.functions import predict_batch_udf" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import datasets\n", + "from datasets import load_dataset\n", + "datasets.disable_progress_bars()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Create Spark Session\n", + "\n", + "For local standalone clusters, we'll connect to the cluster and create the Spark Session. \n", + "For CSP environments, Spark will either be preconfigured (Databricks) or we'll need to create the Spark Session (Dataproc)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25/02/10 09:44:33 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n", + "25/02/10 09:44:33 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", + "25/02/10 09:44:33 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" + ] + } + ], + "source": [ + "conf = SparkConf()\n", + "\n", + "if 'spark' not in globals():\n", + " if on_standalone:\n", + " import socket\n", + " conda_env = os.environ.get(\"CONDA_PREFIX\")\n", + " hostname = socket.gethostname()\n", + " conf.setMaster(f\"spark://{hostname}:7077\")\n", + " conf.set(\"spark.pyspark.python\", f\"{conda_env}/bin/python\")\n", + " conf.set(\"spark.pyspark.driver.python\", f\"{conda_env}/bin/python\")\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_env}/lib:{conda_env}/lib/python3.11/site-packages/nvidia_pytriton.libs:$LD_LIBRARY_PATH\")\n", + " elif on_dataproc:\n", + " # Point PyTriton to correct libpython3.11.so:\n", + " conda_lib_path=\"/opt/conda/miniconda3/lib\"\n", + " conf.set(\"spark.executorEnv.LD_LIBRARY_PATH\", f\"{conda_lib_path}:$LD_LIBRARY_PATH\")\n", + " conf.set(\"spark.executor.instances\", \"4\") # dataproc defaults to 2\n", + " conf.set(\"spark.executorEnv.HF_HOME\", hf_home)\n", + "\n", + " conf.set(\"spark.executor.cores\", \"8\")\n", + " conf.set(\"spark.task.maxFailures\", \"1\")\n", + " conf.set(\"spark.task.resource.gpu.amount\", \"0.125\")\n", + " conf.set(\"spark.executor.resource.gpu.amount\", \"1\")\n", + " conf.set(\"spark.sql.execution.arrow.pyspark.enabled\", \"true\")\n", + " conf.set(\"spark.python.worker.reuse\", \"true\")\n", + "\n", + "spark = SparkSession.builder.appName(\"spark-dl-examples\").config(conf=conf).getOrCreate()\n", + "sc = spark.sparkContext" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Load DataFrame\n", + "\n", + "Load the code comprehension dataset from Huggingface and store in a Spark Dataframe." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = load_dataset(\"imbue/code-comprehension\", split=\"train[:1%]\")\n", + "dataset = dataset.to_pandas()[\"question\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.createDataFrame(dataset, schema=StringType()).withColumnRenamed(\"value\", \"prompt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------+\n", + "| prompt|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "|If we execute the code below, what will `result` be equal to?\\n\\n```python\\nN = 'quz'\\nN += 'bar'...|\n", + "|```python\\nresult = 9 - 9 - 1 - 7 - 9 - 1 + 9 - 2 + 6 - 4 - 8 - 1\\n```\\n\\nOut of these options, w...|\n", + "|```python\\nx = 'bas'\\nD = 'bar'.swapcase()\\nx = len(x)\\nx = str(x)\\nnu = 'bar'.isnumeric()\\nx += ...|\n", + "|If we execute the code below, what will `result` be equal to?\\n\\n```python\\n\\nl = 'likewise'\\nmat...|\n", + "|```python\\nresult = 'mazda' + 'isolated' + 'mistakes' + 'grew' + 'raid' + 'junk' + 'jamaica' + 'c...|\n", + "+----------------------------------------------------------------------------------------------------+\n", + "only showing top 5 rows\n", + "\n" + ] + } + ], + "source": [ + "df.show(5, truncate=100)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "If we execute the code below, what will `result` be equal to?\n", + "\n", + "```python\n", + "N = 'quz'\n", + "N += 'bar'\n", + "N = N.swapcase()\n", + "N = len(N)\n", + "mu = 'bar'.strip()\n", + "N = str(N)\n", + "Q = N.isalpha()\n", + "if N == 'bawr':\n", + " N = 'BAWR'.lower()\n", + "N = N + N\n", + "N = '-'.join([N, N, N, 'foo'])\n", + "if mu == N:\n", + " N = 'bar'.upper()\n", + "gamma = 'BAZ'.lower()\n", + "\n", + "result = N\n", + "```\n" + ] + } + ], + "source": [ + "print(df.take(1)[0].prompt)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "data_path = \"spark-dl-datasets/code_comprehension\"\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " data_path = \"dbfs:/FileStore/\" + data_path\n", + "\n", + "df.write.mode(\"overwrite\").json(data_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Triton Inference Server\n", + "In this section, we demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. \n", + "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server. \n", + "\n", + "The process looks like this:\n", + "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n", + "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n", + "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n", + "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n", + "\n", + "\"drawing\"" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Import the helper class from pytriton_utils.py:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "sc.addPyFile(\"pytriton_utils.py\")\n", + "\n", + "from pytriton_utils import TritonServerManager" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define the Triton Server function:" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [], + "source": [ + "def triton_server(ports):\n", + " import time\n", + " import signal\n", + " import numpy as np\n", + " import torch\n", + " from transformers import AutoTokenizer, AutoModelForCausalLM\n", + " from pytriton.decorators import batch\n", + " from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n", + " from pytriton.triton import Triton, TritonConfig\n", + " from pyspark import TaskContext\n", + "\n", + " print(f\"SERVER: Initializing model on worker {TaskContext.get().partitionId()}.\")\n", + " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + " tokenizer = AutoTokenizer.from_pretrained(\"google/gemma-7b-it\")\n", + " model = AutoModelForCausalLM.from_pretrained(\"google/gemma-7b-it\", device_map=\"auto\", torch_dtype=torch.bfloat16)\n", + " print(f\"SERVER: Using {device} device.\")\n", + "\n", + " @batch\n", + " def _infer_fn(**inputs):\n", + " prompts = np.squeeze(inputs[\"prompts\"]).tolist()\n", + " print(f\"SERVER: Received batch of size {len(prompts)}\")\n", + " decoded_prompts = [p.decode(\"utf-8\") for p in prompts]\n", + " tokenized_inputs = tokenizer(decoded_prompts, padding=True, return_tensors=\"pt\").to(device)\n", + " outputs = model.generate(**tokenized_inputs, max_new_tokens=256, temperature=0.1, do_sample=True)\n", + " # Decode only the model output (excluding the input prompt) and remove special tokens.\n", + " responses = np.array(tokenizer.batch_decode(outputs[:, tokenized_inputs.input_ids.shape[1]:], skip_special_tokens = True))\n", + " return {\n", + " \"responses\": responses.reshape(-1, 1),\n", + " }\n", + "\n", + " workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n", + " triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n", + " with Triton(config=triton_conf, workspace=workspace_path) as triton:\n", + " triton.bind(\n", + " model_name=\"gemma-7b\",\n", + " infer_func=_infer_fn,\n", + " inputs=[\n", + " Tensor(name=\"prompts\", dtype=object, shape=(-1,)),\n", + " ],\n", + " outputs=[\n", + " Tensor(name=\"responses\", dtype=object, shape=(-1,)),\n", + " ],\n", + " config=ModelConfig(\n", + " max_batch_size=16,\n", + " batcher=DynamicBatcher(max_queue_delay_microseconds=5000), # 5ms\n", + " ),\n", + " strict=True,\n", + " )\n", + "\n", + " def _stop_triton(signum, frame):\n", + " # The server manager sends SIGTERM to stop the server; this function ensures graceful cleanup.\n", + " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", + " triton.stop()\n", + "\n", + " signal.signal(signal.SIGTERM, _stop_triton)\n", + "\n", + " print(\"SERVER: Serving inference\")\n", + " triton.serve()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Start Triton servers" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Specify the number of nodes in the cluster.** \n", + "Following the README, the example standalone cluster uses 1 node. The example Databricks/Dataproc cluster scripts use 4 nodes by default. " + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "# Change based on cluster setup\n", + "num_nodes = 1 if on_standalone else 4" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `TritonClusterManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n", + "- Find available ports for HTTP/gRPC/metrics\n", + "- Deploy a server on each node via stage-level scheduling\n", + "- Gracefully shutdown servers across nodes" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "model_name = \"gemma-7b\"\n", + "server_manager = TritonServerManager(num_nodes=num_nodes, model_name=model_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-10 09:06:38,803 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-10 09:06:38,805 - INFO - Starting 1 servers.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/plain": [ + "{'cb4ae00-lcedt': (252119, [7000, 7001, 7002])}" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n", + "server_manager.start_servers(triton_server, wait_retries=24) # allow up to 2 minutes for model loading" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Define client function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Get the hostname -> url mapping from the server manager:" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [], + "source": [ + "host_to_grpc_url = server_manager.host_to_grpc_url # or server_manager.host_to_http_url" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define the Triton inference function, which returns a predict function for batch inference through the server:" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [], + "source": [ + "def triton_fn(model_name, host_to_url):\n", + " import socket\n", + " import numpy as np\n", + " from pytriton.client import ModelClient\n", + "\n", + " url = host_to_url[socket.gethostname()]\n", + " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", + "\n", + " def infer_batch(inputs):\n", + " with ModelClient(url, model_name, inference_timeout_s=500) as client:\n", + " flattened = np.squeeze(inputs).tolist()\n", + " # Encode batch\n", + " encoded_batch = [[text.encode(\"utf-8\")] for text in flattened]\n", + " encoded_batch_np = np.array(encoded_batch, dtype=np.bytes_)\n", + " # Run inference\n", + " result_data = client.infer_batch(encoded_batch_np)\n", + " result_data = np.squeeze(result_data[\"responses\"], -1)\n", + " return result_data\n", + " \n", + " return infer_batch" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [], + "source": [ + "generate = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_grpc_url),\n", + " return_type=StringType(),\n", + " input_tensor_shapes=[[1]],\n", + " batch_size=4)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Load and preprocess DataFrame\n", + "\n", + "We'll parallelize over a small set of questions for demonstration." + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.json(data_path).limit(32).repartition(8)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Run Inference" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 30:====================================> (5 + 3) / 8]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 5.6 ms, sys: 3.51 ms, total: 9.11 ms\n", + "Wall time: 28.1 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "%%time\n", + "# first pass caches model/fn\n", + "preds = df.withColumn(\"response\", generate(col(\"prompt\")))\n", + "results = preds.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 42:=============================> (4 + 4) / 8]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 8.12 ms, sys: 3.13 ms, total: 11.2 ms\n", + "Wall time: 23.1 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "%%time\n", + "preds = df.withColumn(\"response\", generate(\"prompt\"))\n", + "results = preds.collect()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sample output:" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Q: ```python\n", + "result = ['mirrors', 'limousines', 'meaningful', 'cats', UNKNOWN, 'striking', 'wings', 'injured', 'wishlist', 'granny'].index('oracle')\n", + "print(result)\n", + "```\n", + "\n", + "The code above has one or more parts replaced with the word UNKNOWN. Knowing that running the code prints `4` to the console, what should go in place of UNKNOWN? \n", + "\n", + "A: \n", + "\n", + "The answer is `oracle`.\n", + "\n", + "The code is searching for the index of the word `oracle` in the list `result`, and the index is returned as `4`. \n", + "\n" + ] + } + ], + "source": [ + "print(f\"Q: {results[2].prompt} \\n\")\n", + "print(f\"A: {results[2].response} \\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Shut down server on each executor" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-10 09:11:11,880 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-10 09:11:17,105 - INFO - Sucessfully stopped 1 servers. \n" + ] + }, + { + "data": { + "text/plain": [ + "[True]" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "server_manager.stop_servers()" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ + "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n", + " spark.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "spark-dl-torch", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/images/spark-pytriton.png b/examples/ML+DL-Examples/Spark-DL/dl_inference/images/spark-pytriton.png index fed2a16783e7cc52a0b8a02e26d936c4d8e81b8e..841862a4dcb1a80fb2ece9d59f383ad406fadb4c 100644 GIT binary patch literal 42644 zcmeFZWmuG5_dX0G2#Asb64G4*DAFM*-3^13h;$Ae1|8DUNC=34beDp3O2aTn!_W;w z{4dno`+lD1KHiV-r{5gMz%cvT*WP>Wz1BL{xz^*yya{}B#7TsUf^w_GN>Wl)PEwLu)ye*um5l`o%9D`zL<~)hN#ekP#xBd-wob*Wu^RXY1b+;#pI#qHzxMiZ$9&&xDy1?DWmx3gsmR1 z+kpsz76U%NdCP=H8-9M1v!K9A_;qwc3*Ih1jqv>p{NqG^EWa>PriiW|^iMq8zz+hC z@yvtO`3tWG26me{c%^v~C88s97ZWg_m=5kLY7Uw zO>YIL9?;3_lSgg;5ZETHHX-8;!< zbEv`rVta$A*t$2!pS}@}Zg~*f{NgMeq@Db{QupoKcXF1r)!{0Z*tO5XR<((>t56i- zlaqA2NBzAl=$wU)W)N@a4gHW>6K3%xPtD9 zqdEEC)bzhOh#J^(x1bk?AC)i1)EE!*i}*ub)Ea9191wTs&Dy7Q%qY@Ng(lEh#fkV) z)>}**?p6e7WMUhl=(M=l)!uasuPb$J>VgYNGDp^%X@FJ7qLzy zX;dHH#bywfcmi6INXewlNBiji@xAA&@NK)5ZEJJ{|ZbX6W^hB)HxyY1qossZ`; zG==5E^@CWY;@`he?aRl}1g|r`z&s>4#EOvC7;>p#nZnw9g(K@cw6^NHTC{r0<$fIz z>{U|;rA$^hK$81)fnTig~I9s$lHC|dggd{wcWwq zw;{m;UlV_j?^y2e?WpX=qTPBNHTMLQeh0G*EnBzzc8D`FsnSI79{){EY}v% z=4AR{S9lk7wEID&sRZ?!sR@SbXQiw&=9jy%^bL z6ze5Nqk6!?ek-UfQe93!u2w1OZC|CT^r&N!qf zE=%*1i>8(8+CJzIy%+r)-GKNZxTbXO)t-a}pGA|?d#A8H|2<2hy3joNvTmICNAW=) zpKFN0fqdBThIQEz3PoCj+x2Tq>2K86ZPJ+cZ}T1BIIyal$I z_lwi=U|AQcLHS9<9?V#3|HT{8Jz{lNj7Jr{d@cOmm8Pi>^2^S-})ta9V zDtpT}K6{&4*tA!Etf;e4syZrz*u(5DroOcW+LyzZBRNLhLN=dG+fA3a9l4p-Kda~V zGD7Mi8;`0YCGAv_WP0hJ5$?y=zp06YG)(@uqHw${QS>zU~>uLx)!?zfHs%b)?S)&6(DT$PwQW2@=X4ka)s5S-beyAL7rI$$^Jfb_04-^fnKsJnk98J#vb>hNS28 zZ9MdRIec+sbvzX`2R`q-bR8!y9KJfpxkFzYwv`2+1>;6jMl(c9sx7NoXBSHq%3xldfa|f9+CU|nfDqUQXHp^aGyz04=P@?Z^^=3UZjwny3yJ+!~FmYH&M7VA6#TTr_ z5eLTk2}iGmZ?)`Oc3ar#^AgjJ|WxUGs=s`Ib;*Zn8A_Fsdgt@gQ2;aruKma8x*lb#qP&^W%9z{ysF8xe*R** zbo$LS2DD}?pDh~P#X>q+Yi?=H(pd5N09G7d&lx`%r?ttrm9Rm{dJMLQXV+UI)4@a` zL^tBM;yJuVkL)IqCRFzDhzmPpO5sRh=tt2j=969JN~o)Kgs;*h>PEn)8!9+56dM;I zyBZfXqmt4=O=GvLgVrV1J=Zr82=WoKQ1Tx#atpfO%RV52L~yG4QXc74Ggscdk19(q zn|?)s+@Dix&W3f|>qw45+=qL;^vwE9^uh=xIGZr>XoM0r>o=C@g-guu4be`~?ybB3j=xW_>h1RU%}#y=pw;w;YIwC{Cf|1J9Vk z>v-_E2F31)y009SO-n-Eq3JNCTI3mItg#D8PWhDbu4s-hyI0Op{xV-STt9FlP#M?j zwy_Vz*~uj)bg_M#wAqie&riZv{YZ1)>f%A6W!|@jZ^dH9XWj>IR@)bQUWzSVO(45> zMt8+vmkolHyrO6N=br0*W7Vwo*6_&&L^@`&`+?};#bVHxC{`+jD7WaLZ{+#83l9=| zOW{-}yW~Fnyh-Gpd|EPrV_RyHHp-jZ{Gycpe3d9~vr#i482&XbRe9;$&yW0CZUH>&!F4a7e3aIl#>H~YnVA%SlBr~w|9Aq{yrGE z)wq?Wu8XdcqM(_*EvJdOy{QGKyRE}@7ZhQ4LExjUg^LNbyRD6#v!J^O?awO&fzQ`p zgK4RMUgBacLaVE!N-b&cWI_FilZ%s!R`eD%HMOvl`7=RvDVg8hfxkp(pS!p?2!g>- zD3lZWfYaW|63i_iAOPm#0rT*109SB0d)T>{xO3P!)BU>1U-yx+a5i(Ya&WP-x1+wk zuZgL>tBVLN?e#!^UccsP;coT!NOsP@mjx^keBA=(=HvqZxi`>N`1)HxRV#N38$Bs2 zTR=R(7^1v94~2hT|358%kNA(Cx_|fN=HcT1XV-r;{dZR_XA37uds|>k7tz1B=6C0R zHvaA?48C6aKSc3M&Og5e1TA_?82o3`L~mteeJeyk5l4}edaUV=x&g!T)*L--*^T|Y zZDM>^FPDlOF3v&2yB!OToddjE}E z^c#ir{-5NYTy8h;$0l_t9wjvh?blx@KCrLfp56AD`?1)F%-Z`zdlwB)90m2if7~Vt zepCT_Nd7+!*B2Z3qmht+K$z72DF6Axe+Wch6HX~g7WH2Z*MEw?472{9?@{Z<~Id`2W+U z-=^pPwCR^&`v24D@iVsa`kuRogod&n$US+&j3)f7_nicCt#o&H_sx+HqNw_Os}PAy zu;@uZpMjq4Ej86+%;*fP8C2LFfQ{emvegLIh+?y!?CU&z4?n-~d5<_j=FT+}o*%z& z$*Q=zs9%T_Y5?;m;BiS&v#@k+!=uOyP7KM7k zSG*YgIM{SKcDOq9)xOb_`;=H@)|lL-nV3WWWQ)?lSR~JWsQCoCIki+jWv?E2NyNqN zi%x8qflhqJF_@usc}~Q?du$nie&xgb`5li%=qJP4vZv^9Jgz6ystxx@NObZ^i11a2 zK91Z`+`F292yIL(i?qoI3Qak|C0N&VsaA595^NV7B zWZ+^*<5Yd?O?K%ey>@@MVX5D4DIW#(A}IJWhKYe87xW_g)*o*b;6UiWP^_V$5wV_Z zMyd8tKH?kgzc^Zc8Ni{M1-=aYH^&(#-NNnc5TCnC@#?n|$|Za!?BTSA^VI(Dfr1IR zcp)ym6o%jPR1yykT;voL!}GC&`u`p-PVwotaMGBB1epe19?m0)pQ=iog+>uOM<(iB zL(jydAR{9qbq6I)`VW9$PHG9pr!&7iKaELB(kr_$f&JU!)5`b@)X+Z5Ut|7Hmv}ux zymY{ye{nWkBl?#%ZlLWS{b(H>9SNwYaMAy9sW19N$Vimc8?;0#IvKBj!x#-{A3Zs= z<|S-wKABru>X>5l^Zx4p1B^#RM0A;!6J)|R*ysiA>h8WZeL2MaOHQfOPhT{wzN=;y zG{LjF@w*3SW_617U`$L5!In|3-`{w`0?$Dt{atdheqc7+HXfzgCtTT6x<#hWsqd!~yv#Ucg00;9TFgG9n=l4GwndAMl z2@KWL)N)@43;o86;%jKYp`tr`PyIAY+{LA0HR$DU8MP8#AIK4dyZd{4FZm=!{*qlx zax&|=L>Z+a0?r?{Srh)86oVFSEY5E?P|@TW9o9!!JgzQ|8{L`~V-#|<3bi2!VG`w3ewqi* zdL2qW>Xm;8rw|x!G$b{4+n(icUg>wu5M*SO3BxkWsuj=Q9yiP*dHGOx78TObu z80c6RWB6ED>5b0jJK4MA#YSvNNlC*c#?803>4|9Zut3K!XvS)GXr=Xd`Bbmx-BE_L)=h@i2Fhbr%(y+ zA*Z?>f{8lULV5;_aN<)?HwCCowR}-L^i<_|e@qMn6ZS9xr_VaBa9@_I= z_9A?o3j?ie*yi{|@L4A^JhH zSX&rlB!w+bCCY9V^pj%4y>+&?`$?RDo$I6veYV16B?3}!^gNV+YKrW2(jCz^bv<|OMrBh>~yC6YlDjC(c`7;)c*c{sBBO1HJES?;dEm%G5&|RH-Dj9FiWHR*;~If$W=3Ka87sB9FHdJE;6-2GRp`ij13QeH*x$m^vCY zBFK0*IGNjD646m^?Ns#~a4d+@`E}OunFbY&RY+ zZq;umuxfprh!XMKTrA%KPDbY|Tv2A=OgPyKY&YyL9*!GVUpb0CZ9zFc`x=BB#(2p5 zx3#}S4<-pAxiBXEBuyQTmg^r zJ&a^}T7EBh{w>R&$D|=J&k1hqdLY*qb0_0>t?15?_p~yAMds^w&9Dl!S|Bs z>G1O2qz4_1u*#Bcx=>uP^Uvd=iN?Mj>8`Tx>dDx4bg^(M2iBY{9 z1S^A5#B)&(YCeg@s6#8~ch=up>4Q8HMijG_n2BAUm^HHsRwq~*adUG+kXEqt>7owG z#;ST^&c*}vk_&G_L(5dh!11G8g~U1>*Q=4Xnw8p{xRe{S-$>B#cEC?Xh814+h~2RU z%x=KvYxTY-Ki>|*ZlY0DM~xskG}C8spY1%;p#|pEOO&rwm~eiv?RUKH-{d-Y?D)9k zzOT^}W#uE!yE_NpRT8a>YrnyZod!gXgI%HTyrx@UVOUI8*&gjh)5%Iv86B`|4|h2a zT{PzNv83}mN_@&m@iM;1iV|3#rVtV9OW~_tp?2PUm!7_L)~)i#aiUV8&P(IZOJ{_= zlt%mV#2-!P-W^7x$8(`<#drw($Kc>nvySd!qxy!iQ_d4*R=?#W%d@R{NU=sM^-=Ssp1L0RM;sK#X;c)~ME+|~RJz3u2T-pXd z#NpWl19Apouae*O2s3?3RdtOdqp7cz@E+EQU1d#h&Zq@<)o z2Hcf2cqV|Q|E{4T!f*`tYp!3xCT4f=J3ekdwqu10)pRP0Le7AFz9#KNt{Rs14ozsZjXLnNXI246)`3zFisWJ4P}9CkY z*;r1oPxux=j@;_z`P+q)Ub}GrZ#0as=)gH9gBboJBxi7Gi>bw|8Zud4X!&krQWIjv+`5E*F-(-FU zblHS(4AAvS5Cd}DKKM*`#9M^s1t{+^wGEo*;W`5p?gI@u(*XQY?cta~#u^lFHDU|8vu9@dY- zfER$I@cwvY*-vpSm^KI*$UU^$Q>gYkT}s^A^afzx706R3*OwXd9dl-L4a*wXUU7rU zR)%^6*uwY3E8dvzXaMJO&q4$gcXeBpLFfU74g)Sq9-oXj)%bTE;d=SQ_0grpuGnJf zk_^Ego-6%n7ROsK2;y+_)fr}%<&fv0-KTgq9nWIUz51s2+$<~wgQ6r5_xJ3EHJ=AP z)6Qpku^MJ`bM3`P>c3RDtPM+#6&b9EE{_yw9KrMeC3G$M4!$qs7FXwrAbw#91UWVn zpC2dl*>6>fVFMe>k!jyHWXv4BxaVflbsH4`O=hXjEtatHzyx6JORXo^io_0$kJa+> zxC{8#aX4e=p0j}%F`rAUwUZdLt7-85I9SWekiWXT2!k2Nz7ULVWUM)lFVM)VTUpwh z*^;;8eKGQ6xj&6fUnMiEy3cRFkI!k+BGF^B%zZUBas{26QsIOPbRt!4*)9#TZ>#lI)wS#j~)L{5LE&edA)X``ds%J#6qm`~WO zz=wTTz}6^Up09|7F}^=1N~?fSSv4#r7&L4>`Yuc6`sH-H3E98hw%_DxMZ2*-AXZIi z<;j%1ZN`#J{auy~fSIg~UdKlwsTP1iqHA$M@m1Be2>1UM7A11@j(aorJ?1-F@rLK! zPllAyB#;!wU^nZHapbt;;ol7ql9e!D~6z16?r z_+MBNz|X5_HLuuQ#!|8VlK0;+kT^9aD6;mI^-PT;4}kTkU7}h4#0Y3#9s?{!h&wmM z%^1;NVK*KcY6pGI*e5-@Fk+5d5+nr#e{M-lfu>yo;lL*0yd6|Li1B-R`A@Gqu+|uB z2K_yuKcDbuK#}qe(!dK-2=)s`W;&q(FkedSE}G)q;K)c~z_0(BEw%f#6W4krtA3UY z#7pvic(T8#0PzrXAYzpAdoLAJ*k-=5O8qZUQ^Lr|$XpsdChtDmXX;FKhCe3n&s?s> zMB$7v-Wg4ot5X7bj}Wl@9sh&S-{DY$Fhs)ZKJA)%dwbtI3@-jP^qerN#)34kK$!@# zr3STJ>0f*FI>}!d*SQ*Xwttg&FxnG`H<-!a ze@hb$Kyv&cQBhC{jpx6)2Ve;Q8~;ak{31JoDFPnfz>re5T(i=}3MbE?zkz{syHIq9 z6itGI%k0~2IRyne$ZcNX70DJZ0M>oI8NOTxm=f4si?exv7kKzoPFb1pDa&^*Cp=Iz z&F81BAibSKzTsvsMCU43<8AVQH-*(bD2OwLR-#9e^0KUQ;tTD-b`yfNW5n{LThM@e z63T=;IwPcp9OS9jxdK6IeDUN)1OTJh2b+;=dwJwY86H$0KE=%bwpr^D#?xvFcJgoscB=;UX3kOIO{}9m7 z(9o}_{--XfTu0ezQOwIrOX&8m^Zz15=mA7@@`ixqWdW$Rw)UUb13?xcb~EUKGGs6KaVez^!=W8$t8&>*vfD?%R zNrA~Z6dBcP^E`NvYuxM`?~D4}N#Rw(Nz(;eBgezc=K*o;*qfzf{0>itgvyAn<<~+C zP`tMYFz>N_wu1=*I!z6E=!V9Y_oFQ=exY9v@H~dUP1v`{`)p?^M0fO4)o0U>k~ESf zgw8P=PKTYdP_h3aplBt3orvk?yRPz4AtY%$1eeo~8&=_ODuMZn^WWma(BY3{_4#Eu z|Ih;02AuXfwm`e{$mnn8^p6+%q6W16-4BpNf5FrBhFhq0ZR?l>!J(l9|K$bfYhFT? zRDbVbhqaqA9ifj9(`{!fN&&|OXJ0}+AYhlqQd|1vb*4uM0j!CfZBcXRoC z^X@FAzjjjm9p;X^&b!;@uiYdU+A%2IsU^Hrs()L_2p~?#0t_V(&tmsIxyAuM7pN;v zwavr(JY>!!8_|5rWpj!M`?s0KEb}k;j=B?U`WAfqJ?PduGH}e?E&d#vUvi^9M?0t-_-9HTgauq82&6c?PaFVEPXOv0#lL-$ zrDg^dY8S--!Fo3UPnN1LiP*HC0E9=5Sx01_ z^=`IS(^W+P7R?*1Dy!(x>RRqgvDlq&cSCv|ZLo0YS2)F zqJdaKd)5=d;aBfb^ga9-V1a>9)!<|`J4))+VHmhc7l%?<_j%&#>T~3|eZy{Gh4py9 zH4?eqY>#oupyXiXR1Dpog}AzNraFEj-JA8=?rI6R0W0hoE?!CZQik|lxK5Uu1=XjM z@xJ&;mefW3#ol7RV-#iY12M7n(lI@w=awPH%350n-ct5=HlYJ4t z=Rv*?MjYcCfM{j)WYQb`i0)_Je7>hZ%3~gkJc;r3I2e*$I&QuKNc=`?_30A1T7c0p z1cCvkDPJ8HrTTQ=Gb@k1E~ce^q3t+;J&@t7nFC$Ng}~-?15g+YL|u{Llp<4FykdWW zzJSkwac38dWPseRS4hX9nY$P8(vL$ZQ?8+E2KALVVd&k#wkXRz%c%;hqsx^HzowCo zT0Mw~1P+6`L_mASvwbqHWes%rjK%hl?>}>0@x&2}sAhUCUaC~pC z2VC#wcs1e|zPb?Q*gewG!*vc7qse^3*17w+Z*kd;nb zwJXGRv3**JjME2_e*l}}-D~RQ;ci`SAEs2yH43DYjf{gddP>KL__)4piD6KfiMrC# zE_Wd~0U&3+ir-udsuv~L$6%y#9Ku!%?_I+&_TGIXk{9nQ=6&*6_Ip zbEpzmi{cWhU#h@C#g8EIXh>jtUk1<wsf7m2*ZMz6b}Y_|4gvIlZi2z5r(nO+ zF!*<4fSL9k*E#6Ej-f>!X;#}A0fsALRVp>P#VJ0+_e>5TnHdueokAMrw}pGUx}w>N zYra&5UF`EUpGLsZ5}P7QWuqu#($YA9Jtp$mJp63(b%T$}v!8tV<0tyf;i?xed*Lb6 zMu$6usA#D)sx!hq{m&VS#9NyE8Ecp^H4{*FX?IO6jJ#4w0#vcz3HrnV@_bLp)gJ_S zctu6U_d1}f{1)ZoEN@D%%3iIiX+SZ#125J zYS%R8LdIrZI?pJn0s|Gz5EPkgK0C_nON$o*62z}p#w*DX@8WPBI;|F7>2iX(R{CA} zf*+;n6w*t-y4#4f{_+}&2Vk-!g5oNSfGkSfA!0ntUdWKB>EdYmh{8~iRP--w0Hh)g zKnxK*!6XXyJ@`Nls0qlE?RUJTlzQ6&92tw&m*^0zVo{&73As`rEb)>%;qLKg zz0}?VEY$6HnR@~`gY%8YvuvJhg`%QaQHq|#X&VqAVCS2;!#6Wm8v>ALc%}g(Bbttl zX98_`%Pv175E?vc$Zb@fJaOvY-}=LiRqVL%EkyK|^kbBO77*NfKO(JnSoW?_u5=jj zUWeGF+BKbcZAV1{8v4l}Sq;;VbGBdFjDCz4!AMdjz8aLoC8}?3z7OP`%)8@Q+(34N znUAw#C#`w5b~~sNw_x=>=^|bm9!&>^zddj_Z6;WQ2Eraj(IFKBh@C#-kq>}3RQ?e! z@7a(8(5^*x4zRt6fEH*T9JvhajcBNZh@MsZAl+&KkFz*YYIf~~ChGwg+TlFz54Uf= zaPy+%@%frHI|fPEn}%PG`4TPyzGFiwiLox-@6w|{EjzF`RX}5_W+AdnvxobqP%tMz zg*qiwaIV`JK-ftPNXX_ZfK)K;13vzMiFgKy?}E-`zGQ{Yj^530IGVD~%9!tV9XGNR zYa)5x+}qnLp3*RepnHAxO=7@lz71=#!J~YY?qxdExGzm;e75M?R;#4VG}cs`07*M9 zZ4L(8#FfWp-@@@PHemHS01?XM*`W=bNfGD|p}(;9(>Opdtq%BLV%?oPfa);J`H(+h zs8)hg$o||X%QV-7G_JiF$*Y-Fay;JFC01bgw1-V5UjwtEaI|HtW*cfRp3`;}bq}tj z0*e|vt$z24?K7@HDn}-i3pv}qu|fP@5KRIs-f+*;xj$IE<|iUr$0?rQ;>4;B3%sOO z#Rr|8z%#DSErs6aMiH;4=_@31#sgW_)A=xP4nQRmIg+gdTr?LxV6gF&!ebY2#iB-h zwp;c4X)XhXvGBkN{b7yx!7c!UU-|CQP($|CTAV0` zErrc>Ut9{ec{gp`om*6N_QZD(6^&H^V9d%E1;C(Zf1y6W+?LCvj{T};c$)c(f-t4# zQ!0Lns^p;d7mZQ_`k+kJP8ZtR$eI5a+Wg)2bBffRK(hYjZN3|1HIIVbcErs@Evg=gdV;MJ2JVt)TH<9`AvHUC)7d(kP-$*+fROKb zjXPtxle6>JvH_QCN`^V)I_{H_mIl0~yqudQ6)-Zj4G_40b$h;c4}=+~0R*lb1qB65 zpI9n_V&`rZ@mXSL`S-_WMp3QX&s}47*AL=yMV~&gPu4?A#+%srb*LXa_|6?6mMM&-Q|U^89SS12(r4aPAe2_}iSFGvV9d`o^LO^!@X-dIm5cg{EL?ZYF1dCgemmK6Z*(4`nrc< z^Wy?XsRV}>14o-%bq3W|2-jW~M0A77heoE7ON*D`83!*W7;0p~@6nt&!Wvz7<{l$= zQyLnC!5Dtnz!9m6I;@Ux?I#=*+P^$3@e?R9smRw2fMG;-#%^=QSal(Cpn2NuTLU<) zgoDlGN_X^M=1KH@zy#{I`r|$%)TRMJ{Rg(amc4Ib<3>=U=)>|M0gyqJ4GqMl-y!ZJ z!*vSi)_Om~X|;`>}JZW=MfgXdq~KgM>D_f#FM$K9~iNfyv4kH}-=myR1zU zh+XnuMOFcM-A(}4pSu+U2JPCZoWj1zuVAt3er_p6RoD-7%~9GvYtO2*G0dCr`QjKf z4LLDv(yJ7qsF%qA)IZ?!wG*WPuY;w4SQw9ZfdVTHpbDUIAtZhGh4Ly=(R46n=i>_= z++xt*y4>W_tMpESg~D&wE2q03S(FfK6=|2z)GZZbn6KuOZ<>)me7>mFkx&r<>G{#b z<+S)@XyEb!3F|6;gH1988>=T)?2`#X)+Eg~BRDtM2|I3cHIhHJ7W3PBDWlL1IQeJS zsT5Mx`SwA6XJUJRJ+jAzGt5gH<0mMvsg`ZXSy@~7HG?W&jB!>p$;ZE>$=9!}8!6H# z85H$`ZpUSJ_?C`WKOfc4DwOKkYlq8gAIYt`UZNCe)E8?O=xco(&o|lD7(FYNUYR-b zfK##+B$$B_SDUa(jp0Z3Zv!Ke<-Z0-zz{~p%1g_gfv$xX3sbzqbxJ~OgSmxmC-l9B z$gPR0_7`8Vymnoz;pxH~6hwr~@@T94*W@UwFB zrPP=)C4>Upt|;gG+N_+Dh@uI{N{?&&7t zg+oz(R%*->mFlwXy*eRx3g0U_aw_T1lWUwp_rtAooBAvwNnAWdGkttTdj~Q0c55@a zb8<69p**nCu-^S*q?WAWN?$O2R#l(3n9VrXE}Dq^SU#hlW9j9@n0lcMQHdF?D-G~c zuUBry=K|_PxGTO&HQdv;@7ERt_IJL0*od2`x*Ev9{_xY%OqDBmF}x@VP}d@Q?gcpE z{%IM$^Qd|!&EU;+QRKSNUcWWC^8?lRWKZRb@dS^8ltrqNCTgk%POS9lLn<)cjlXQbw#X{s7-#+wmrN;9}F(YN{c((b(d`7zi1qS3JH) z%~pLHPiYbfdMR#!_1x)s1+U!25aFa7;S0?|T6p7`(AtckSZxJNUexG(2gB(^_tLyO zVtqV@Va#n{@1hRoG>x6nr5z=eWQ{b^IeeREPesjJNTkd~CFUW|c#bge*o@WeI!r25 zo_1aGxWCkgaZ^DNwqCq$?zwjpd2c*4{K`xGh2~)~@@(aq!t#&voa61FR5mc&Cz}1l zBxUAuT%h>tMh1D+{kMhbsTO|5XJ>uGMM{s$MD`P7owBtC#>X6+ay>^`_fGEkEWO9y zRUef<@Jc{lit1?RZCC4vI8URTx^3blm7AA)CBh-YN{nLewO#@-&EcdE7-Q;H;CDy{ zPal0B9u6kC2&dc4qKZ>5-Xj4kr#vc(tNbhkBKlS5;;5ss_JW5Oa&So^v36$o-pjg$ z$7(p|R?!0rHm{}H(yhgWqY@oB_NGux_xJHv7`RWq+>u;Dcw~^s+gUZ1Tl!Fc1#i)L zxou&(Cp+T8>9(78%XFL+pvrm;s*LoWOp_pOf~-4&I^*Q_4KzVG_sgg(ZVy=rD80ir zrx4&2+v;TA%Z4h}%6DkrDdNMzapLIi+R$*?W^>+r9lM;mbv~^F%wX$a$|q62kkJ)} z9Jur?H$0kn*xbbdTZ@=vTZ{=3i%Zowg=*H~j4UZor7@PAGv%r}DI5&>NiG7(E#LFsQJ)9SjWMthd}NHQv8AhPCHv#`ApAiEET_hi(- zj&sCvI7_z=>UJaRH)I(yuJ-B54(dxbVZ;o63>df;Xx<}3)y1luQIsNrad``dChR+{ z=dM{9S>5O`<-`)k2$yZAeT&jPE%`d{J=;sO^04vEIF^i}vTCrdo&(1BqE`b6g-ACS zDrj%r4a87amlNxFw~St{QP@3?qe#@@FUhNyFtAQf(3OED(<^P8DYmQhYGu1x;W3|$ zlyW~jqit5nGncmpC98HYxdgha^A{1l#!cyqz%D7ssI^c$QB0f~?`QJ20w%~MC)OR@aQt?Ca~I{m_hDUFSm+xxi&$LUVfuCF$i>`8iK zfp|w4~lsoCw7wa)^gVV$j!hv(Gz&oMUd zgMQ#FgR!teG4-f%#@$h;;2DFbvi`oA$bKvI$n!!>siyCqSEbi!C}fl2=%^Y%c4ouu zAF4jV_R75s^qNz@Hx;Z}PDa(wU~#4)5CsNE53SRoafB_#c(<&~QS>e!3H@ig+J^eu zce2neG1S7^GtK$MdbVVaveGtOnjK!#;abx+Qb2QK;IK)^ zN~*bhW||kjs=Y2-F6>g+`yj__x=4GtvwYQK`O+dT=L*Rw=Xa&-@uUBxEVIZl_P1lw zvHOtXVfqgV!h&bQKwXW-3PFlP&zmjRv;3tmyh5;^bmNiD_dn46>_?wDd8kkh2@ol! z>9iKou}kHMR~*ihU8sfvQ9t`sP|;6D%Jg(GgrsLhCplXwf~4_kR5O_p!^9YZtuOr1 zem?(%E_BxHw5)yW{#2FN_Y2s*LW-MZ_s6B44Xxs?O_6HEF<>qlw6%)V`+Un63R2Jc z_&t?{w=3zeF<*zY+}BB^127!3hW$-ltdGc&O$P8l_m%VVNAnq_iw!>7x&$LJzU^)o zpLhy{T}f-mgPESQcIa%8kr-9!w#(G&F0mABJAC=@AV_DZwzXS-Pnl@OLwiclw`?|f zl@&JikjPK2Al>p&Il-FRmh6KYFcUYK^0ToN(!$!03wcrEFJ7=VtL|}|3{jv)Fa@)O zQi12&=OZR=1at$Swqn_f0rDwap=4wQNEf!v=c6Tt_`kPvPMboHRn%##CFir6FJ@#C zx+=y$4mECG)ZKTfmG4WgG&FAX)h=>X3NN>z(alBe?eOHO>?j=%c z9AOrCh!}HdE^v%e@pGZ#=qsx2ye_j8rw2-KWcF6SA3I!c+{rQPt|`Cqn#Fd?fS5`* zM9m{AmDgI^qcAvlp->x)6njF%tYaL#tk%CZWU%bL7u~qrap%HG$c z(H>La5iF=f!ADwGCUX&lrn(Rg9*fv5q6dv?n6lUHB}|uQIi>yV-ekJiCqSBLU5IR%96oAjmL?~PPq=e{t` z+VuvU^!m<|F}=4H;tBuCkij~Y8Q{RdEZjNkc1$=>P4Io#iE zoq$cm^U(Ept!Y}OtIZL&=M(M+`Qe4*u2|eiM|<+!`srK)8{A`_Yl(=@<@0hkTs|wR zQCS9~N1d+^ZF`sMFSAc>6t5mj_Q$7y5nGT;lYP;oodJ7T&)HJ9=zUm^XGi?X>y)}5 zm*XpP9Vs6=t+FAB0Oa8mqY^aP_NzNQk{GDQX+pNM>NOWV-C|2CM8K~M=rD>xR!K^x ztr`n<-=@tBzak$$_O;n5jN@=iE57m?_C%VgweNET21R+0uR79UxH?i5hctRY(%70Q z*lO(Lul<<@=y64L zbSMSQv}&lDXo0n%Y}#d7S|nl*3N&#A4`2Sx_Ib!B}-Xz;i*@~tmouZ zw)7R6cXz#_Non`8Ek^>O7a(qhUS%^$3)Dw)Y??&E{XWLC534xl<{j3|eMz*hfKHn$ z5oer?fsScxQ#v|dr+_P%R4-W>jNB|22PzM{J6L%5o6-)%kS<g*!=@ruQBA(@pAcG2lj<#=Dq>V)i$GV@m4 z>BamGVIwfyIb{64d9mAGx%+h$^*q!R7Y#3%AMmNcgW)kg`rj%{H8 zdr@o0x^VBBTbXICblcR|t9Y(v5pFb*;oN%>CD);h?rgb+s1$+g>J>h-^;m+fIqjz% zE3DY9JaLJC+2NJ^(3TTdiz#YbIu@-{mQf`_9uOLD!EIV!p!6 zD_rl@eZK#=5BC0{JoS8U?GVXXI(w-zsyy?2zC-I^s2SdT@@6$VNdLet-%16CNGaHA zsBZRMqr}$bv<}b0I9hb^czq?L2}uh?&WUZ6@m?jIPaK{PuYPYQUkbCZf^-!H$sR?w zHr@jD6x)(@al9IE8_(`gI^C8>M1PZZb3&*W&W3qbINBdyI9Ux;K?)xR#GieqL~xdV zJ@WJsnm6#H*l>+B!ib!>IPIILw`eAbnr3t4K4Zy#U$FFkNmU%WthPv zuO2$Ig*|?mJH>-_fEZWdY(94avWifE?mxGp=+=z>fq{QremTZqbN%?0P|bBU22dIj z6RoyWqiAvG8S{XU>vCdy`qpT{X>1EgfwxHs)Tm=Wfa5L1w%m?StJwOZ zQCHG`KqyW73Ui}2YYDWckEs%n@OR*CQKvVJD#vb08$zvHNTlKERGl^3g2Uso&)SC{ zofdLoiz?$8;65@>G5vfD6^aKf@-k^OQH>XsHd9GI()Sgr(>(MgJ?77voM0{eb*oyC z#$(eVskm91<>a|?Jw;nmfy^*rXD`3nwbOxyqhrH+jwY6;&ko&ApZc~cu&s}MWYp6` zhlWcV=ruK8{UBJI;E|b4FYopVN*>E5gdcBANI+^3vebP*O7_C{b{46jP{ncY~oy*qK>T(yNxU+S?>WA zy&$3=Z18Xfo_aEV zKLtDl#R717jcFU}0Jr&mvrgj5a6Myy$!?X+udcKN?nCREtK8kx<*8U2a`@;=RUi42 zv}6Y4#hZTTN?!zOSNUaL2d}+my{<%KwvE=*fO?N-m}wr0JOOwRb-$$HV~Pr<6#J(l&n638=}JP5F77WZy1E87Fa!CTSRlnHRGUh&K&TrQj{R5C>0R3UE7OE2POmAqOm#mnvizwH)p{ehqdL6yIw?|`XhJa%I#bUDp z8wcdl%>5ohD1fTrv>jRq`{dzc;Q39aBOV>#nbBG^*&aWlL4!mfZyp*%17cEMY#u7f zU%uS=id$ydehWyRSF9k0r4oUHZfaFu)bVozAdSW1XEEY09*my#X`>owh|?C&lNls| zsrKUWd+eE)-f?XYixRu65ius&H7zyY@CRy8fz$<159b~{IaC0G`y8hUAI~@|+fr`s z;SVNK+|_Y%#$E>M@WQ}SVnDe}oOpE>i4N5?Q0gWAbZ@cC?IlnG6@Ufr#UD)KrQoR# z6nI`S^8ea<>!_&0w_R8e6jYE9MnM4uq#2cxMx|T2QMv@B8^LDi6r=DbknNg-*sL^7!txV=-IY%Io>1Kwni~MkYrt?CHQYKY z{J_A#U|^YbO2CpY<*VCb`Q&r!@23u{C^sZUCbvCim}Xx{U!cXw1Z4^XeW72M;4=c5 zTVJlB6N$gc&=TWCLy!Q{>#~e%)a7@S6AY4Y4Gay%^yy`9-b((j;myl{0}w1hDf=FhYO%9(eFY!_=>g=yJ7kZ`*LXiMyth4=*T$>a zfT(GEx#c}r)OrvocxuV9EHR*)=N+fTk57rINmPdmKqR3V`@G;v&@Q)n?6a?{E&w2c zjdB9z$C7osd0~f{A5XW>Qwe|Mwuju{wude*1Lw$PD5=2ULLr&M762JV1<5^xYq``ZFK3cz}9J z45#4G;yj>Ldw)Lz)DQ|xS|}hN&djZ?_fHYnk65GJwE)mfzQ0}TP&`~Gc(ITTct3mV z5ElPu5}}{)7_cPhzc@c~xvi_Uf|=>6H`q)}Oc4C``WaCP2?S(bF?UOZ0nJOu-HMNl zgq0B|njC&X5b!iGyp&Oc8~15=c+`*aN!}y)W@Vp{`3=7|$uxdv;#>TSmyPEQVJIoU z82~t}EI7KqE*yNHXy%DdEYXFYSNGbu^!xxa><>HPG7E<6j??|=e$r)tkH##(6W))B zzfYf#CD8h6d~Qmj7niddph%#ldBS_-3_$Z-2xL69UXpNx!H3SbqoMPr0lGS8Zr$iQ zMhgF#0SNnl6MVGj-47_5fyRRQ1~^M-jy?>HBIDr=txR{)1K*p3l=S4ayd#=SBSRI= z;KcISR3q6j*kF9AV7ey&Rqu{!v}-l zdj;PBM$ia=j%@3uuD<7^pFEgD8aiwj7Z(G-%^;zXA2ss^Fd^QL9>6MJI&*OAiq(LW z9jyVX-?k(}8|X2p*eY+KRNTK*OKJ1ELTekUVuP3{vQ&scmB`v~f=;{d=$ZF$zw+ws zEPCy;0JDi6esZsy1Wv*C7$a@P_yUaC35}m?)OQF}@}U@DraOx|p*#K005g6T`GQ1j zAr#pt;M`{46F!`qJD9vo7{CIwW!)qg-aMV>) zRNiKH5FBis7^Va}=C^!VJ{)4^s*JdaNl9XW0Ccc1@ZvCV>+4b7?;Dz$&JN+-2e|L& z{#9SQ_>F$1&Ohv==JOF@#8pyOexc9!LMg^h^@E7Db>;yE`~s*wX~Yoq6XUT$+%4pN ze0EYSbLawk4}Q+f-`aQrOj_im-_e~vy#&r$6F<@n(S&Od0s?@h)x_2iCE#2-hkJ_f zG9!q|9`H>qPu&^{IE0qML`POrK2 z$^fw)0-jY1-hYmgFVm>-c9Qi;a~~Wr1F`l6ti6u&t&;HqE=uIA8Z=P+a6d^LcarW0 zIMoks>$wQX5Z$f^%igi0cfZJpka)F*2K>a z`mJN{gGSLC8-P--q786el~j?yK&3h>TR>zhKhp0t5fxFg3}LcTQa3YqIlqiUo<~QX zX-XMw65W{OX;PJh* zcyw%Ru*n@C?#-n9@wxh|DLGO6bnPN}AjI(+Mr-k7niU)G_MIvoy%-^nl9h$Qetfwk zf;$*sZu1o#;4*~+6zi#nW}h%5vjj#g0!0DDkf%JflVcdcl2ldKov8WP4p|ABbX<2p zp?|@Mi84z>x-i+zw5jHvI^Sw}l{FGvo@K_b#p@EGX~+&bkd{j!Z-(|ED#r^+4E z0bH?;kbnSkx_saX%a5iRczC*JjVdaWm;37fNYIJYfUKo>vm)j2eG;A8^H;B6b^|*R zV{md&EVZy_*&LbO2SZ$Yd0gd5)yJ&N1ZSf}R8+zaISE9d*p2+I$v+_r$ZT|9Vr+3S zc%}pr9PE~&@bGYYZ=Wb>v+F78A`rDJbR<iK5r4Ufum#L4Vd~=?bR^T zj{in*V3x$-3jtVVD}#-XFB&IO|B~9V#7!{3*k9x|yh14GAnOC6JC^{N^W`%p59lfU zh!|%G*?u0ns zJXbgLg9!`lml;&XJ7C-3=ime-kXprUg%MykNTwX^+ECBfs%3x&SS7glq3Q}Aqzr$J;D2RYo4k~d)a0i4M7q>Sce^>9(z#lFWDfoXgN53lE20;_+^>^lu<^o+G3*iZw*u{+88FLo#1 zWmP`0B{iqS7C#$C_ar%xiCw=!C*ptt%3V^)Ss?;g7nu*dZ2$tr9*cAg`S$JGgi@6n zd_cmv8_Y+9$Ew)j{A8jrJlCj!1YlSdMjCqz!XVoj<}iCHE0Jj|nH$JU0T@>Zhg&|I zA;D#G$p>S;Y-RLJM1-@%j42RX^i0=dpsDf3-#^@Kp;!W(@Mj+hgQ)Fnh9O4a2jDVm z5J<{!hw}i<)cE8wHuCI-8*y+gg0J8^fX2@9=n2f{clu_%6(qt-#CSY^go%hsDPvOi ze-~-{C_j0Q+S=qlD|(U~1Y$T6RrjJztdU$3V3^FnzB{B&DQ?0TTYkeUagXRE%N^`mXrR> zY=^|qW*mFI6UxsbZcv^nQ9kwmC&=>c8uw*!0GT}+%6tqN><@4EdjQ2@vEK&7|Vu-u_Zmlm5Ck)`U{i?vCF#h@VMQwiM%R@r}abW?_ zP&e{uaE=ZG%*RA-;{%!=(z5B>bVni_^OpRDMxqtf4u{|zqz_srvUgHdT0*Ekh(mgaY%H7xVk;n4tKu=?AhK`+p)>8+JBN_rCX_dYn>|Ot;uTg<|VoD zuRqI^$9Nx=M{jm6*>BFo4?AZdGD+A}Q`LgC94w{#tuvDMv-L%*qu?iN{y`C-jdgo|SZv7meiZgKMLl>Ld1r`TG%lV4Pmf5(B* zL)r!pC^1?TcwssaI~WW+Gq1z=eDRYB^3Cq&ffFNbugdwncIxqaV+$43*fkg?^~o5L zsEZrRcOTcn6Qwdm=#3YcrDBVut*Q8RC<G6D??|g zcKnC!^8Rqdt2TjgYo(^udJU~3B6sku)jaIomxOkjQWnNhBig4L$xwBx8b2p^^cfL* za8;8XheD448x!_mtnRfODnYc6ap0XC+n)uv%kQrAI4H*L&8OKiPll!I?vUcM+ptGK zMkSjZ(tgue4{W^(SPnIa;M+4R5IOYFlO*{||Lxom<0ZA_iup^qo3An)(-egBlGSaS zy23rx>!eWVP^GLVS9r_sv2@Q|^WwH9qXqTQ>JBxVP(Q)b6C-=yNILU9gX^-1F)L31 z=WPPNKx0^>)+?6jUUc&@$K;ljy3v>05z-e9;5)&g^HTs0twe@|M;RBxaSZ%7( z9(_24N5x|5wMee_1$N7NZt>9$7Lw0u>56=D3~3KN!P|F_7`d)~r_hO~Pw8WLTi3Lp z-Tj*C#fezqFZ)4mGGm=f3_ea78|FxI-tnBlg1z*OwT^`Qf&fQgH}53uE6*4IE|f)r z@Jim~2%pQ`hw}Mms(I||t!2y)kAh9$%sl~P7eGlufsvh3l!7$e@GIz2x}K}tX<*r~ z@Yo)HHQZRT$m5#P`NB=FtXR>{K$e zkRF#0bkLfKUn3BCdg3JXASv;SGdcpuM2P@b>?!8~G@UUCM5|gLh-lKdH*r}lsQpvS zvK{IgP9L(+(MGq(C`o6l-*aK2KaOI0Z*O|s<<{b^*n;seJaV9C80c43^y@ZtS^Q#X zw&&e8Dzq%vHeGwey1s6~l~!E2{3r1R#0`KeQ5TzsF|)T7X!@As0LVJrbN39%ZFzHb z_(uf(muEnWS-aQ}BakEuiWoC?Ob1X4DXXUNyrw&cz% zu-j#gW8%Q4W4J2|Y-g}Pp0QsX(%o8Lw6R!z)?q!0OvqMZg0>r^lNNoEgYOq16eGW` zc#7??x8CL#5xM9g5pY?6d6C|Obvn&Sx~RfbYW{6>?s7``(r(N|NQH@8!Z(?M3bFo= ziOaTEdcv9bxfddw>OANK*#p8)$;~t>Fiqduf3{Ov0|2)&R>K@Jsyqkkcen4+z~OE6 zPmz~g^AmG%f!>UIuiWGD%{QD=Jd zqbT_8nU=ID^oOjC_goyrPn@kW2WhEZu#RH{g&b#FZ|A)j*lx_YwLg5;xGRxh0e`}w zW8>GOXvx{ujPa1!Z7D>dT;t$7yVLd&+xwpHWcQn_dJ9~#N`=Y=Z+_G~Sv+3r+2N&8 z<|cTs#4u+wvpy(NFV#II$E<6WOc3-puENp1TxQ@A_(1d+es7zw32n$A=B2G(Ub27H zuX5a~T|ra2G&4nJ^hfD<_NvBs@JR{!?h?Tq^I37FZ?1N<2RClz4H9H|{31XugeC!; z8X^?jm@f&t3uXSi96wyRlJ!Y9MIoMjdU`tK?OToesxOTNRnx?^myKThLRW3wbbw?v zAPf1M7;x%kH|$@=iN;g`7$+SYo4<3;+h7Jg)>E%rFej}T#5d;q#deTR`YdV0eH=sJ ziDrV$0*V-ZIwxF@a#>hVxO}GP?!i4_ZtSlY#phrku08HKRS<`C|438H!AW@*a#5WdS3R9wZ^$(JaOsqkMNJPKa0MoFi z0TN+ZgxF&e&P%C9l&3rATms5Nw@{e7#Uc9s2R~%C9E}ui$c1|a<-zAz?|FI!hvQ%9 z%{B^c-d^YFFnxS}`oeHr0{w-P)x4N>0lmCBlb$zgPSgZ4-U&Nf+?PaNxhiB)#8SYGm*+{D} z@_B$%H(4F*-#AJ^;XO}+QV`#Av?mri;aFoxyJWqj61B1~QqkIIgCrAzMh|x817E4? zGq$Y3$dBO70$s{To8VIbGG3aA3(&3~0_h7EhqUA;#v9+Ce062$Z1kT*A`z>%oLbq% zF}R*zK!{_eGkQ|r$H8^xfLUKf4AEt7p7W**E?x}cv~E{S-0@S%;NzMXRY#%cs>rhc zm_iV+bLv^C=#)A8Ip0i&WAkCYg9b<>1n2yQe_zS34dJs1=X!4uw~C@!l=2Rw5@Kcu zh|L$$v>lHmRVvU5-wCYHt#?x|U9tk8GE#|vriFZu5Qke5(Xl8=3eGGzmxao*m+b)` z>Uk`n*y!}Ez9GCyG>uK~3k!_D5WAIVv1);=MbvTW4dg52=GlIj&O#fsL<5h(Gq&9e zBxf!kx{Rnl!jSdF7Si84^V0FhVSA=;cD}I6@oCw}r>k-8tFg5%t?%oy z3%i%gQ*hhR+(ZqW@qbsRv252BUP53YHTafKsYTk|9NTKMtZQw2#lg@*hKuEkrt9z5 zoYq~6Te^cX6h(eacPtMoCRO%O`sD2pZXb0u_ADwR)X(o6&}*{3LQ{!-~_4 z+6s3&I$T@wD#V^_2bLN=Bg;C)GYGJJEQhaws5M|sKaf|NH_iNQ`0psbwIspF= z)Zg3UZT6KoH%vJu&jGbe_iwnBwFqIEG;XyrGhDy^bZ2y37-yFAll?68=r2|6L9_uG zi{6hK^_R7X-V~#r!TsSOkEc?qdIXn+ORZiQPnYwEMXQL3+^NAnLmiQD{rY`FQ8_;| zSt{9k9J&eyVpRsusbsF^23ThX zjfexUKPFkucpU`u7bS_*JMC+0P_fZ6d24bLkE^eC*W|BoW+bW7>~GguxlAGjil_RY z=~i!E$S4mFkIfWzSVey;F~HBF*wQdeMNOpke{yV|H#Yhp)pk|asO8nH9Q}72y-qa2c@L2u^L1? zDg5Qwms@@xdN%PtJ0*amL2j*z`+0r^lnf3vuhKSh-K;)t53+Kl$wE za%vKJ48dKg+3z_cPI0qhR@oc)11_?!TIyHPZm;+S;zWE;aj$(-@)e=e%O4Nl)96Sw zX_38@iXQ!_P#NN+r>kgD_BOGuRzD!3^0n7FH;ng9a)%}zJ&N9=mw8?xHRV>{kiit` zxrohnt5O!>6Q!C>$SMm@<64$uoyO%KB|2w^HJIi4xO{u- zFx3?A8q&9%<|O!;HM6J>#k2~qD0d-6wy zYRgns3-oPl@#VFU75N8s@2?fgZ^y95ZWV^EvT&EU>TYhIr7Q9b;I!^l{54bLI2{>x zRj!}YzPqRRQDC#z?#bR#%gR$+l*az3QH#OiWY2!}jTW;C`SayoZCDK*Hv6}tsQ5(V z$)k^Q*bR&i(4#{BFgAHsD!AU>Nd#;-#GXysCXcPe`Tnc}p#NN_psm=SxhvHjtu4nE zmE8Th2C?mES@uc)Z7l-E!KIfUYJSv>;SC$Fb zijE<-3X=Eh^(f$lkJ;t=ID@?RyCw9to3yP>YPRHNB1qt^kI@Dt*V9<_tiY_sQov_m ziJ+&7zE0WgvPdHffz1jeVp+Y|866}f^*vLD<YI2{9_&q zcaWM^y=QHn-FTF~2Xim4rtO2pJG}eRd0d(iET`9hHN`~!YT`IwMyI}K_Tim(j>SSm z8pexMNq#CgM21R8X4d`UcpC>OYmfxIRJ+HbTecBai-q!cizY%tgx=ov{lPijA zCDps51OeEpmk}^!2a-ez+mD`_eW~|4&B`{abDVREYZ3aMAJL=WywC=f;yfo)qaU{^ z;Bg&yO$0b3;`CM$5wP=t)?R_~u;TjR}oWn^SL{W>FRt7jBT&Xy}OYnV$b zy2vK|%_o7V)vp4@bEu!TkN5AmtWOBUUMopD#h;;@(;l;sk<|UBOtPzvgtEw*#&D&- z80NVgZKYMJX&tvA7rB#bnC|$kK4-GYYhS2|K$m*W{|@_O-%1&gvz^bA@~*6-1y$uH2O%Zp9PMS35AC|0R6Iv>bTY+D0XX*8$h zKxzdTSwJ?@W#HRVvX0uwFe@`lDiqf&+c@tJ)7NDDmU&gr&+!x-srnp& zP*kP3g(X2oVgp-gVwb8}clitV+ki;ssS2Cs9=TA|vbI|JA#$!aM_sS0j@S~_>d5uQ zHMYBpzR1d}3~8b>@E$29F(klBkuDlg)Nrvh)+4oSy%D=N$2jtbjSfRDVi;qGuM$sz ze7dBTGzUOLrDIdK>CcargYV?;9z_-oYDqpbp@0*zQ^D69(W5_vTZ5vD8HnK4Lrh85 zLc-Ve@>5QioS$J`oC?L|=LSZ@m_q~e7+>j$P!M3IO4wApqUYBPRoIW_xtQq84L7w6 zKX8}^g*~xX0j7-(PJzMcbABQ0#Ca_`F`am~G@^WshOYUR+^(oTCKIjDO*VfwrGCUF{ECw^Aq#UQL)(HpYomC z4UpKe4gAWXI){cfhSj@@?u$H%;MKf`Jbzx2RlCgu$}@kc+dhMsbyld4ky3b&HI&*# zHLjZvPn+8c)?YDFNnP(6Tf4c`FHz5{9>dgM*>@E&taF{av=vRxUwI#K>Xt(X+2f0pEKr}B9GFb+hrR5^tSeR4Dq3IFb2{x! z{eE09Pq*xy$UG}pkhw>?1*Oz7JmCK*IZ>v)jnKQoYeiV9d^VJ8>&JraFQ^{sj9gJn$G}dHFh>TKIrI{<)GH;Pa+;6KD3# zzGFk{@G}l}_=rK@Qi!ly?CQ;L_OXpFKed}yo_wR_lWyw7QBFN~sZ+nt$#U$YBb95{ z3G<{*VTJAGcUp}3AIanO-@g!al5PrUNtSvi-tp)=(^85272Q%#HlAcVz}lS~ELRBC zbNH~hDL9VcdN)3XYnd#T^3fLE^DyM2L)nmvc}4U08Vy&I=+@e|ixr=@`>w!b245D) zSTky4oGNU1P-Ns}Ur&+Ea8*(GscVWlG^(a$In)sv-pZ|0G+Z6Zy5_!O6dZkwX)0`j zKM}Aj0hzujSW@SnKqpzm{$o@fpRHwNEutm=*KE5pTG%Vk`KCnZ1I>_#xa)3C^PRyO z#T}D7oyN-G&D!OFS_Is(Y$LrELBUV;0~qPTPYgAPoLxvUMTABc+OQ|XM{;WjWYR^V z10whe^xe$+lM3c~3+u?j+8(`SF*kpKHRKqZ{Hy%)*@@<#-i ztW~-z2Dg*DKDQW;e5#Bz;4@)zwS9NlMejo zZHuy=@`Aapi)|4D;uLTy#H`ieDAGy4%lV0@0&f9WUY4ACIwkT#+J)8hWoz?%QyzPmX(QRv~wsF)znJqZ_Lcr+)JSR}H>LcktrWR5u$;chCy8tRcAPz!`ap zr7mK?mG_q9$+rd6p1}X%j#$&x7IQ2f<}R?$h+$nH>#si2Nd=K84XzF_a=5&U zQR?=!{K;Z5V5-8*(ojzd_|nwAIn)qLUfI31(t``!V>9$931EGFOve)ks^hGR$&wlolWsGy$) z3}p2)GW;T7)thr+M;n|wM+r?A4%vZE)(t?gQ3=%kjm2zku6YR~LF3c{=y z_Cq#mGTotw(5Hh2Hcf&{D802S>O9d{L@3$5lg{$2xL3lj0(Q8f!Lh0`5;I zn?c&r!ugO`ZO5i91@4BA3OuFP5_z?rR!rF{lA0ifblA&ZIk1#O9oXE11D)hGHGJVV zh&VIyDEwhu@&g5rispV?$`7Q5vxf?OD|sdP{3FI0qo9x_`>dJuksN=J^_|c#t2XLU=pg^v3@d_(FI1i&qb~ev%gRl$ zZPbDX8*}vg7b3I{UmWLqXzfziBUISby!xVI@6y~M#h4Y0(Um6Ah$<2vLmC4={D*T+ z6u;3fvp>v~8UdRnuSEdU9xm9wK7+TX63bTl61aO)V+i1hC%s;*OhhNjbE*jFrN8HJ zb-J#{`X=rX=3%jj0az|5fHC6|_tzPJuX0+L`|M@&tUN|>ldtCe83jMggJ|va^;u<1 zyvD`txvcIF%e4$-O2{;P4yEmN$hG&Kj1I}fPxo*mZ2Za-*?6`xK6Be%Tdjn0qD#e+ z-EVIQHzXr(C>xU&p{o$-G5IF7L-mu8_|q4#=i@yj*mu?*n8Qdxq6b7lA?Jtn*m<>X zRNQac+EP~Fiy7K7D|`EnsTUmfi+414YHzvxbjp%q^M`S8>J20-;PzPcCsld@fDp3O zmj(Kc_lJQ*_j7zwqxM9xrAC&l=ZW7G)M(Y4wb_bS?LH{C6>4!L|JG$|@A|C3AB#3A zSV1Mm&&%5+B^4?qzw{(c*{9C?(cBP*1zV30nskaJ;JYVZ?S0SsWekWObg@}~;rvMy zW08IV%2xwn6E%pv0L`0D;o~#s`mV_M!rY1+I4ssXqDcEoo$pI*rhJ&0V%z7k$}q{0 zo%WKlnfCBTBJ|D5oHZzltVc4hY=&^Hu@)ZYj?O{83xw?eq(l6d(oeM>aure(SdGY7 zi$$w*FlgUeeaGci^Zny0ZxqDN9)A93ZGRoSJb2}?#dTORi|f# z)_qs~wZf?2edD6s#iHWk{$t!-6^tz%8@czKMv-F%PY!D12{_Ul3)dCrWwvTL&D0bM zt$I@P`f8Ns`t$SyF0au&t2=A>q~|zYOZP_Po#fK{L`f zahTl+a13ARJ<@64=z21WXCb~rN`9hyR{{4Z&_1AHCCv^ zFUjoPy2~dy|B`nJHST1*r3_r2OjCytnF{{qmKY8D+{6z)KTR-`7$W}HArNw^LV(92 zZyHw`2Soe>B#HM)HIR5R z0J#v*?Ir?rC~5?7)qC4l6Mi0C9!K{n_TZhxt9>wq2SCFv7eKv7k4VgfYSk$6G`0JU`!<2whibmwXYg=L0Qq3yeV>AE@kXbG&An?} z54uh}Usv+7JwSm#a#v*<#qrpq$i~%i3uq-@!vlcB=XyK4=i^eJr#;3$oJMdoijvXj z$^M9&sK7j<&oz*2#1?!48ZN1iSjit06h_9+0wlBq5QL|L!xzUZc^6-ckenfcUw@cl z*92X?9vuDokhI|Y5>gT!vN?92h%?x>e1`)}6Ep%!*!V-$15}t@MT3KV08zVe;Q|i! z3-N=Y4Elu4?TWzl&c@VDmzI~m^||)}O>5}-j5uE@;HR!vJAke$&l8i(=?<>^ko>nm zn_%L_Uv3HW4J3#A{t{zVBW4P#IXX2%5Xs0if{HDD?i>L zKmz6F=Y`03a;#A8pdM6{Uc*S!w2LV*F62{uXdMF`H}Y%&(98&HRxSFdZ65IUE`Z$$ zRKcnfIA1v)-PKcQ;LZf$%N-aT4AkcM1uqynza7Zs4(9Cm_n7Vanha0HhlQS=zVLM9~4WER~IhN@|zOQfIGb9yLYF1+v*c&(aIw zJtRADc#n5k>{BG-q0Yt`R=;xbcYlWIaO;5kNyPTPRK5E3EL@Zld8Ek<-JS1Qh}47R z$vG-w;@!p)e~i9^x%2<_f>!{ZQqk$(t%LEvVA^Bgdtlk9l#9d*d8F>iIUhAa0E_+m zq3y8j%{9&iT921F%@oaqaX1gxOu|_KbPeepn``s;5>&h=nj)KkMk!FzBr^c&jnFKR z!7zTL#04)jLcbpH6_|%eb&hlBA~aVaADGwH=uPCt4 zDF!KCIre9-O~=KyRf;=Q*GyNNUHrt}c0ZpG@obCTo7S`pzcH%BPu1eF|=?si8UV(^Rkb z1Z#)mB`kM)D(T;>EPnIBJqxPoH->G6(+sgXx-)8z=%w^w=Gk9+&`~v)X$1d3}KkpKs=3?6@ z5@=A@>`;=g5P17PhL=oT16_Cg(TJV*%q0L6P6gV0Jx_d>n3b28my|Ux27s^9f`xML zxGM}FO=OlXg!uUQyq(|OUJvYb9CYYz0BWO@d7$^n!p;6Km3K+Ncl&_veo;Q!rvF|- zb$>V4W?0B&LoN&co7;Kch1>T$E^=FG{A@v=@g_Q}q9$Oh|LKA*^X%|g zUMpYPl=0v!==g@B2Yrq{H97}xv;1Ih8=hRze@_|Tb{DarEl>@Ir)Y|F|N~S5VXYUu-}Z?ct~ddzx&F5kw{VmM zbW<1ST|iH^|G77Dn6HD(GTHK&)j!_#wjigeR&BXb)&DOCP(>E(CFIU+%A>FR>*@*7 zKqE}m?)2}qe{F0su?M453e=qXqGx=3^?{bVsLK&{bE}SZVC%m>U8(ynFhM=&gw_r zdqoInEWZ3jpqa4kYL|M`*6M7N0~_J-%1hOH>KyExlPYZZutEsXNY|+zffDp{ShkT2 zXbtuOG(%&s&w_a6GH(GTZ2ozxuqK1UbVu@CsV}6DCc)8;6#uk4B_ZsqH`-^J6%M*q z06N_O$zbiVgHH1a1aGSS9N=YEJ4#(4tr_uz4cctpjbvXyr($N#kot;=v)`1i%V`(|L@c5l^25X~kfNvbA&z_RkJ3d~pK#z;`62hx`>3sdEUvANU zNxd6x2>`(fxap`4z-prlQOO||0FC1^0bMP##t|mG2O0&2bfu}+UAC<2O>zX?Sxurs zl_2qCR{aVAt^Lm%V)O01mT3h!ACNOQK%YT{^p90Q1l{9R&zMv2^4j9?Vy0o8x_ypi zOB9EMTDr!4i&EOfhqW>hti&=GqBPO#a7M<*^SNX5Pozt`foOr4eBfSyQ+)d~-?D7a z>^lB=*JQRzAUQn9?k1YEf$4xEFCPH4SwK1+hc8kziUNDIdaU{>KuNL76Z_tQDgWh%#^gdjNdI`X3X}@Lk7g7njIxByaMZ37K^_p7mXA}+1=-Rb#wHsEh z6=aD8ZiDtB^ISk4zoaeGVdis+tJV!6SJ`$L)NP_1bas&l?5-mcl*2A|u)Jp!CE$|V z?+8?F#JP)yx9Z>XxUQJNd3bnS{hOO*7s1k8=Vba70C88h_xI_iLsSliC)`->AUBoNaam&%_G(IJtmBeja^QjI`{h#yu`Lk zRlRuD%qMfoMDNfti;#e=6_FMC1~isvT7UC7I%&S&2~+m@jB^c326)jr*uQ}LyksvL zV%h+8%yf%j>;Q!1P)VOA67h%|yxVhm<(&$`gY z>pT$sa9ZpKO+}035Y?sYZlIsF;Fa8kLUUavg|l655zfquB={PvIms`2%BgooP7i^0 z0fJYsCKlQY$>h#x-C_!v%UkDC-!7iZZPHO{Ai2+H_uld);kM&^?jtaswpl>aMzts_ z9gLj>i4MvMC0w~XZ&MD0@hA1@8q_&2nb+-)QSU!yKQFYAkP-1kqf(>P{w0!RJrBcn z0H-C1IzT$Rd&odNj4KpmyDrQ5q=BG~8IZR$Xh^a(mJ9%8xFUjGyRbP8G$=?Bs}=Mx zYMlq9$2n)22V}Yxpp#AHcuq|>=+WuBumt|;_|%YSZoQ7}F9IP`NsFpOu;8c3bZ{9Q zEuSWFIWh5o>2&`E!8*T7q|0hN1CO8euY(8cDQe3ce1GrsaN*K`DP6Vk%V5VII`+Rl z1T754FMPi*zEr;G(dM+w{Vx8x)HrDM0F%?$T$?syEuQ}P&b>d*ZGji+D0GfmNQd*z zSbsO@#N9p~!Kzo*v)j`O_?3vRu8#YP1{_18PBt#?_s7(0AlUnIO=C=())sYE>u2qVd;B(wAI_@SWZI!)hBEYJaz$frC# z23qJN5-C@ZpcBhrD!r98=iTH-Rmj0^^nA&N$K)xw6) zbj(iOZkHg{wY!E~l(Fo8Gn5iQhbaEA`ktPJtsyxup=9?RkBws`tyKEkQ2S>nn4Od) zf7_W6ZIY0qFfasC+I8QRJ6z8bRA4>Z3KmlU@|tKW2 zl;uYFIU(fl{1vE;?Mtm3g|%5-l-zUfYi=>>qFX8)EJItQlU7sR`xR=X>3CIk%CD`1 zc3TlsnGo-D1>H8q`qI%R0jNgZ{EATEB+R80%u31gE_6VVTVA_6H*GR` zePB%M5fVs}1jN5f4xCVnnN6Xa;clGMhq+dwYu@#Yaiemm=(EmGw8hd}IT9p~awL9P^@2v}+qx@WR$)Pc3m0v2+V68WrcI4GiUk%~s&px7^ zD14;7j)IhT20YdWm9q2%+cjZgmchk1!Sp8z*wv$=)w|Zs->~K7j;or>+_~f3V_Ge~ z*#*wd{BJ!+*))7sK)$bNrPpJPhe@a(P$8hCFz6TQ_M?sJ1xtm--CA#!dg_UmUh!v4}#R7J8gi2>GQS#L5&-G6r%`6$# zY1sg40=EGzFEDwtle?b>LBxf!n+327uDY{?1v~e zcYj< z_V<=^<{b2WRzYeuEV+;@9qUg>uBZ{T2%_O`Z1HW`pSSTs4C$jKSTkbP;^%zMvOf-h zeCx1DA=*V<)G*ZgX=k&|%Mq9geufYLzB>Zr(C|W?ce@vXVP77dk|zIt|0I-{7E!{V zl4NkxYwNtUFCcfyg)gx&!;++t5-_ieoY4mHugF83w%dfZt%lbUS%6(y=Chv~3COeT z4{Wuc+68?g*jzPM3WQ3grfGxP#~iqYZ59TK<4)nmwW3G`Sk`8!uA=CBy?evFGE+AX8d= zfmQDJ+(HbG zh+-x>t@MnAXi$U}3Q|I8A5?3hu@})pn$!}0sAXR#or~*TpOq!jQQc5$K=EoZ--iCV zFtzyE0R9D{5bcL^nteyb4l=;bW&HtM`Fy-@EaHz}ypJ>k98i^1ky~E0DXZl=H+OS= zJVAqV|5kSml-*bRfnZvR51?yt;_7$N*~H?HmKisATxWrsOf2&|=P>ROSpvO_*qQ7= zZ~%7x<~Jqe>03T&zTU+G7{0xUQxvdPWcvB@?{OmW z;AV&nk(({K&A#IsyJkiOGv*-hJTn{{Y&K B{V@Ol literal 112801 zcmeFYbySqw_Xmu0h^Poimx#2IGBij_m(+lC_t1@s(o#crclQ8FcjwUE=+HI1!@Xbc z_g=rhyVm>fyVlEEz%$SDoOAZxXYYMJJLZd`yd*9*2{sZE60Wq=8)YOUv;rg~=JV^F-2)HF)Bp|TN86DVzPBD@V#t0U80n6mOVRsA zKh)ZS%wS6UDjMAX@SNMo9&O71Gv*7N3aX4HEhJ5*jA=qNaitM7%_rlaa{V?Ks~u+J z)u6{=Q~jpUk@3Mb$I;@^5HCJbH)ohsG`0$o6u!x4G06P($g6Uvxu>ltT*BxuB)ht8 z8WX!RAz>r=Tiu2Ed&5=D#@yt)i;K4!UpIe_)guXi!|Xa5tvAP{l$=D>cs7WHq!u_0 zs$_pN=1KfM;VHwz2>iz>sO%B|rRd@|3vNTX_a&F=^s_9vJmveWtP>b3rVERc3wd zIyH6GM^>xAh&AwDg_N{Gs^S}3kO&F`(@hq02;Th!!+W-!KA)nV2|jY{mox1cd= zUgpbHc2B(940d6)d3MWDA+;jH=LragQTu&+;)hcuxt^cv|wKxP_l z67uFZIy>As)SIPyPmwd1c;k=`Tc4ncy#9P&iCRqxXEg9FHPh#N^};C;LQk+egzXh+ zo_%?W?e-+|Nm(KKFcoV$l2YBo03=IL^EO#RfTHu-bPLgZ*;B>8Dog&Roto z&TKVk9uJ_w|U!SF$@wDeOO&$xBFNpP$yPbStmI6;s6DRUiz{2yT`KOPf<_7 z=KDSmeb5&JeS%VH6frMRB}E_lNp@3P(T+a?VDeyQeqhMpABos{d!PO}KF`CxF9Yqt z?IJ6r#Kgqp#2E1piT(QB;~nB@*}=x2j0Gc~NIic>XwD!lIxAyAl}tNN3!#qcK{w)Y zVCD^IikXvY7WRGvr;?4~0vq?ijHY_jduw`6dzzzZySMvZt;hDQ_r~;)M>SHP`HQAK zRebm4fiX*Vq(h_wn4Li(rZyTcQXy)Q#zzqzbRCB*|4sOtieIi`TAALm|94CR@rmq; zJe?g03-S{hQ1qv4;?e0UBdu5aFN*`Y)AQ2IGN0zh6~D`O)v}ku$~H=CSC>(n&2duQ z4jZL6#x>4i*K@0MsB}n$dBmVG+Aw;?(8LhNc*iIdIH@U2JeeRbur9!x7`AwBL2aRF zaW&3b0C^{@y;SAfm}d5E1nq3%JqZ6q@nmA>YNE7I!NSbK)$%jvshQ5OYkGdxc20be zW`Uc8qfT(adIsGv;{>4aB&0;jbW~+~{h0OgLrp_Xc+HzeiAG5enJz_oQF^^x@y&Er zC2)FTnsxy+pD#T(nL7xT`q`;tsg>bw6uvl_}PhIWOyS zL=ulNj}`S;P3!c`E%Glm4P39&K*Y;~otbxl#}nTtX(loTSNyX5vrCPejRV&$nedrP z)Uqe6Ei05TC=yhzCb>p$Z*RZddBAz~)cw@{RPEG-m{a1V zApQ1S zJ3YkoMe#&&M!%1~gMp0#^%s7~XJ$Iwrr)-t_O0cEzL>ulqgcuZtxp>todcEH)IX5l z_aHi;XlA(<`JnXSSq6tPtAn|b;o8-U@Vv;p^eyob%Z$>Fyber*Sws8sujMD@QL_Bt zJuUWzacA@8QNF(WU}Lt%3LR^=9oQJtn!24jS#Vpscn5jNRX^6sxWn_+nV)Ju@pTAY zc+Pl-LN$6xH~Efa&lT{K@MRu%K2FAWc^*bz|2&2c@~{k*z+a6CPfD*&RziHs0PR<}`ZU!1^lE zlb(3csi&y_I)We`#H=l-9p@#`^~}_9a7?FkE>%5K!%@R!!+g`an-tilqf$wzMcPv1 zn0(OS56awyOi@k?&qY)$Pk9DkM_$*GW^y|}F?&+P_Vi-#sxe1&M$~?JUQRx-pheP$ z?Y!Bv@?=9Gv4!wwPzj;-6EK0#!uNTH6O2B}UZ3+CuJ@Kh+>O>(HHNh~wTm;$4XeUm zJf&@Vx(9uei?feMGe&D0)%O~dO;pq&iFMi)9(i*uO)q=EYojEYR@z)cRX#T_fEyg^*N|-HDyx-4;$v z(oYXQP;6OL*;?`EdE05@&pk6I5vn>K9%!QogeDFoSZYOU1%><4M$!&>hh1F^Dz#)L zcRlS&2;ZeF1&9bJjbm4DRLNJ-RpVB7Rh^|A3(U+ckK3;h9qEzCo2M4=7rVMciC5zN zITi;uQj+=U0cR_oD;Hk}36k+x-{{rN6O2wcjvTJ6%L2iyP8^LxK<8%9ouJj2C=Fr_ zXvvJOCB(+#roFVLC1tC8*BtWRC-Ww!Z6Iu+17doqb2YdKs{?IwU!7LNTj2|Qk6h3X zSRs6zt)x#I!S06>H~m*Cc$0Vr=)n}q6tM!zKB`xVKW$mWoW*hj1wNGuJ@B~J|G)>k zK096%3Mb^J7Gm%yIH|Jlx7Zu6n>rsbl}!m1Boat8rc`4K1ybCO?GNqZkKkvP zwwE?T7GM%5a&6j)waHQ6qQV!lA&KsK$08?fe&HbqX=exuB==s1TJ zN1MIy?c0;$m5@F%zox$DJB5oBVThErfa+UkuUlPPG5)w+}CnOBS z|Hp{mH)+Vfo}v|`q5Qf=F1~wFL`6(m8u4Gn(81W)#?j2yDXF!;5HZ!1x#~NocXDq5 zhPKwM?~QB?j9Fc+?e2yk@w);Lm)6Ek@2Om^t!x|tt^zcFya7O5-`!@Tq59($CrbgE zcXEnUVzv&(RNSmDSzpo!VpCC3@jDoq0F>WI{AW1glK_pGlan2Qjm^cyh1G?F)z-n3 z?G-OCFWXCYHghE7K+)XQ*h=$_xivyOh&2Sc zUvu&Q@&5lj`R|H<8~N_PBVTcHz5M&ozdibWsG6g(gP5%~VofK(|8C8H2LJuxe+Kfi z-Ld{RQT!?AKkgy~Er`v}_SdEfVy|F!`XM0+BT2szQFXnyH;>`^M4YtsCy^*7HT4HV z_d|%w!#8h&sHBu0J$XXKZZu+R*YiIR`P)rFW>zZ07kDLj=6kd0{|P~NuCLI0D| zLn1Sk^x&>Ush5h$)MDu2^on2pphDQ4C=Zy~a9z98ay;02L;D-w5f;c;=mjTyFU1)q zlIkDcG3JYhRRnCF4LtayE%KWLmL_qYk>r&qH~Ks6k6lcjMN<4J#|R13%QJnYEKb;q zE%bj12Qiz`ak|H{`tQvDCEX@PU*T#ZLC%Jd|7%MS_s%1c{4Dg`6XtOKIrU$!FMUKt zEr9$;A2I&7O#`E1s$PwbtpWdy3brRyiu)h&LRo(o&@Xv4KNC)y1*bee`#0@zO-E44 z=4s|c{4c8WgYF*kMDsCcJ<-2WiGzlz>ai@A+4uX7|3_PW*{~kwoo46uK>ly_MM4I? zKv0Q8Nun$9+s^&^=qZg5Hk*Ll0ouP&DT<&Hcs{(v{%^vTMF`t#+uVWe->4+QMo{@Z zPy+nuuRZ>!*!3PGge|KO!D=HDu1Gf3}e{%7Ixy3W%l{~`@FjlS&93LjH;c;g=HgHH;RP-AyROLqlTZEU=CPTqj=n9|6;@T}rl&6B zyU7Gc>OH0?Emb>WqGDN^jS`dafrWSMru8AWi_dngwO!1YIsA4)3v0UT9<_AO8j}`X zaqt{hZV#{8)E|EFQ2POBE!rTsSNf3ZBv6C@Ip9x6^p~~XSya~BG|^C47><>0ZCknR zq9(*B*=uTAkkUh>rU(>P!A)untelwR1Tu}kU?OHWvKudta`5AAVTlAc;vGNDn$&2q zgZp2@2A*t>!NtFX0is|$YuK1`Z|+H`F21o^U^GTfoU)4gED%^6{NV$-?EdqKVy~;t zc$HT>Dl0LkjE0TUyO)tYi$}FLbBjbhF-$*mRSVRBFv5OvJ$w9Gc``MX9Qc=1_L%A8 zg2d@!(o^+YI7El$v{L|Dc3q!kUx?1L2dR7c4-1I^qA^Y|Zq}V_<(ZwhoKiC=a;9bq z!j@jC`3x(W9$39Amu8$Q(Lc~C)t=%-nI^Dos4XX{{L1X7o7iUD9#J|!I1uqFK&Rdc z$1IO9@RU4uIiNXmN^o~`G6frTw7_@sv#!=Y%ZLR#lsOR;L-IN6uQ$*h?-gv+j&3Yz$@RZh@JZG?l%Wubv?~LzKFYnDa)Fa>e`7E&7 zR+83X%X$3I{L%yWbpQdDs>cjRR^#7|dWjr4n~{)bR3s_*OEsX${Diw!_=E&6Gk*Yo zl6CO~-tmFa0ZFNQ+4O?$6*(wI*MB7}VDFi%{z`Yg1ANYHch2+UkxZ^6RC+ZQ*JyDN z1=cm>gNTKRw7S6kz~|7Z!?!Q->L(&_QAa-`#npFU=Xo@0&8q%%@>VH{Ww`#KksnfxNarsroFi^ z$ncroMyl4V$#bIW=&i$u)?fn-WoHj*<|0qv#nD57^SLi#=)tiAn-W$#R3}$Af>8uG zw~dd$n+boHH^MR~P#GtgH!znSWE%zZ%9=0$bMoSRcy{?G5PA(}1g)re zSlJIK=t7E&j*s}cm2*quc)ZHVFvBD8B{QDY%G|yydi$P1sU~jRMoWd=7@^+92VVzqw(9(Vn>jt z$RyZW6#mf`$^;;gifu*8G(Vq&JO=Ui0*XaX92lu|MzB?PQApLToKOpDOF(dIjh=OZ zV_wMzl;Ch6G3&0{&O?up%yLi=wysPx({r|_G3}s2jj0N~^?r(@Z3T4@xC`(Zc(`>& zd0Vy{%dD30{+K?exXxB7-Tsrl{wJ@~I#XA>DK#vof$h>qXao*R%vxF_tUbm8tk75>92O z*2?AE)G_uX+eaWgXA=BU=%PBHvoi@iuYEdQn5Aru0F@D{vm+OIEp9GV{9`}q5Jq6s zYy6MH)-MY)x=^|Im4)Vz^L#V4pg_9?)Xc(b-r|H&cEUY|%r@PW2^tJk=>;C=S$m9p z`_X*XyhoT-Q#%56uYx)10%Ka94>$^})wCEaTS!LCTG?IAhxQMk>TSwvGaCv9*$e7z}rPQ49X?s#+MwPF^1-`z9sEjcOWOBbE?ny5&@L~svDv^X|lAAL4s z%*(=}skqAW3X2g7OFi2s8nhD(n6AM{w0|mzSen0V%FazA?V8guF)d}3bk(JvDJu)) zb?Obi_LPEmt?eAcB}OIU5H$I!jI}m8dON1af25~b>lr}C>#EAC*T5_;OYqfTJxFSz zAtqb#=k{opGH&zORDq89V&^Q#8olhQSzYPUsE zF}6ijklyLZU~yKxGG%8QY%oeMvu*lI!yR?aiAh1-KJSd?+AoVYCTi>0 zeRKoR#he4LC}zd6Dw>HI>?6$L_l4DJ1N8QUPl4PL0-!R zzC~%e7_BVfuT6I%ak0g%-4PWHP@|NWU4D#K8c*u6Ul9tM+0}b3LBV6`lF0umEf*x< zxC%X!Af4hmb6g)_ux_l0OnT3&*1UD19)v3pbRl_icCc@46Sc=1FtA}$+OuB?{rEWV z+`fItg446fF~zg~aft0RJFJ37b=e3vV9Ow6o1*Gh5i>#G!nheZD3Bcte_y6ntZyk_ zKH`=+nNJBY84>b5^eKpDZ>7DR#lMIl;^Ou-S8l|>Lid= zr*17yJzKZJXu#^)KRsZ&VJlx=77l!#D(5-my#un|J!T4T9@nif9(vvegzq&Bra$@z zLhu#tMRSUnrpQ}x2h~)-wY6`pEn*{vAJYCvq&7s)S8j~F<}7P>7o5DDQsOKj{c-wZtcn&ZM#wc~_6#BPaRjKL+^Vz$F16KyEEnF0sJH(1*E(L9Xm-D zc3leX#a@n&_>!ZJ#tCn6JJAyTOMqKs3Frp({X zQVhLemivuact4|MEzi?z*L=hz8qo3wEVndn@(=57<)VoWNYfl)#d z{qu?)*%hJTa7x~EK@fcOsklu_pgcwxlYBC2q*L|8SZ2 zP?govA}{%7fKJ1jMC4AB7X|FL`ia8^fAv0g>z>n>n5<*o^4;lh zInuLSPcEluh>Qhi5hJelxZ`+vfUp)3m16Q_>cbmHd^@?@_j>8)LSP*&wOzGW-b-;R zamO9~3f==aP$BV2P)U82Q*T&2{w;SRX=ysISeA66#OQ`!V@L52a=WHRr zO`?kTarcTZvwg6-4ik8?y!!?5`8nSX#xC)0LPzhDuhe>YcBGGrb#uZ~jjP8<8-^~U zb|v-P-#PW-R=(L2corpjV{_o{If0{^B0z3PHm1^i%*NzSunE=b^1(j%`9|A#K8I_q z&pfqzsq|92MK&U zk<>cxWB=8t@h`MK;1SAOGw!dNJNT=e)+90@FdBUkx(3f|KIJgtz|Hk6J ziw}d<#eGgESXNhohp}Y!qW#Y)^sXR_8VOY;Y)W17JsVuwbLwwST}LN+JBvx4Ima3P z23C1QgooBxlnctXRG*s6G?^q`CIW1tt29mV(Z*F%#vA@5{D^5a^b>8-`}e&b%-0; zT+a8=g!%rL;Z#fP=u-901ote`T{L`}Q!;Kshw0-*lilm#q;y0Sp=7AX2G?iXpl{Kh zvZ~f7RIruh6O_|s_xNbIj4A-T=s3Fs_>W&ACwo+LjA~A^ao=bVBaFmkofvjz?Ji)mF9k< z!{wTJSF;zr_No4rO+^ie%4-L2Y<_U!riSg|_9WF;>|}#y>5(K}+@%8BjpkwZ92=VT z%dcVZa9iSh)qdwDnk6~pki65JTyZ_zd*n_m>J52u`5aE`8xz`e8gLy6Exf3F8A#Bi zYEH&ZlmYqE$+38+k<{xQf}q->;9KgVUEL$6`mkXeJ5$}B7#_jfXu&RKYYMCFcV^Uc zr#)3Oop7zg(^(1MSs-}$^qW_pxX(=R;89U7g&X5lTe3q$b1_^qxrTS;aZ)08ui11v+35zu`4b;3PO6OlvN2fwN@G#uF6c2 z--EhDk{nM2EG$r5Bs^i$P*rEuqP-o|w9%0tVx_q02_HQaFqJpGIV%-V)J=CM9toII zR*sxCRXD8PNd>`fBn`;|WeUIU*%ne&H%nq8FE9)5;Msb+Z`U0XSia7EVXahnzf`wj zf!Y93^r9TbVVdnL@R{r$lxr+z6cQ3*(5McnF6G%1dVAuX*2now*EF_BWKiJ)$a{+; zcPJrn9w;WN!Y#Kwnk@_4_ZYmM*+`6|?Hah*^%BS-Y>+B4!o!u`d)f8d;otOzidj#7 zWLtor0D`VM(J+`+Ws~bDO2o9v>zafBY7BbCMXL^g0b>>}(rKXgxl>aeddJC6J;~bx z%=^2X@d(gembx=Nm%Wu^*nAkjx zNJ^aU2Ss#g3o+TAUI?Q#cJZ(h>CBcC2@}2m-w@efNO) zdL7AsJQkC5fw}dw#ed{w@j4fOo0IhPBMMHQD$#pZ98OgYmFT_oQfWu=@FH&6{2;nH zW&5@M)wf5>0NfsajTVY9yPOS7WtDhX9{(s-%v0 zaVQ@ow`OxwC3)h(pw;ksZf4opBc^LVP2aR^zFJB7YR_Hp_Y#G=%fD+#Phr2Z5_RYY8U5+&3tI##Fp(S z8RXz*&+VZ5o(6fDgUw@#p~x+VUmlz?);yA`M1(41dztBmU$D0LM=sQM?sM24t;>SR z2L#qfM>n6aDk*xc5QAyjK|zu5d;3~20*8&*+2ch*#8C|Ij2Zk$92^U~W~g)%-SFML z9a8W~xY^T&s4Wyh*L3#WF4S+3xca)!x}*jqCZ0w4T%W)Q9dceMEk=<)5;%Ua<5+xjw`&|Y3BKj(ljcmg2dgx+4yUFi&PGo_= zgp5zgQ-|CGWqyv_E`Kw@ubt`zG0OA}4$SYURIb2t0ewk}gn3CbrXhVcy0yBR%&~k2 z=2m;V-)vn#O>HB#0(NgPYC&2Lz3@)IE?*v=2BvznVmsxU=$%yGVHY$GxP4=hyn=y( z(#mXF6-l(DNUJs~ySuF9Ha&6)atd;vNumWVV|-$m#tjNrx^jJuV|0zpRGqPxq+{}~ z+qV$9FaQLf93=5PENLaLRJrTM{S@8X;=}PttbYg2QxS`KxXnqv?LBS5z0Z(9qh37w zR76U5EY%2#GUefN07fKD2ihigfA}pvO*Qg?cTTZ``Y9=(??NX&={+a-ne$t4ef~i> zTPx+(rg))MU_a2UEoPjGE!kzZ`q`7b$$bjSA7xcLp1HdW6laN5=Fd?|BqBX{JUEZ7zJp|pK&=7C6AY^D8(<$})nk1OE0zs02~+%Zc`;OiUD zb8>WYu>gWCsUu=hON5kTW8LNS9x5VZp%s9DMZF;YTSd98z+%B%QCvhqBUC5D^A*Et zBzR|4Ban2=3z6A3C{k}S_-5f&(#Q1Ji^lsh6u4e{WXNM;5eomf^p%Lx$rJDT`oxIW zo)dlpS}5B%va*(3L^tpkej0Xv)Qqt&iBYAOXRu89kVn?F{S zv&EIMC!X8DtW{8kt=K}vhPTTE`0U9c(`e&DTv~$syJqQWY~PMJ zW)Ly=lPwAM)iua`gbkhrWEq1=`H(&PX06p*riEKI&{iDL&UeTi^aPnAhI6ak%Rheb z&QmK;8#(vex@0ga9~H;fab3#ByOdd!FO@>%h1-sWPE0_vwc`>o`{93vq z_b5_jYB&2JEMKwGCy}0yd#{2u29e6V+{#U7bW!(Au_iZKtaq!`j%cWpA%J!M&?s`8 z`7Nm`yd;5z8i*>b$>pJm_nlEKznv>!oTq?N8d5@mRU+vC`~1sMF8@ras-eu zKR?XdXlyM+JN}M`NYV%=y~|f$6z~D9&mQ0~%$BC3%+Tg|zR;yOqh2L_Gl8RXT)J3bX>3006f|=e^+y{O zl8@=o(%D!=XDpM~4jfrs68;sgq|Sz|<7Na`9V;vz_XuJ>+I z-?pc;S#-?PW=Ou+)wfG&;$0=pS}xwg&o$nhw5u<=Os>_BTVsmuD&I07IRNO2Q4L@o zAHI4s$+q|c>-zjH$4wUPf1s3|B^n~3Kofw07waO48_Nh1V0-6c>CxTzXBE!Q5_MK} z%iubMjMFS46J;&W*{&?0wHoa>@HJZSI!ANKJixjdv*!`>#BP1~3k7XR77t=#;ZfA@ zZEfN{N`LtLHwoO<*nf2$EXCaA(qZKz=Kn#yFVRRi&8RT&C^%i*?m8Dnt6RG$eVB)Y zZao{gmp1-WhHd4$D=%+4W={PgTum4wKh^Ui_|jqL(AYjpr9Le_#KPd;dQXzXVI<_m=hQe|JNe zgz&ZJQXbg?W&9@(si}uQ1pH{Sv1Jk9vf0IO?h&%Nl&>&|#Qmw$@68QWAYU@R z$Dm>pGD##q)hBs~Y!JH3q^rP>gb+ddli= z)zLk6JWaBlf8r7#OoHedNk*r{)`iLWLHJH($IHzYn|Kef{WuvFvu!?d&T=&TRguin z(HEX_XK2Ahl&<_NvA2Y|5LDVUA^Gi)+V=BN>0|Ow9H2dWD7^F>hkrV^(n(G4_rMmR z%g{SadU}-ZBKSNlcay6`b-3#w2r5Elja}slGML?a3PJGln}YEmShIPiN@Zt=Mxw#s zX}IWH_&b|_DEpEzGBKp6#fj+m34YDj+I81OGmWzTKaEVk_RSU_VFcC+A@F}QB*Ev1 z{**3?7qtPuoWgHF%{39>BWyhkj{l8{AFmMX2%ZYplm8;?H-=lG2r50G_?a#LadZE) zWJG^c_+1xGIY#gPZ)^Ycap8_iXv))@e`^Ete2Z|4;R1>_82=)xAKVBk7f#!^*#5;I zEEytp(reGpf$!g_ti7YMhF-$w-@0fh?}TlL7Y+T(1OFKx{MroECOHI^wF0j$ga1X? z=Wz&OQ+lM$J@_{&t?wN2{+AKGe-rlq3zg_zvnJ2^+h%ObQW@L=HLDGz{dtl_*rkMH zk+6D6n9;C_zJBkJ`K;GDe#6DHF{gQPHMCePxOLXl*yc4UNcwjvg-{1-wdo5f2 zjj%b?$zn}vTmG{;Z7)#*k#E$a%|`*FH$P3E>G_Fbamgy*GUVUU5V0nK@fP;jXeHz-b|NbuB?&YhQ?mC1xC}e*PjdfLEDi{* zW4P;t`Ad~~l&B0lLzF9lw6!i)ZZ4AB7vo3lR>Mp%Xgg)`YVanU4rde2P|Ia$JV6Jd zwbYN8TU`B;l_KHsH@QBnlSj-ug(c4M*G<1FwufZVazHoBxa&rzPmeuV)t`r4&NcugT0&smDr`@$ ze6G2bXioRb!_-QEvOC;X7W+`1!Q}9-lr0yj5E+2bah?%g&vG6g=rXH6{%51rOqE29 z%^Vf=j99kQM;YZBQDgJjjOz0%W9KR!mq-T%EZw#y!qWc=0dwvxf&m_t#Iyf;$GT-? z_v0g#A-o9cQhsa5)B(A6XqwZzW@PpGc1p?K%L%69n;e(wSfK8#)c8+BiTMKK+wLL5 z=9rnnnx2a|M|OkhM0IG0x3_xQ<@88fm|=FGTJ)Dnc=hZo&&KUZV5arYNaL)jF}B&n z?wuIf^=eZO>2^3I>=MoA)_hN+NK0&iR8R(+h~?mQ$sNtvpsx}J3PMBnhzB7=&*&MynHi+F`3DuFQ;0$L{zQv%Lza?bfpIT zK$o)Su(y9Z%2;?2=x{MgCkgQ~U7fvMgHPfbx8t?QoM@fw1xZ;qxGF+`9#VkGb| zRQtbBN-7r_4Tp%J|mwcJ&3efm;^;6LVdMPF1==*_bm2P1s!Ds$>tUxWtAvr*4%f!q9r~**bHl$GnG$` zRru6wlAJg!;c&>1zN-{|=Mf{E(T*{X`G84O;oirAZ^#6ao%v*j?yxW_YElb?e)zKm zLgAO((c)5EAj(yL&|5cxEb{H!V{!sxh}>~c)TFa0sjde<14PeT?ASl_$n=&$EYmQE3ZZ{PlHl!?w!v*Z#u;S_>?^4r>)*K35kQ4#(Z^En7sQ z_|}e;THa07ZNdZBI_MjFnl#5kxp!45_B$L+r87-jG&fymr49t#8VErM&2BAzlBfAR z_1w5W!i=cGOx5_1_5_jtc&$vznK+W) zIHEf=Su0ZK?rfF%)XIia(@UC3*yCL-^)>+Z6UUQoRXOR8Cfp^Ohxm@2N>R1KE0pNm z)+4jn%6#%ogUKF^rxEW1^vbnvOJGWgh)%`b+e-iG29d8TblE01o$64>O~W<2_IrpV z&f0*saE3xhIOY_n{%DQB=eFNr`MVp03FCDC6%hZDKKm?s7?dxGfEc3s)I-wO}Kf42Q^E)J1tUf63(m2Q`4z9Z!|SXu5}ORqG>Q|-cwett_I(H z%{79;SJl;489#OisYuK;oz;=?yZ07Mk$OBxjjq4le-*atTtX)?e^sQ`Xx!z8?*MZu zie^iaWsL1+i3wUieQj|gHO{Nq!ele4@Vr`o-C7N&??8;+_;NIPN8oy`M5`mKZinLz zDocGqIF;xvdqnV}qvE^mhK0`pY|(S~eo4EGPWA`7kg93@D7(47M{Jtbkca(%ULBi& zA$EG%lEM%ow=hbU; z+h^ZcM%AOmqf2q&L&$cC)bS3zNv$FS;x2cEpvL3qh7{FNnqF~bNOn~Xk za^Zm9(+EkB0Y5h;4?G|~FVtxuJF867+iZZDgntTKZDE5luz3-*Op2V>Ekx3D2H#Ju zknuievPnie`XG%{`x)L#{P@L{w|j39H?ZplA`OI_q`XP~0zx#%?qtJyr`W2ZgTz-= zRJJdDcqk(<7T|KWLBKKxSla36`{5_Nq#ob$c5~tL$6jnBtanhJsvZHRqL{keFsbZx zk>3u<#`SPfeYKMf`@;mEsp(JEu-N!5(sanuxyKJHZc?s);Xbuf#K<~qxB_0>SSVgz zU%b2Um{m1*Sqxx5sy(lwK%fWN#=&_qjn@>!+tV!z$V#}NafSb`;=$Q(9h8hpT zC&$|8iDb?VaJmjTzWjmL6UiG@RhW7nn76^3e*$=aK3D(3^Dv*W1vyuQ|03pH=gi#9Ch6!uoODT*U#aexS2>Fo( zDOv!%k>lv6a&thHUSFx-4-B!+*{uJnc$Wcs_M&WMQ9R5L@H(kQaV1~q8U>kb^OR}-@#mCr7LeNbv2#T?q@pyZ@$cECQ_iTSBrfN`t z+))V#cP}Bpe1MA`G1b5m-D~jyCd!ow3trTa9p85B?A`{qq|g`dKGE2H)ZS8h0he~O zR%)zUQ>M*5Y2Y!**Dmobhs)XNd+Og#bwJmDD!{8QW9c~yx2_lycp8gl-KCmS%4h8W zrN|>gjJcfzIMNGAngg<0rR7Ad((UDAk8=iXRqM-`m9|wH=70-;!Mks9MBF7kdSX`j z5f%#aJLVKP{GwpmTM{tXlo=q=g{C&k!_2aZUIjX57a02X?JkCh$nzaMyY7qo8peG> z!UJM{yY<=?)Fng(4|h}OD$OmiIGI(>@uZunaSIsnyb!{nd+|DPbv#TrIrzm+02=mo zQXX-U(h>rlWLf8yJWdeEU7gE1$m$fJI}tFyJF>Ir{B{_@$E`M@YoQ}Jp*H)WA+eWU zUA@crKey8bJu~8?86(LUdvZNL@%ukoX<^Xj5W8 zJq(3O_4RiCw9P4f(4F=}-=qO0uPB_koDB0VFEDxgcE1SV_( z4=u0Iv)}m4piBUKtrW;JBwg^H*0;*AMDlSe#(Kje*7^vDK2MrU9TaJ*I$b`}b$aR5 zdWN&c$5k<_f~9otJVx(S@g9s>V(IEy-1L%BVN)>>!IO?Gl`$n$cX#J5FX6<72nwyd53g7qY~mkN0rKjvi+%BbGalYYp2^T)U8@jDR#I z-?kk5Yv*)>zn z*RHZ3m_Wlg!Rk2rs3l@?gdAR*&)5e)D|;jFEfE{Gm9m0pl$y7Mu+F$INR)mum8ZtFl35$l>Wsj3#GtFQPkp9`c#-&BVLC;VAfrmydts>&GK#Px_(PX?a1ZjG^`qPX2W$(lH|SFx%V zPLb*bN{gT}iMK(E9poKQ!_@ecP?_7RrL7_EG~po{&;&CHui%KtN!qpVmcS?C2m*Nwt@- z_dQFbfzBHGZB+EO+eThsmfTREUBIDl-(9k;R-Wp`>vt5}^9mH6MYUU1Bs`5y?q#{! zX${gx>XCfRdj7gU`{JaVQ~6ac&$bzES(7ZQp+wDnv)nwQqGId80tK?n<` zT6%K6w7FhkviV*zTON1%aF-XhkHbQKfXmjvW!ncQ%|;D=Q*=OM?;^eW#-{g zgyZP!KI%yc8uYW)s~b^C4Avm>siF+#rA%cX0Unxb{ik!{R!Mle_h5#{!?kTIJ{&%^!C3GxQBdlosYP0w0 z86DZ~$(N&$sj)~VmuX!0T1&ke<(LHJ5(BlF3j5-T^MxzQni@pY3`L}v#PyFt&zl(! zc8}T83e&R(l8pEW7(SZQ<^?lL_3o6~{ZJx?C@mpkhNe~+OW@iKmTWP0GWMGx;JER1 zlH6#8y!7_;3qd?kx#vuAjr(el1$2K9JFHkf|5bAeVzMId(Z=UJqSzvTIBPr z3>HXQb0;N}v-Y7SP$ls?8yzP;TlE{6&6z1hUy}dG$_AI~UA9mMQDbQgDnlGr<`M`b zjX-%Eiy=;^h)i$3a9<-|5L{-T-Pz(gZ{1QCvFrCVE9t$XXR2p+SMlNRoCF5uX1C0@ zAgPr4o*j8}P-v}HBquC^5FT^Vd{{Fj7%h;J>l>ks!AJIK{>RJz8 zi&jQ+_#9v8zVf}8Jx|h*$*j?Q$UeUmrn`&e{N*}*v=GGQ*5CySDA$~zVRt@7H`Lbi z>7Mpk8(SC%LBjy8cSUJDJxePsZn*@N7*y3{rf-QN68P_T3WS45(=TXsv!2 z65SJAqh&2%ZKPcP_Sno?S3Wry#F?+inXAf}BFVicXfW<`wy9;f+#XvUOnL=q6)@_5 zJkvxkI3RttF(~f6PjuY3#%=`ljA4W(oPV zGHYhbRBLS?cGer-`Og61n6BnC#a$qmxt)i(TH^=%wA=0r^;S)d``h-3uOiP+Y;f1B z4~d>yA0r2c`>@OAK<1SYDJOvyEsTiNORHFWom`k1?E1Cac}cH>#JW4h(G)t0{B6Ab%oQ|y?k}{Qs#v+1*e8bmE+7&3KlbjTt#Z!se3|? zT|CdyUzl^~XUvVIRat4^aKdYh8j~6&cEZUR_YI6xb;9=SD|vb!#pt=G+Y#v9sgmB|n1jA^Z!}}51ZVelOXNjj|T#e7UWHw`(&nBx*|0vj7f50Z%+(RRScR~To>ogU(n*JW zeg38t%cum$h*SM`t3^x#GN;ADxqa=?!9RFs6n2@YCFSiiq3+rxh`jfweN?W{N22;lTqLqOng?qi*~ZG*hRP~qwM3#`U7 zj#W_m_=Jhd$kW+jHt)gQS>VpcHz1^5Ece7@HsZqCTIr&=t>=-s9* zeF$Gw*(CoADBYqEQy}Ci)$JH1*l-;>iRGerF)E5%>;=Z9)?ta)7*)<6ySC=zbgs+3 zD_9RIdN@Tz=@X={uSH?x zH1pCf9e6L+rp9s6Ia}l9Sa!_;IUP$e4nKbSQKNYv7t{X3U5?aZ)K^4~Ejh5xZ+R6= z_o*QS_!ou99qAmvqg)X|ify-%9HBvxsF6`OWsU0Qd8ZK`6 zTbMeDJsEvah&NUGU$X$Bk6m8eexLPW9*7^QivZo$MI3if8jbMRUhAjnQb$oX!NPUI z_h>CQOW(Lvod)}F?)2GKNrmo|mB0+lR&~QSjSM>dyg;{T3wYxOy3Z+&)HW=485w_N zE8eQ9R=U3BN3d}=(C$ihJNRnWdFMw82R^M4;&xQ?=f(M??V4g8c{Eq6YW!y`nNI?! zE9DD?+oJLjL0^qeoHsMQvJHAPvavF(S^Z%ByG96aUXRX3{}+%A{fEfCH7GND23RHi z$ssRY4DGCAU#feO=WYzA2?G-)B4QLu>V(hy3UnK7YwGMr>rl})Ku}4FhcwH0-S^Mf zh|&vr&m0;`EWy9LV_xUf?>gVcPnAznA#&r7&@~S0NWs@&|R&#Sz<8T*fV z&cEHSeg74c^8^03zduC;1+4mIdIH5QKc$=KN^RLij&R$D>#6Y1Nl4>s?YL+d3_ zV6@Wr=#=)98}#ZWGH(6GZ1nzZ`F~#gczAo3{hi_1t-E~nA$WH?4>EyEQ(-D}m)Ct>+s@g)Ans_C90P^-fiE&Kb_1&Iuo)b7(6 z=5hmXzMxloaR^^dSjX>x5;p@`Ikzfr3-h<9LZWw*TkDDq5^M1d)ZTk9P*9?3Y1t^3 z-iMJyz%`rlhLg5}QzwDgPk z#|e;1-(xmlE4)SXa9sZMOOXp8$C#}$xHvW`kQvLU@4RvUH%a|-r-Cqhce>p3_y2a` ze{ByQB4F>s6Z|Q){Lhn|s-Nou9xmtj|64cS9lrqfKF{$x$Uoi*F z(t-a%S*onB$4KMm8KvD!fllj9L<+Z~_Sa%%RVL%!2vZH@JWV2J8J^9f0gV6SCin5br&ks5Q7ny(vZ?`G)0K7hQZ3dy*JgB=xG(P1TKcuiWg)#wf<&0kL(WX z;IQ{cR>T@i9Xd$J)`Q{nY@MGy=!KkNE$$VF;d|TR78n=t@43CS#`vF4D=(N)frH`x zbbw(9P-CjxY=58@Ock<~X`=SUL3Khc9X#W!E8{oTx(3j4Y&7nS&PUrH(2GTMRCJbs zss;kmaWws1n51#77KvxU*w*B1@Lo%~m{>&H82sX86wX8G0ClVdC*Yme6XdKf(kB?P zjOM-9X|S%G3svd92>3A1qu|bCDS8?xwq3aHUGX@sR+Y@7QhU?11R$@NoDS?bNV^n> z)q9(ttujH||DRtT4{ldUZ>FVmtQq&&lf#FJ-##f<>I8BOppKK)hFYlGp z%r&&6Q+P~5TBoEA!J5k5msX@wccIlbiiF(G$~0U|-vSbKtf9vv8i?LFVj=nocccDMX^@9 zU7n|o=ZdRUJ2yH@U3ouSrQ{>~um7Skl)Z7g*ll_?yiP;_Yp^(uJ|@;47q#HBKVLNe zA8x6kuKm8&HB)5wcyctiJ~3=Rt_?;On%$A-jKl;9&nF&A3xBTE>`Fh z2lC+3)(&jd6SCFW8V}HPq(c;Vp6qiidA|uP{T@@bF&xmZ-G6+vOFYwQm@Jo-b*U+! z`^leZv~{ZWoI@5~ovW%nw)yAi5^Tdn3VLdi+=U5A?l+Qe3@3FNbxrv&xlFl`251$k z5){f#a&re=`0+GE9=?Pka4S3;-9k77cOBV(cto%A*NWv6ssyfE_Bs&!LsEQ%N&5F~ z$MPJF;IID-y3^4{soBrt)$AHW9z?T^26b=^2Y+9PaD))h)8hJTOfwKe1Yd5-V4R1_ zXU82BvUCs4o7On*guU6yjRK%`CUrmVS-P>5nb|7iC{e|9kHogw*NV%_H7OWQj~-ht zq+*NBB8-V#4K1;a+3r5_@F@!Yym@b_q z*OJ28nXKVeH_zxdOl$Q@Oc&|kEFVZ?XA##(G!{5kz*!(iyp{w(`^W`(CuKCew#bHC zin|WT=#u4lk26JpmRif}FaAj)=HbhV%l3Rcs+;3%!P1EOZ~$chF7^$!_g=P&LVCi* z?kcM`1@{6N?I^MkZr8DhAFWNvH0Z+iL-!MC05)CvE&s)xJ4fCf zy%*n-fJwWlkORLUEo0m>!P}hs*R9a`=Ur%Sk>Pnwf7be&z$0|In{0bNE{M^7>-%nj zFeS;(T-r(AU=aP=wG%d-;QKD=bip%f&#Ocf7PhrP58W(Op zUswZd^@_Z=k5J72SG*;!qHFn7Hx;nE53vDCM!D*J&3~$2gw7Ige2t>hw=9{Y4;)PT z$31p}H%4}mLUd%(2ifA4zA>(;)@<|>?64)xS!Rdt+5n~+RU?BV`ORok9D4VH7HC`P zZkI=PNYSU^NF@))3F=vIhxvEMDy9MhBy>sT$OE0e8o4F|G)xwo423GKUI~S+Rv3N+ zL!~bl{ln5D5oz!P^CM^dp^REK#s;OKooNcEJvEdLN+6EBe4I(D)al#CQVG3X4-;}A zjT2NmvzH+iZ+vh#_e2NU(cz34^KrN`ls{bf6~eioL$pz$+6m^m)4I5SUlAV>J;K6t zPBZNFU0;o-W3LXa12+Ha*P-<_JNEi??2!6u5S&^SAMzM!u?m>;_f#xk#-H7nhnqO)@zRk|f>&6oRZV{zG!|KNd!&pAl&fylFD#_U{WdzxStWcK8F0`YgmXBW!2<%rNs2834b zUf-TzE(@@O_wJd^%!>Gdf?T(vj|)Z$FK~S2mTdN)X^i(dS%ul6#AnyrN&97+LM99= z(_=lWYlpy)8LRVr!`(Mgq%uqUhq!P5r2*dORHMztjHHC;{IyhiSDf;#0s1ybI*_(9 zx@9!tXXvOkZC6&ID6W@O!GD4FNJ^E4G_NcQt1Zbwlfx>E2%?kwhTzhP8KLDB|Ha0_ zaw(TTpFPugqIm=oi!?t`EbMEh95k}{XNtghW_tzkNMGJXaoOFlzXykMQvIRb;z9J> zaeSptUuUJxhy0eCRVqmF;)8e#?7^QE%F7YiZqc1SK2dxsyPmE=U^3ZK%r4WLP#!Nk z>j+y{6RhIAoCWy$ov)@m=9y36rau);`weWw%e~n|h@7`TZxOMGf^e0S4b_M~(a1 zxVRlLNc9Ijo+|FC&_$T0bEU4++5`CF4H@v1Up`3-5&^^Iu;pL(?*6+;UXdfpPETiP zIV0$)q{p3By?YfL^NsT>L9^4z_uq>xQenab%HG9DQKXIp)N+kkmEgL{HlPij9u@vq z=;hC!#8KtCa(x(aBTHms2WQ&`oBF>$7fWR{`g!Q2n%pfUHp~iQGu?>m>S_kT6cgjV zc7n7o7ymL^jx@y{t!>@OxG1G_Au7(#YRdJQjmZJ>OzFl%YdD3sQGZa;f4C1|mZ8&D zt|Qs)B*Ok;z11ucsec>z_clmXC=K`68)$u1EYh5>9Qh|qlLiZrs`y(UEd%fZxpqFK zP|~#_J)Dacsuz1JmA@R+al3B(@@STmQvWK+J_4fLl6M-GH1s}-G#c!ZT;KbF!v@j)0)}NPQCC>U72~Wc&Y%j{3Sq=cV(Ajk{H0O^M|z~bTBO=70ge5 z$#`fn{Pcc2Ro!nmvbBQ1-#WX)?6Unar&jOsa5R@ascIxR0SSx`S7%THBJiq5m@IHi z&K~&5Pdw!z`|T1|v~O1g;OoSZ!lQ_lsGHJiT;j~l1P4BrON{kZ*ar;YjW;rjpN|`1j$L4mtID=NBspWqA#e}(a_sN8c z#HxO7NTbY>Gf>fbDuveRuCCb}w-Z3Vtu~ijJEk>r9-BEkUdwb_%RF^*zU!-gN-llmd~bk$G4k>#vH~y76Xh&1SvsFR>g!7=wfoW+ zRi8!xuX-Z| z;SG-mS`06Oy0SdkO70z7fq?xf{K>Z8Df>?+d_=)^saOdus&i=esouw*RGy5mlu`dEeRI)n+n;r$I+Q?_fG8*tYF73;Z>uY%d zbGbs3>;J6hyMTU?JUQasiT1eL9x>K&ByEP_s7*{ggH?XLCzJ*nE?M zk;9;zRpR!z$kJEzeew@+-n)kSxDOaa#&sHj$I&1|qZlfQoqHxVZ85X(T5(@HJ%q#9 z?hzo}PMPkf&a?M9b}UDB56y`|XjqMA;@bk+JiT_(SrKCOthhl>bO~m7Wy;mwIL_R| z3DyQy<)WrCiA}TGU2HHs#G{#d3nj4W4cK2v_`R>g7tt(#y-;XRF+ZLIivQPrwoH4Q zY+qPET0037g}zDPhqEP1dF+x2cNpzLVoJ!yu}ei}x@U3u5ebr(nWvR6sk!xKtVWIb zn7_8`5Qy;|DRB2260lW|3B1)NC{+3P;9YrWZI$t3+TsqQ%ed1jzv}n>;td5)%h{Yw z3-x!lO<`|Of}Q+v7~{Ke$W2mzS21oC>oEy$(m{o{2MHL>{R~GE=gp16q9#Z;fqhb0 z@w`tKY`jy#n-(9ZvA|Tpk|#-1O5lk)S;A+`(E_5&ZS!y%`wK?dp7U z7(0)h!Q++t22}?06x8J+O}l^j7-F!NHnm(ZZx|{rtc6hHle6E2i%*H|r9e6i8{Wuo z%kjT<)r(2d^7&IT0>2i*4~$X9 zcz4G}xm5C_=jBoN^iDbNM|qyz6-Rxesq?_`o1aVa5aX84B>i!Wx zmGRp+j@X>cRnC}XxwYWtJ=}0kS zv|1G7gmMdMyKYOaYu@qd-Ms0~ZqZ@I)SSE^e0xBdO#!Mi4w;*;dT#Y{%J$k`j%hiO z-eCq(kqEE9`2ApY3;saNVu~x{xxuGDGfa%;HW;TAEmH-Ggm&Ll+0KisFCf*e2%3jR z1(_{9r~i{y%w%UC%WSwozM1vSZ#I?gFYNM9G#~axXe6xMB2n!^nP#4?~C%SHazot zGVI={v~kS$FS*ixw&=&kfC^qT-!SZO{#m%!WA4%7^qp9+i|=ObcH=+{!g3+?0&Xag zk8_YkMCNYl0UoVMvcd^m+i8IvSYbewu8WI3Q-!%VV$^!?PmnU++8%JF6J6NmW(8Yl z0kxnZ^t>w`R6`{ubUO)#I^WrSwT)Sb`JBArJMAwXy#*#GBlA#lv5iTwuL6@QPpjjdoRYEqJ*XKW=%(eJeI|Dl1sWjY)Pm zEvznS*N)T?N^R=Ky?Ah% z^mZ3}?nTSu#`RbS(|yNus9P@S$V(tv5Nc=b9jy?j9V1-|BdC(v30EqIFTu_*N!7DH&p!-u!9DU|$O@DaZs4+l7bVDma%Qu+#_XAN^cM=mDXj zxYYSrP3m=qY1Z%Wt`4I{mX53zM++4SnI#xh4&Az0o6eooPqFCCEuU`Yon~`v^pkz% zZ1P}%I@CAPSLfy3Mke|ehCJRg!f*y~VB;E#?&D6wx zNrFh9uQfE@H?4p^d-6=p96n9Gc0Pn&kuO7QU)z0D1MQVbc2Ll74xD(3uV#^lK6E}h zR~_;7;h^iY+?bue9^*}?YNbA2KTbC~3UhCDwk0^+E8expo#$+mH1_0mu9VcmJ~P)y zWNBz|Tew3m1Ssi)s&xuUm$;OoS!*6LXTZ?6W>w%7PCGfpGw4Ku?__#!QzNUk#cE<` z5)_KiN9f5<{fk~9mRU3Tqw=l9AvVBatJiwAUz0Ob68Rj~5WGN*tItp@h4Pg-(xh?s ze%#Hs+RRWV5{Ad-;JyB#SfD;%F$(w-t)vh07)6tDG9uEfbvJ+7vWN~47RFAs6OTJy zXB+tFcnC>BJ(SLSSyS`XL1|6^&6}266tGnTDWCaVu3QzCoL2S5UWAAHwNjCKrdiR& zbWuTSuDAul-yJJ?XRgNuJen&)BJ9jHr5{I@oc~f(YSp`!nPHXr=`|uS*uW_lv45o@6b|&(&ls;QA@w`x1q&!8sN^1zA0ZEJ0wnmQ`Ou8Z6 znZX*NvA~6Dg?!g)?V2DQ>LM&g$-O{l8gzPbBXBO_?yw)y(g`yRElI zM)YQf>UU2BfhxU~wknlw^9U)B4~6^PPbBtbiw(XzAEMbic*xxd5yHJb6iav*o*!IG z4PME6aPP(4EHyN^Z!~%SOdlYykNn)BEKWC*OKqpr z?P#qwT9yBpZP ziKtS!mJV&OW<^`n-b+c#>xO)a1Y{FAU8cX3&TBC?^A6juNOgRI!ZVZ{@zINzDF9j> zfaDt-H;HLyGLlk*7TtwWzP#UoZ`1WYa&?+|SpXJJ(D0qbSBm#9ZZ_BJH}8nUs0AC7 zG|Bt?V8y&($CLPKIl{@z?Y9nBeI0pj2+FCj4x%Z;i9R(Db$*u`@9m%PE-y7n(WHe; zFWunT%t*dBVW{uRi%ofpW(-K}Q)0Uo`P_%ya3y#m|F&>=4py36x>CuEuMij!#t||; z7-9=P1A>e)k|PG)myXj z1gAv9+$M2vuH1kl0XsiJ9hQnI2UC3eiqtC1QwJi0FnL#@?@!%5_tZG$@Ys&${VW)Q z2(R#TUWq#;`v$BI32C~`1mK_;Q8ns2x$Qg1{5kR0n^YW{u||G`fDfHQFk}&Iut?#H z1-(>*=@2nOV&Cs$1@>B#R%)WY8GQ+$$c5OC_dnjSvM8C^gvJuxYo5I^RU$=V?ceZ^ z5$})W#trhjp7kQ&wc^yqZ-V_P@%ynv#sFP~*wqoQ#Y?UY%ojcF`SU&!AFN#_-n_3M zQB$T-_g9&4S4JI9k7VE^i6&^l4GCGys0 zYouq$Sks!d!Tzr7D=vmDE@!x=>@vEy3o=_QvuXE~eN%%sMSyI)RMvv%Gzd3^kh8^YMp-O({m3Z>+*{^>BN;rI!9CnfC<9}~aePZL** zcQJq^M1J8&{vIKR&6_Yba>b3Dis!eT>6ISR&b4up%=$1`!6_&Qj>nb+Z=&b1EWAod zGFvsVA}1+l;FJ->&L4(1%c#bEW%W3dg936?nWTCDU6ck>0RBS1L^i^VRLLnHF{v>7EkZz?nnM+>jCv(?N6}QP>8^gw=1wsP~7R$lpq=>Y2>5~E% zNaPY4Z=hM(L-OD42y|p>lrf=ELJ^pe9CYb~aUM&;!8=jQ>Ie(3UfQmDzJHoHc@({H z7OEN&KX1n8ZPv}+O{6tkMZBXi#e~zRYcUPr4NGtGTaIl2n2oPU=iQJ8;w9Y+(p0R} z7Y42DLj)ReK9G%Hy8UiPp#M@XyLLXSuw4qrzQm?(bP5==b~9KuLauN%iG;M1Wo-J79@`PakOnJ-~z)J5>jMc29pSJymnOt0UHgm^#@-{3ygk-(_l7 zzt9CuKSa0W^TS{ea~~4yB;`Q=%;svT_z6K=0#7S>Db9f|7MnO#AVo%2FZ-!UAguJ+VPs&*|fUVho1moH=5P*2~~ew?_;8`D>O zoo}H+BIPWvu(uywhNq&Vh3lTcFnD1X-so+Z*c}8hoBLq!^!Sj+wU~eI`t8OUp z9UB#X&l@GwUse5r?OLk1y4VR56$mRI%@>ZjdE!v-kIOXLbb&>sT=$wME#>m5wXS#u zquz?V8It~!!*f-a#Zn3D?W7zL9W3Y6(a&^+Eu1dwL(ObE9*kg)^tAf>Qi!G+#}j0W zXse7fhdwqurR9719e2?{cnTzIByz0s!|?%Ha;?Pq3vqu%ZplmzmeFZC4&2ZU{Olk1 z569cA3DsL(bM<8<3}}cJ;&FPs$h75X;u(7N_4$ z%-C@5c{*QmZxxgH^>DeEsa6zHRS;c%8{CK$j7J^uxMA0(vpcb;U0+WX4=3WjIJu^M zS~@4Gv`eJz8ruQ5%s(c9nm9rO(&8A`Fi8_ecV!`gz+Xq~JxRK;MCO2oldzjd-G@{{b?~Ya)ezm8PvG4akJmJw)N7a z@}R;!n|ik9c)`fRN+-|+3<_jDe~mG|gN0G7{$r1J+Mvd__Qz1^J51fLHu7Edapxo5 zKLjUw8%!8P)e)dT)8?T2@vo%6e9bKglpROdX#GVn1c`r+KP}oJB_sb3uG5zp%i8Hk4qUmj8w56cX^ugL5wn5?^6{8$9 z8u?t)&XI$(EZp*16s(Nqaf&Arij%u9#_gIS-ORWhW6dW6{Wq}9nSp9eBw-@x>^_4)BRSve?6^sS37B8Wa2hU{QUxl+3%lr21 zJM{|~ASnFHz!0#j#9HRNf@~i(YwyS~i>%5;hiJiuOQDU{hMZ`2AiK74v{lRd-VmOUqSZLB|!@Po!sTwx0}P*ht7S86%BNZ zk$mQ8-&C3~P-IVjZp>JplNb+l2kw1xVI%3&ODg`Lf5fZ7-seJHNF-pNJGVJj>zN#df5)x9lV*^GW~M58Fl{=UWSkRIn(vvHB=$i&+$Tpj-Xrv>E>)}(mwi@gt6*r-T}sTioox~Jp{7R9^IeZ$%KK@ z7uwPT(mPmUR>lQM;3Ke~-dN#APLCiiCdbqwFzU~ulpnp%kabV@g~oQ~H`{KfLE*(= z3hL!KUHea>-lQ$*%@EEwpTkW3=J55~10w@!^)7E#s_#t~k!U%4qrv)VDUI=kc}ro| z?+^>K7ZJwq*Z9aHd=Ub0WAHj+uAANN5HQMbi9A%2X|#ibDkX&lJU%{W$N&1|fMAZe zv{Vo#X%6ikLr_JPO1?ZLeMWFTdzJk)TUpvWQQbgDSi9rL80px7FMn?~>{5>W2K4p0 zXBkF+OnH!awQ^#ye>#WXZgh|C@u=KZb4~NL-8GSBXhTSAxqF5`y==jUrJXDE?}7qy zN0C!zpJTu9DsL|x#PVsOybAtdcv~Ap9R2!Nb~SR8 zlHjjz=&xAwk{qWK!>ofmBGitxBA7;sxtP5K z>GN&dG$QSGxl-Izai!6P12_CESc7d`pX3Fa7%t2j4SCN_JVDoQC3r@QO^f6Z>; z4LMruUXAkyJYOJvT11;^vFFap&X&zp7Q}C)7zDsT~&?Q zXPF>eMX*!BK03pc?%97;!}g|m(t8b+d`t{A3f|JK^8KbwRRTN5J^PZdjq@+&?P(vC zVNK<;O{snayZ!V_f>rAD1E1Q#gZ(fFDM!Y>sHZ^{T7aicCCX+8j2sOM%CHv18()-R zWzyFoVh5fV>8lw5oelxWh)xLwU?`x!)k*a8I4H`hsjAa%x~jo!$zA(^(KtQ9?GKTX z$I>n;bLx6((Dg67wOhywiaSuJ(Q9#ET)Vct;Ge|)_zH+A1-#z8sQPv$2ll0(nk2lk zG@Si0E&i_INu4AxNUg9zuIqb5JxetdUg~=TWekH*!uN`Jx#f>hbXsqJ*P^QUFnWIN z+HKI?zufB_d9U)a@(@$yrFE`MFO+XCu$cdztPBo-9>WQrW)pYR*Xjp$t_ac&P3JOE zd9GK%_t!_NTvrn8KVqq3#^Sqf@)cl6m@L(+2k}63++P%g{H(K=yi>>ndyf}Yj!3ui z`h;mpx37g-))Lrq-bxSegqPp3dpAT%tQBY1c%b2h!!l8L)9)+m-iGUZ2A}GaUu}m3 z&Vg_E;l#yx37Z#HH2S-`g_8X-Et;%&ToB{e9^bNK<&62N_8rUsFVAKm;zlrdZ5hJM z_&Bg?#_|$b@E^IuH|9&@Q=FU(2lG%=+q1{EtFSK})4V8KWn3Yk)^xBEH(U3OMIFw^ z!>!@e`RlFqdf9w_+Zbi#rMN=TBld5*YoG5~yjLmcA;%ofuzN*q7%FaKZ^NMsWpoML zmC}Jj(T}sUYl{Se8^AIQ;s;44{O`atm$mroCypd@43qKC^h*QslKl=P3M!QfmD_s&x(Z zWtP|_D>1vP?wWr~*K=un(@*Fn-7>c;xTYrQrj}PjNkr1o%B?Yaal zfFz1?pp?nL@T+5@CQ#5u>sB6D!cB!s&@do0tlNv2F#qx!@>M)%ev`|9CO@J7h^CzK z)WX7qo>Pe!j9N_lAsBb`DsjuHWZhPpG(uh3hI*IMk&lR^TnC!$fP?s@tTuIwfpz$@ zWJ6PH5x^Y3A-P|uv6w+$7||%pgl5jbKbzZ6LMz7SZ;kA%|6-qPi`B2ijTqs$=U1Gx zUkG{+?xG@3~7@VAs&wuZL-Ky`5c9{8s|O z?NkGKzlfb&Zu#_;duU|v#82ykOW2!&gKv)BX=Swbri2r6WFRSJ$<+v{HSNd7=hPb* ztnRKCdm@V`M80%&B%<=<-vF8F#At9GIi_&iqCIHWx;rPbf8s5gGWD8myZ73T!3rCU z2NDbcKDs^Wuj76N<3I~i>eZ?6>diOx>$CUr#=A3T+^o|WUo*RZ#)SzbZnsoYD4 zzFBvR30LM(3dPz?RDCYvSG4%WyMjmx9UxJDHiMYT^?YDwQI9pN-CHhcyvvXUzvwtU zoz6TMz47_1+IEE0f0^U|*gUr@Y>H4q-vZp}!MEf{ z-Sec~S#?!Av6Hp-tUuCu0NRMl6?yN;;-qkaW_9bZ#&M6G+e2S+gUwA=D|_o?!G!7s zGxE);KaQGHBUp;O%q83Qs`dIBXs~^k0?Nr?z_M13YGJhFN7ETj{p{@!zOWo{&=;p< z7kmMqWlem~GRH;}CcSt^d^(3gmv9!$U@^Xj(EK>8VyewKiI?rNnu7g-C#Rl)*LLp% zL17Y#k_B7)Od+EEQm6&$wa5AE`^z)O{@e4q;b;)+K2K0RFJ8W|Dd|}-YWX_Krrh~m zR_gVA$4}MzQyuycNoUj5wGSs3iZN7LV+Ca+6O^9WopvabR+hY&*V%j9UEQ?W%C}Cg5lLKh|1vfZfsUBP6ldk+f`av z%_8(wUIqq6#}xmZu4Xfs9K4u=wtuKM4|Kb(nI^Sk{pfXkFZ(*jw>LuFK-SQSiVzi9 z0>CJR5y*_lDdVii*hyJa=j#n>p$Wma>oY?`hGe?B(Y*cP)U*(dewq?pz1p$R)SWrOQv_CCrt)s+ZBXlm*5J1x+NfPlds=81~`avroK+{c75 zR;7i7PNj%G&tCe|lDq_YZuOiSqa6nM&0;u(p56WXjhzHlRA= zZ_-TF&`nKm7Eg^o1n3^7bN=NH_g0WkmdCb*m>JB@}=iAf7Z`6G#`1?VVW0ICk8)d8^!z|@7# zW?s)JpW!E4j*Xq0Q`vKu8*pm9Ne@NHwT=!RN5z1-{;SvoVr2tA)>L$k2?^<2ncgfJ z&HVROQMe~i7Ov&W%+GF$KdLSzd*`IP#lm72#yua~IcXGo;C1eQymYP8>y}wwn<`b^ zg#ipPz@a#790N<2A;IOW5IVKC za=3h(H7@z^Ru@XYzSG3qQ;qDRW{c4KRh1umMYTAYMt2O2WzFYzMyICuFdNxp6NBz7 zmZu*$sK*+mI`q{?W828d7W^X1G#kA{lWIG942(ah-C9~96~7T*5c?E)DbwQRYwHO; z1D~j3;r>*m>ws@#mQb_1V!y)7+%ib)99Kg|E!teR-JKAtTK*m|S-Zy$Wh@Pij|!*C z z)(a8Wf$BE2D|fet3h)`ugRST<(Ld!nuWVQ?RvpjkEFlRe5S>&BfhrUEA5OuRSiXc!zlUrVZN zBd#`|+bMB)XT{V0d9#dxVXk(cy@LVc%ysxEx;8+o#89pJ+j>B3iYi%urSVN0he(mC zIrDQyV1^bmswv`ub6bWA@$3HfIg(MMoAX2!!6Y}Gu^T8Wz2g-Cl3SmFk_K(TWXUZo&zf~o3 z)=|=6SuHVyoH%6zW)GZF_MAO-(@z9Pa|2w0vo-Yw)u*4Ff4_6ER1Z$-FB%jZu8N9& zd|$Q0>A4j{u#^^?!U$SVNvPG5DBT!N;qoiAoigVa>2@x$MQLz$Dw@e%k5O9wa_+wA z&fN^Wclk=$NHm(kevjWvjV~~Yr=j2*AV&i55&o4E%5;CrV-77f;U_jQ7madhwsr%n z@B;<-JTFK*lscJO!t|YZz6=30>l~q$di^9DzS5#cRJ0P>NQ!^tcctdv z@XiQM=f^x?er*9?$-wl^Du`;|sj8Wmv$b7Kg%1Z2eQK#FbM z8jV(T_6loKz`iqAl8-T)^|_8n7Qyo34OE<2|1Dsdx;ckC=F*4UB0WKW1|Um z!2Gf4N-%|bR1 zLA}QpKR%o0myoYf8ekojkTLZd)BDk;k%#{ff-5}BAo1D!)J*Ygj%!GWlx@&d6iPOd zlt5mOBQ>2Pyl$BW4jdftl@7(w4D7PPp>(z~?V!Z|)SK!n9scrsf}g*4m-oMZ?J@@H z_^wxnX0)Fr{eHeeR9o8H_xeN&`G64*6${hoOZ;J;-8zGzs zy9n)I8QT#?=3UV(gbH_WP7-S*9Xjg)iNczi+!?sW{ix|XCqak3`REbYCMH&b5(gLu zLn)Y=g5ji*)M5&6c#7kUGAE`NsMs43Zv~JK=L54vFW0Uk1wU0dCI@jhs*Nxf2O?c+ zolr4x9mB`r1Gz&E?ej*$$}P#|^*A!RZ_=$penEnWK9*Th)lREaRu$G|X*H%cEHTWP z|JKja!QqwTq{Q&ZvY{~LeIlSjV(n_MhPG{x>b#X-zST~dPw&KLf`D#{-R$Tf!o^DA zS1p9zmS`4pC7|J3m6i(T@!MKS>s7Z;IBaIYnS|l&RTpYM78EGMcJ?_Ptu$iE-3&NI ze%ZfgV0?TV7axq0w01>|OA(}Wv2>6OU3yzv(ijfG&h1DS6N`!QW9xgLAamt>zvEHd zy1tCZ>%wR93N~@C2ei%ooE4D?c?sAH7$(R7Dn<1o2q|!_lG=?(g)0b&jVqBKICz5a zLD`0c>GwBfj;m4y`6UAiV0{Ltn!}9L{)vvX@3HM*K05ZH> zZm40}dyZOTGGtRt^LJ}TE25fmU~y4Zes_VSXlWtLCSR28{4^XnguBh2Mf44ZByTw3 zv^30?4!4cX@-)F&xIVUm4j1Z&PK8b>|MF0xH_ndxGzg--!8(ChyHkL>#kIaRFXO*9 zj33Ph28qRa)}i_O1Q?C{DQjZ||Xf72V(Q z{k9i9ZN&B=EY9~-LYj~tF5xxbtu#%Mvj@j)S1Y0#_bN##i08DMdW%sT+)kS3?#JPs zV=p-0BXm-JJDbufnYX9$sJq2SKEAZ`*O6xbv3Vv4OalS=TbeVGG6RTWYGOLWv z;EVie-HwipXVR4f=xky#pH1lo(g3rnYMO8XuNOdzQB5}RNV0k#d^V#Oojbkw36;bw ztI&WPo-{4LOEa2azRs`GSW*&lu21eL0M4Q6drhd%>AD37FM@CHozrF4^vU7M_VMbI zcIT{7-%cW$$V{5j^cLKAIy*AFay{`pvJC2B$8 zewU)DB16_U-*0a!ZBUgNlFJ1H#6rwxP3tqNgrsR^akCX~XvA5osoB}6-L9Yn=7JqT z&mmMYN0^~lLO64Nm>yp^8a9e1|C4A1k(!Acl*q9ndrA#>UVs!bvmTg2E zdm|e|@KsvjOBE~;dTDfJ3fhf{VMCOd0b=hOA^>R6VWun$mEsxiz%FwFwcMam+*cue zJtQzz;VRU<|CyT5J9>Ijc8evH{nbVoE<1Je1GPaUhrTak`ZbAxqONPJr0yu;Q;u;w4SL>0hh=NIWztDch{yh`v z<1+Y5SYn6y55)-xSb9aa1x4N9QBbS~@6Jq+T&pSb3LcR34ZiBnP`)4Agv1XC^3Lrc zfT`&)I}o?-zn3PfkT&q?VJy46`TiYoWh9YzE!0D1En4s8x6RX6U(#ydv}n6%)a#TA zS%?DJDxMiun{@c2Rpz%b`8jpB$Mgy7`-0tN==DWZBjNopRg3l7EmZQOYm%82oY@() z^Rtt-ujFUz^LIRiJgT;I#lCE4a9O`Nt0L&ZXD!w6X`9bc zmu(SN4A3%kbiLv1~xqlIJmftDyo754n2^ z2VDCd|5%P4L=oe1B(E!oW;z|lnd{T-+`aqMh@A1$^+{s075P24BKQ_`o*AfgQyLSK z26A5|AF~GR_|;By%;1|>WTh?DXFS&`UMV_{OijT^f=!J5o9oE}>k^~8-j;Obxd$}| z()e%DUHg*7n_YuFBA@SIOa{yMfHc-)XH<5&t0K~4h3C%xoqyXLjIJnIK+fu;-|2Hc zz6@UwJZ?&SXWG9(Ay;JqvMh^pAM)$nM z&Nb}({1;7VhrJJPNX3*U6~Cz}ac!!g1&*5LCMu54=i`!lI(# zx;faHqbu5V9$ULQ^|%b7>L&Jbt6L( zp#)aNV7}WYJt{X`kgyzn%Jwq}Dz*oxf^gvLd+DGLUo`zwpkZW5X73&1)#^^xtsH?< zW`hOJKpGH{w4K@GX4T=+TfM6a-Nz7K1P+9SR(g*0&|`PB;9}_#As8ck$GeMItxqQn zG}M7LfJd9<2u#bl?U5yy$g@s*O<^xyt(Np+oQzL{&{I7DeCohnZUy>_`xj}S9IR#C zc%N0>q?@qRgF<`iK$BIc)?lIZ?LhV<^+iPbM2>B;`XbeJivb+K3qnb}nje>XNf)oP zp5=De!F~&#NVC_)VYWWBH9v6t*f0SpLsau4PQ&$}s=7S7%E+9Y@AG4WGE&M1kC3gB z)-QeYbnJ@XhUablajs?k3Tuu=C_hlM@9oS`>e-o%)6p+F)YZ-KycQBf2&fSP0mMh~ z37eU$`WlChPDDihOTMJ@u*a51#&@-a z9(B!SH-h?u&7Ec{@J>;xGc>_t^vr_N#}D)^rRovwKPy5$xNN!GRLdmb%#BH@xU}7v zaRDo29$xskHV{uj^c}@p4+KQjQH2Tbd*FT4zSpZJL_NytRNpkPLehhDv+d4eslXgyML zJ5YI%h+9em3XE?#LSg{Mp4=xVG;IpdA$3q`h#!^n7WlM9rd2F(H~Djz`!{LwBi~wF z22c%kD+kOVsMfQ&t5tWd-S0K%DpxQ%J{F|kI;|&gxCT4UnQ62kHPu;m<*X>>?iAQq zVETG03zjY0iH+9U5+0P}UrgioP?D<7(5k(e$AsRH@);;4>(1>^ zRi_RhQ|~*2=B>>(K-!wfV@1qRhbPW{5~Eo00T^k%T)p(VBM?%%?0pF{J@nlYq*-+Z z3hzrM9l14+m8;Jn-8GxBC0A}yAw2d=N;=@v8ocv_pTkl$EilG^uu!4HceElWFc^%{ zy-%wKu^2FE@@N{^cCQ)`j7Kj-cKP-7$i(Vd?Bg{By7!|OJE5g3lC$;tdnM4d->AV{ z$ZvVHq^|tQS(}lpbwon32P@IMF3A{4Nc2f=3Z(T#=sryUR=FDD_f-5=tN4v6Hl!{k zfczfLmHvX$xYYNziijs~!j7hFW+YBT=V2*J>ZrtC_gj9%?;TP>burBh;fC%j_&c0G7O2lw22YoAA5368%`ZoUmp|N+ zh9>;p(=rV81R$%?e0!dfrjKE1Sp*}<8o*H-Nyq&Oj-8b!8{Xzt@G zFyo9Oy+Wii4nIGdu{*+r)Zqw+?CW6s*d`*fn-Lyd4Ny``+!-(UEXsM;jzen(u!#`n zXDD+ApXJ9e&$q67s$Xh8+$?+!TF}L8wf10Rp+0L?OwxX=TEocFv)t%J$I6mQ32rQb z+=Hd|rTXmhZu*d0Fmqla1>F#2#GX0Q%aCR%K&Gv%SMw^&Am?1i=}ht^H(!P$BMRK8 z0SiZ_!o@lK9^yb@5uZ57r9CDhwg=q}&#$bmC~BAqN*@CR1(SkNKM|gq{Iy}%^bI20S-s;BAZGskWs4; zN_V9aD&E9Oo#(f|H=sI?P-wGrDV@nVdGcit{yTps()P25OIex%MZ_Be5~{Bw`=G|z zU{@dm#tO1E(Vy!tcit2=HWGxmW(wL7!E?~6!){x$c`s|xNn5%#6I%LiVq1F+q|`d1 z3<=T(LnS#`OX^5CEPf8rNZ5 zus=m-ihVq?%Ttv>DPKhXSgsCBC$QL6g+D510a2=aH8s(f(2J7a62^FSBfA2o_cHH| zzkU2O5PkpC@RasUg}AfB*egb@LdQMv zg1Snu3j#F~TQR`!rxm{m1}FI8Nc8Zth zfP~nRE@s*O_#9TlUD^*dDwaIq>Aq3gjooQ2xiz%K4Wg|uaq>DkBCjae)5_zbN3(>Q zfaE7cw!-5+bCcA@r~!mN^;4H35%@pIg)(@3R&d+yL&hpc^>Z194D7S0hbF^9D8K-5w2K zftzR6Gf$TD9%$It>C6FG{d-&9K61`RDoUUm?5C;L|4|rgo|aVhKN%~p=CFOM27d^v(fTd@)9RXuIam7b&!q1RgD_L@6H~n0zR?5dttb21uDR#=mgzH#*WsH>4AOdh+}gd6UtAob%cNIQ2~-3z1@kLwVe$wp!H_s zyar?Qni3qFUlStP>?^`8$~1f_n6-H%VmamtyGMn_>NE(ld7a|Y zA}>mR$@Kaq5#;+Wc0d+Ash2|H08W3a}LbS3=T?%6c)85Xkd+MSLAAXHk9z|br>gWvE-0dYxrwBjM#;Dg?rrYpA% zS2ESxxh10QD^?E+KHEccd4&`#G3VN^?5-spj_;Fh@msMc)w{>1=kCQ<%3it|4L;D` zMU#wNM?$@{50gS*70ph@k0D!hC$FoZhapd)TPhwsFBe|>hfFC>HG*CV(lmz6&Jg7r z)x>W3?N7R{_S-Q-28X2CoS9TFeeDwpsNBe}(tyA&QAfVW2G1(hwVec&I5X>*>+{5P%cLWCSlfU)FMfx&-l#;SpM z41JDJ_)HjAr=X(j!Rmnhs4(f>#=Q$Eb%WFBr>wzKdTsK+-yLCbfB)XrQua&8L>tYL z4?ug$k}T0RmMIAYSh=R=j}6;z0MMe=UECV}1y+nm)jaEVW*-#D4^XfapapMob6Ys| ztGbLP$s%>%%x{Ka8ml#7P*Ip3Ou5Jxn@k0m4N&Nb?oZ?r<o%&pg8 zR}75bGw9n0CHV+^?lTGEv!miZ+ib)_8MJg}(miA`VMo1-R6M#s-j5xO0D4w90GIcG zDEB^<h2PJdLG^%kRpy`9m``=DNZ$cV@=%Ik0|z)W}$XU zd~)R&oLUvRRcK~)BBersl3#I@N@+th!1FKqT%j#jjIU?G8S@`^2NWm)nP_50SD^o# z`{N$q2=zrkkm5Ibnw4U>Za}=)W6Wj>bfEgI0hrlQCXi*8$r;(QclZNmu4-*K&xA_+ zFd5)qf<&=eW+x&w9vw}QRpj!q-G-S&vPRo|Zh$tuExpi)ux24s+=}U(_Ek){; z(x?wEVztQ3LHgmwX-+x|bcmI&50*N&=Fw>gAEzEu;KUB*juyx2h*MIRaL3Igt1J1u zK}}KRt}mk++;aBG-K%49QW#2Ap*p=EFX@$`=tD2FD=jeEqfzofGno*8x^La+bK+#Y zMXy$DU%9w-rlQ{8NCXb%sV%qA?A$z#9H^hg{hO)$HwOGBnjEUPQ0;^P#G*9WxMAS5 z>$S>!pQ1U{t4!Axq8j^?%7~u~HaSoV#5H~Hq^1`eIlkx?Satx&$LT%+!70RVx7x5L`A%d_(lO{?q29lJ2;!jh zRWdnR>aqGRT#M~x+yYHI`a@=91i)Qyl(wJGVye_(N!T8$3C*n}f#!{7M@e|KN>2~( zW5lzaZkK;`i?5y$0YpVH-yOj?yjHSwuQmHBZOeq9!fwEM$J*AZSopt)=}$!PJB9-8 z6!86Y7dVsOOHTW%)c)ZbI7XI2o_siv`Cs4fxA#sJ@#L#PUaS0#w(yU^>-DBr4wQ8M zmt;iKe|XUE8V!8K76ow1$vX^Y{&DDk+!RH&5^aS3m;dFd2Syf(=T(BO$flT70}MV@ zOndXP25@n)a=qm;X8c_|1iLx{G`%A z_se6Msd)?qn%!aMh@X!U90<_*G=u}=1X+|XUG={4`Xn$7`-W(pFw|rHQS@WJ>ONs; zD!adWg%kebx2>+0H!HSP-_Mx`vH-^Zmo#`kle~3T3Rf-TXNx}gJ0^~v>YR!*WvZ&fZi>+N zFHGRFX6cO&+OokTiO7Mh2a`hp-C6J6sC1kaQZ}@MH~)cW_*l@nee0QBvN3GgskNUo;(L0muoV1D{fJz~dV>&VR9HPgt)* z7WSck)vEtMC9jVpz*07>jt#~CG8uxmY9L>c?mO+3e_N~gi_-mV#yvN{avxieupMwa zcL|rwu_Pb$*st-}ePn!^Ru8_0lp8L*P2ms{%8Y!o2|tvkpC+Z{1kE#a4<9mf#F$K^ zc5T+TwGmWk(DAFm$^7=tF>%oGOdHHt=1Kc;Hu-;e3RuWP4qGg7*2CIQ#Z<3_-wPl`=TBg-r&dDmgr^J zB22`{F@5ZsnB)p9Fn-}?s{J3L6UhT>tSf+EC&B$>^LAppv?nADD_iC0T#?B>C;Vd{ zej)3GN(L8RzY2O9$>CJh`?s!(N0;Z*`5U@%&P)|Bb#jJslJVP6C;-MD5sw@la zQ)Bta=mD#Nfb^q_?_VcFgbQS90L7~*1qKcmMylmYm7~%#*!uJ7xz&sXcn?b6h=$*K=I4Bc%=#$5Ln^xPHSb>yDzIrDJI7oC z0?WVXAdwAeJI90g*EaSHEjnTTbr5}lU={+s+Ryt(EOI(q|j^qhRh1pJ`jhg!^40#Pm#%42G>B4jUaUaNa_Eq8(smT1Ru5V9yu#VJ5%}M zF;W2Uj>n{dkoqQ0s_J`7DMzd)e#?q6I4AR=C8;?q59sn2jOUoRt%uPbHPNa!srlJ)C``Xq`+ zX6rJT3(pSqK^7Cn`#tA=TW0>7#s5{vyz8$qZzSNy|L-IJdglw-&tLcj?`Ku$AQ^{! zY=r01fVVl+)gk*IgM3HG8?yiY{5zZEzX<-f3F#{a24FNzK4|yXu8jXXUfy?vzyL5e zKU=@9;r{a{QUCg8NiR2y4QzcjK z0YT~mh`jHyyt&KX_e08U;rth!6AU8n&bR3mGW6NqqUg3XBjBbktUaGw@OUa~S*pVh zo>*ZUDR7o^K=#(&Wm}JeT3NRpoHA?`#qs+k8{m{?s8AU?Wjc4qKVe?07{as#y0pLT zPw}{v{5e7FWUl-hOX*dQ2;>m*Rjb}o&3sod$#qOrF+)Tais;j&-v0LcIP{Qrr`v1_ zX}KFdx85`cA$x>uFT71pd82)7!4%t539WT42>!B(byUuZ*-+6FIDzG&eqmLaxy*TI} zW&_>PeG%TtSJUZMl|>e5;I|*e(-MGta6@vSSMIJzXqKe*hO4JIAmFCc14h>Aa&&g4 z1UAd&fxS7rQf^1v;igyq$qcSeG+$TI_OnPJ_{AT|(*LnK*$CdgTWJ%mwSh;&87_Fj67UsP4C*wM(WgrlmgTSVAgyciyv4PE92B z(%w2I(5;E`o;hZqL*{%qVSis{HIYJ5K;rhd9<9H51HkzqR8FI0%qPxpp$_nOc#y<4 zk7#^e#)a@~vPPS-M%}oHk#^b6g9qI6M6Zp=G|hR5Tt{^`Hj!}aUU8S%6ibEWvA(2# z+`p-Y2tgbqSjh_14EC5}i|8$UTQ+&r#V%!$>joL2kHpLYD?&Y?-Ax!%&nEj7l>Xi@ ziX494g-fg>CsIN$V}p>;=hmRnDH_WpbGtMii#++Xf@Qipd=9t_WFth(Dj2tm2g3E?;^8TkkSjB)q^g~5z56%naGZJk0Jy{P z;q4MmZ_n?m|7S2?q3Sgw_)h(O`4|0*m%X1f)Kp4%W-^icoE}3J(zn76=W2KS$gKt! zCKyv@_9EZ#d*pNmM9;?BA~=SW<2ROm^MKTB!#y81qu*nd5gQ86;8P^WSs-`USpI94 z|2GxzWP&mt+!c6tTJMSqn4QX$m40W}s$Fdgld4@uitT4lw5h%I4{R(@TBrfNO5Gcx z6#c9hLAiFt!Aqq}yscrMO3p*EM`048gu#>B{n**Sr3DnKvyd@5K=1Xd9CEE{oI`0QjYLvvF3YPjZy zh<`^Zeeb~vCrz|#-s4+pU9{9m(S0pUoPkRabb&~#WF+p?8tUB zXnH#LHb?WHsK{$b;OE07_~x7oNk6Gy@m%;4y=|9<68h#+HH3xI)QOlq5me$Q*(k{K zLvFCL(~v-DI0mB&qNi6!@tiK|%_MdS8+A`iHoDw-%5C$OmG3jll1{V8^`BJ%L9% z1Hh*VSF6*lus&jRy*w7bHNK>W$_fb&@A{68&l)l*smkTW%7YQ3?S}TM(wF-vGReun2Mjsycl8tJ=1?tMCMgwiuOKr_HUX zdQAezlx5d;z-%!+0SNT|G*Ul!Tl-v8nHz$7Y!0Ci*BSW-&^r+~2nKw4{(1G|A08Db zK&-7W4H4PZ;en3XtS(TI2-A>-J%CM+M@;LLsZ{X7OgWq^wtMsB9yBv<73n@+de|99 zCZ@~z*R~xjDrAmYxHNWZssz8&$+ndQa=b=&&fI>3h!*gKT{Skfr)M?Ngk58>LykaG zzYAijvYJl?ZNWP>KWq_6Ek2cJ7gWMQ&sVCcm}&`29H(M>zRNQfkBF z73fyHzdvo~N@Q~;*=Mx~lPz2_+Au7vJw^yQ^c^?CZ}y%Vd>QMGE1Z^zpP5))UpG->^QE_*~k_OhDRs{RHR3pRzAAQ0k| zAKC}c=2Jfp!?$*}9v@j1>L>toC4C7HYKJr#zN`LFcBhfw^Z0Sfw`Hjf(mBdsX*&oTXm@ zdC!Vd#}6?ZL4fZB0GqGfrKA5D)?Ty7DahHE680jQKHr0uE%PSgmU=Of#*JeTm#riO z{&;2Csbp9N`~41@3Pn0b8^qn4>iI6Bd&g~66eXZ5G;$3p(C;$u1h#9XDmSs@M>kcY z$QdJ>t9G5HR;}3$zfJQ^5G8j_;E2NpOIdhu@Z@?8rJIaRdPVDd*jg_&q|=;T)6u&-@#$sjG0jT713xDodj* zfMBZJdTUv%Qi0&&VBX%WA~%-1!PJQR8%OewT#;9i2uSCOFRL3A|4f4jyab@Sx)vw5 zkb3lvjEdpKWv}Jzj3`NlY{r&}g$L z#aFDZzMDTMB4AKnX?@&=yvPmFYjZ`HiJBRj`hxe#cbobAU?VR5X!!0gF$$Otuc6m+ zjG+r~uI6855e&sSzYb-ypkv7yp=bw_%^y=aTW5r=e<2{6Fp@E~Grm=n<+ihPRb!Sr zo9Dvxu>j9oxaghUA^ORlD-`JYP)OO{fp06&a)fro!Ic+6Pf2|wy21THsm107j);iv zV8h?f)4cTO-zzMAB(LdfG(^9_9{>Wl;AHc1_gN86>1`rbt2Jw6LJT(`?uMF3`2IO& zn&YJ7&gyWkRYt4rsvYA!);ZL`Mn)c$QAcb;%eyh}>c*fh4j==>b zr8|Rjj6hUYnlonM&&G4szO0Ub|qF^)8NUX;Q7d)MBpO zPUn!`J}>~jl%By?apE|wv6DlrZo0(q%u9DsQB%$M+n$Ja26n9GrJ}3hS_FDD3(j=z zEYRqbNxQD*SPVXYOD-5GfG!Zs%XaJeKL!9?i~#$!>_m*aE8ybsf*p8%*LGuD3`G5# zI(vc*fnnTthI;?q{(paqKmHrM4-j|)KBwvZk3tL}??k}Sv1G4;|0y5+FCcZtU4|om z)(R@YKf|!2R7_gbY5wv(^Q-Wt(%EKy>*RQSU?e3}*K&Awzg~d#{wyMnQm?2R&Xg0- zRx=uFx}613kqUT1do1P1339XvH)n(?yoYD}Y~AU!UW&o}ib(%-6B4hKelq7ob?Dz8 z$Gc62;6nY5Job&77}WSTZfo_@i2Va5`>x&9M{q=hM0oU(3_&xA_e7=Y1B;&ZjS^A> z6eK*ZHd`I7OAA2H7GKKMeSsvy@Vebw#4`Sxd|V_Gq)M(uPV@P{%?E)S&>Eb^oepaB zmy&lf?@&p~JeI1jag4&Hjvdd|`jAf@DI{%7fKle-vu(5WGaBGKzCJP^8h^ zos|ul9RD@S6Cvqh2(VNLr5n`+50`3i%|F=+jwWdG+e)Bpf)6Jl_Y!J?8huIsxhI!< z0Bah*p(@}zJPvZ}Q5p5k|~;?0geW zyN%8!w1C6@EwM9r_moI=C>(|fUH z;j_I_arHyCki&=I?I zSSYI@W_HGleQ?PLoSe52gg?C|W9 zX>c}Axvo1|7P^H(73(VR;kKh0;dlVEw^$0?dKszvx{*OnKo=VLHKrGYqgW)rc03+q zu9SVz?C++s$_>o!jml~8pYF^IQedfBr#6_ofb|0>7P%OHd&B7j%N2~TKVh-5=ttx6 z4jLhJX};>NFnc@eR_UvqGPw`OgHgt_)uhYMr{XDh2GgM!s9%1(*px^Y>pMw2xcF#p zZ--HzwRuTpKfw5$j;G+0y6iceEO!HK6NHwn+LPv!RC7l^+H0s2u9^W>f7po;jid~| zC^NJY(f(z>9)sgn*t!t-}zCP8n&TGTF=94X%Fq3XwT;H z0S z>J4#gx@R54bys9UdauL8cFr|I(Dlwy95Sg`{6s)Yuo9g{7bUGaeiOZ6(J-FDMek9{ z^v3m(n&Pnf(qgqXZkFJoG>Y`PT{(w97n=2_8b8OPcBkJqANfys+Vce*ne90N+iMTx ze?B`%2rwd0{h4}#_(~O7Q#F+CmpY_E0SoWxvTV*|4Sv--nc`Bb(qwH#){oK6;iqGm zGmX`U@;_GME$R&0zE#oqiCq=CXc2oX`K_Ty!nOuYzHp5znIvKgqT1FIejo}bc`SDVjO zZb}XG;SgDto8NlhUPa(8!=R%^H1X!!iGUY4TvtP9TAgD+`^8yNn02+gE!6Nz*BgIr zI%u)kVbPt$zok3Iv*%Y8Q9p2Ns#{OzK$l%3as9Sb*BoDtR<{neih&x!32|QMM5iw8t*_&|Nw3e`s=U=xI6EFKpdH0- z=wR&?fID{HRpC>gui58|Mi)<_21WYqOSdw;Oe1HryqUSn*L&YGTl+LHcP;FThTMA}h}k?o_sP^Ka;5BbF9(kiT|dF% zkt)`9^>KUAVo&Stbu_S;gPgzHlO~ca*Y8K#9L>Pj(b1t$SMKgsGIH|z`MNbN+uJ)< zB|Z^Sa$0p89QU_g{a>St8;Jw1m0V~K$Ibs+)V&3`5?jBrKKfYQ-_d}*&=P-xAxyl{g#ffOdSqNm(qOm z0|Q$g(k?W2oaxm!y*<0wpZ$}Zc7}I|9JYs`78`7Y%{lv~uc-SPZ9b?dIb(#~@yL#p zpR)@eyXLDFajWnffduSn3w{jPzhzFQu=JNrU0z@<)g6j$ppeP+M)WD?cDh^mp|6k<06lFKXSt=IWB>7*s7$4_1oS)f0$(B+M)-JE-My0Cpw&TlKl z#hI^~(hoIho8iXzRE<57&Y2TII#iLAQoaAKiSUaU=3?O&nR*_#oW^f>bJ)$p8m$n1 zVzCpA@Mr1Bup?HLGXZYpBJm)dsn5K3R9B(*&Y#9P;44p+bY@SLV2lZXXtc^4+Q zp{SuSc3fi0(DT{hF*J5CE?7BRBWZ-2EIY4*j+U_!IJ82p=Ot>OVxQVFY zM@?%e{IPnQuT0xQbX~i-!pOj|?3U?Ir)}u2umoPMn=5h4!$%wDb5tsw^DVHppA>2Z zWY>qYo=rL0{n1=yt>%t6@3HoZ-O=NRlEUSybEk1#{CgP?r^(3`w-O{jw^%uE+}xW+ z3u|}_|M=yeCH~kK!|8mPKmVS5XuxEdQNQo|^*lRurAh%U5ty$h0s;shFC?T_A1^P; zRMeVWBs=6=LIfg^8M-u|wiWEE1^wZ4a&=nVTd(C1no;FS{lPGO{V5XQPk%-rIzHXk zL^0kUFEu%LR|f=CFeVG@*3G*51!t-Hzx#!RzTaW1-%kJXuo59j7F~yDz7l|v?4on? zAR>!b7hFDvN4RxPSNDBEUx1?U3h-6k>$jjDfa zr)6XvFBJ0(EcK4g_^x}7c6Iwt#F)KqK9)r?zrAR1xnB)}29FhH=(9^IG0Z40j{W$^ z{qw9m&Ttq$vuEqNy}78nGe5i(fg)#k0cXCE9Dv~7*X-BjB~c7HddAK^d*{ zJchXQlV?I)wFICv7t@r^bKCAljt#Q!`dWsZE%xodDrSg$Yj0#?k6b|Us&cmtV9J@+N9QYsQb2KW zD_^xKXzq$%)q3f?LAlK}+=RvOTzB%8wjA45wRbOlQF*a;)Z*`|Qj~6nZJhc&_~m zY`56NB1%M^Ld$x2ooNs)gk=XqF}NCe6;Y$gg64g}U9)4j(*%x!vUfw#^cz9cDP*Z05n*5_6tQy!GNT#RgUWkA>;z*oq^|w!LCrKsHO7lgoyfUElrtX(+44m#4`Ng7!(fpgCN=i^o`SJb1|7T zZao(0ttd&#!nygz&oF9kPa8S2nypq?=P`UP#{(_SezrA@!@pW}DAn7};lJQ6gfOC- z>@CDnsG%Yol}!bp>%1E_MeAxn4!E?tU0v+=+2?fI{*HQMhB(@xc!2BtUZ!{L&PPi` z(n08#(~1ZTd0=!upk7F&saS877Kuh4U*CghRz8L|7=<1_ux zZ4*7_R!CneZN%8kcg2L8$EV(O21{4B$d4*wTTiI+`J|op^B>x5pUfgN2)NxKPU|`0 zX-VXzb;_!__32-gDlimGKn53+2Q?|?cb{g=w>nj%G2Yd9=6pwR%8V|EPap zCrdD7KW?lza5{EeC~gZZWrn5`e9y{R1 zyfY@ifwD_X@yV`s(R@-CJ(rJqvB?zFb&oKuWD=yOcqFMVvNE|tV_DXPQFy`iQ4{PqbwPutN|;qr#DjyyuNT^lNSqRm%Q4`tg;4N98XZ3xXYOcp z92FX+taSyqCRHyfyjXF`M4>p&J#m&VZ9$q5E#Hn|&DyU#WVF&H(KaPaVv}16rf{_D z&Pm^u7MHx(*<_*z0P{Kjxb^~t_~|hL3LOA9hBBwR0)&@ZL0?{5@^bSh*`bsVs`%kz zYVf|4rrzvm zZ2((BnaO-$oIUUjiNp&z;70c{a=9QU^n`Pv|UPuQwbyzjp}5A_#Sg zwM>ISR{mmTN4Y{5d&=W{HJ#&zyqmgLAC5wcqMvYZ1HvIT>wbG!e4o>lMGu@B-EdTc zU^c+zOSpxFN<`R-;Y~#Fx?)0N6dYG^E z^wA)vQEmf0Tkm3J7NlhQ?AhtLIg+MkPDZ!U9fB?tY}WaUXlEp?^Wo+sEV+U;G`E=r z6@Ov=UM(Lh+aK`rS&&}Dn`?~jj__7r-k`Zt4${l3uto4$ijCBpJg{H6^-YiXIDb^2 z*Sd3d6?09AUf<_&vX%0ao3A$Uau%H{mXvBWlU;7P;BT{ly%IASxna;hq=~WnAr1Ko zNKl~yvpA^SO&E;`5jvqz=v0;id9!>)FL$6QE%c=6oYeRBHXc^HH1R2l9M*ICBbkNq zDzzD&-VFA)FJ^H2?o9NGVa9mV)XY{6sq6Ulxp?rnU9nyMET#PV_~iQ8p?Y6cWbyGb zt=|hFP#dH;^;b0dyXsKDsWpt5t#<`w$BvPT#YzwCn$1=2m1NBssBZ$hvXTiFx#6qj z5QG@+B*8=xxM$%XFvOO~Mf#Ofikf7G7#(b#b?tr=tE7RUz#Fyghh|TmkOaLk>2!ud zBI$s!gjd*n-prgQXGUa7cxg4!!>(oJ+-7%ke1XFmWzQnLQ!?H3+)#}iWDS9*^k!CB z<-jXKADtw3-mEd`Psi>SQ-IX>%d3H^H$t~X3qhd-(S$zdtbp2TDHyO^xuxp1Sue|| zDd14==_le!=jw5BMnsHSu|qAK?y<_-!(gq>fCgnWe=hub&EY&!R(_@Vn`T*##at%q{j8Lg6~O6>;m z7rgmwnjA^4jMJ*_a&B{LpL-oj$$krADNWr=B8JadT_R~L*^TM3>2|t>o8jo#f?9XP z)0j&zXXUHENClB{?#`k1ZZe=lj&mXq5CX4`LD%oxT}t6T%N1aOzbXr~yD_5y;3fFJ ztnS}U48VqK^uhX-v^YO;qrIg?uPwIjbe*Yi{9J11AN@i6ODM)mKrpO{GTv#TC5cC# zV*$=WEU8htHx;3BahK?n)5iXt{Cin$cI&6GD7D%`0v_L=QUt3 zV(YCps7P%KWPgBAW~($1AaR4auH0(pr_s8)xiWZ6u|N-h4*|vD&UR~4%1G#GLsmUP zPamDsF@y$1O3kU3C@Bn0Z-$I3gvfXw#V?b}(TGl;5%a^{vk$|p=Aq|?StW1p{v$I> z0JV-mhd8vJj#$UUdBTpNn5N@FtC88bp>#sRXi9Oj?<`2mgQrZNR^)TUn7Bf%=FVog zS&2pXX$Y9Zw+|jy9oW;837N+69z#SQqkT~(aVon9pFUibq&WO^pZ#{)kk^#V+WRHU z`0IEha&t2$L&kc21pe4do^Nl^F0-)z*?6ub%<@3o1nFDmRAd(s3{ma*4q8W14^z(F6P;ra4*D<8P0E!L3P19< z8(OZNGIMXwIw79q;g9E_izfGE3e^?#hoWJyZ!|u8g9Tm9Na9LhE#UXUpU5UaqfwT0 zD0~%SP<0!LBfk6Q2_3Ya-Ohxs9rC51?CdV;sku7m_C+FNts{(QJggd))a%7<+G)(Jb7rn6`K%>|)!A+RW zew3apG<+t_l_kIQDM7mRN?qWocLD8$u>V)&W+}m$ZO&11f<>u=hgE{1pLrKX6_o znPW|3f&A1cozK6UEMmfF+^ImF!=WQ5%lX0I;MC;-OQDdJk1FNkq>;|l=+)rMcd%}I zme>b}(!He;*g@EcIX>Dr1ryot+&(8o=v173Ygg9SOB5ql_Mu6QU_bg~VQ&ljp7!7B zDYNGE;uxj0Ie<_FlxcG0HHy73r{9E^a2%|NdKB~5ruM9{<1IE7?7QC;+fcKn`sBJ! z56pt@4)`(E+imC)2wsgyQZwk-YE|zo;%hvF{z^`4*>PMrEPOs)E+Igja!p%WkyH5r zQ*BrUao--y^#K24bRP5|p`o2o!Nhj&;sO4eV%WpiGaeQM4nQx+4Gn6gdBgDRx%;S@g6A zxa@>A{EFKMDCHUf;nS|Dw#(rCqm+1R4D0o-?s!nvNN)@7xdDr{;+q2AwlX_1^GV3=D5Yy@0)Se#N!nl7H_NezXlr__~d`1+SL?(Tfs zzIA_eyN5iFR!E9>l=2ws`aG2^$>1=rY^L1u<|@TOUeQ3cCd4R0Z?XIs+Kp4L*-i@~o3zQ1@i?eMA$0aFjzi3 zBx?QCynO&WUTH$y_4AtU?<^C$dM|%tG(eG%v^r{k9b1v}u};Wm+uUQ@R{!t>VU>a4 z?tDuwrNAWhg174DG)<+`=t2-Od2H6Na@>GS-L0D3pVP6k^XzJ-A&_49q^lc8euQqS zYaI<6_oZytAIXuo+ZJe4L!awyPk3z3SVD8l30U8EXjH5_ANZ4+6bWSVry7=y*Nr(~ zt!m#~Um4FglAoheVy*_KGJ5UkyFJjjn?0t?6AgJjUK|s1Kb@dA_oGzrC?>*^;bJ1P z@qAAkYe*Ha1j)(|U(P7&pZd{gh1(n?K;xwoFS>_)0c*u@ILn$z3dFeE(F)+;OJj-P z*iUt(gy>wcMFDo>h)N`^Hj*+=^E)X>>l*<#g>vDPtTtJog)^r9e1ZetZOSr@SL_IG z$(ShS#HVLf>*4J`IFOPutG#^}D~KKbjwagXwE*<^6EBuAIR+aGn!HyRhQ+>nyQQ{* zPc?STKZoe#Z1L;JC5Rbia?Qc}1x--mZh^VzI&(l|159^_fXC^Z1fcTzW z;upE9{(&i)qKQ`1C97)nL0D>RG5yDmvt;sU)C*cMNR@J{uZykjh(!q_ick;_uxa@x zm1c1q`vyC=Yg}XBRfJ{j_1m}Fq;z`Y*+qP|6jcv2B^X~LM=RVKx^Zxry_MX{m)~s2$t~K}T6X`qx8R3qzcUlpz zIr6m*q)J7zh)-`zq2;txIab?0sMP0WjsnUysg=qCexsRV{V=?>J(OUrGJY}OAqY9j z+}XrUyKz!4eyZtniO1$*FOz08belRaKm$@GuQdfE+X24bd3mJH+s_dBJDWWL5>aK8 z6|8x;K-^kokwB;AtIrJs19O8rxc;;CBP?|^LzY^}=gVlrY4FD%MHQ-C?AiNaOE(&X z80s=_b>B$)<)cO7b;X;CpgLl;njc8YwRu9eW5w>y6$4`LV7%TcZNws1y~;((KKba1 z*%Rq4r^CBWOi*q^82srs5=({7;Zt`jsX9YS3 zbe1UhpeR?@vSCNj4$;2Jj;g~Q=By<5kw42q(=?*>nV zbjRv8G}6*kN+ks}pbzdClyuJ9uxq7(l&VE+N9>20ss?73wJ%d(+_l2SjFxd+X|G!F zzXZg9yER;aP|fq^Xn6$fj2-X=BtW#=-CgSDkuj^8k`3L0zi5L3p?lxT;rO*(8N7dZ z+UIZR(cHR7z$KfQSNRcL{&Du=NjL~1k}6*&1aqRIqbHK7Q;7+ro23%DeneI~%q{%^ zd${;xFJI}bUM6-*t@Hv%oVusK8qAq^#`7gkeKEK1B#(PCqnqa=`5zz^c{xVT5EOgz z$^pV@FP`_o~`AmrBI;->GpK#7s}Tk)W>o_0=Eos#lF#mKePn!swe41Dlj zE+F%7d|mV#wC-%5RpQc`f^!$;wr2rGG__J2 z#P#J&tWG zQ5nMCs}awJZ@zEHeNZ(}nA{Kk>J3xn#Nkn#wo1C+mm8(d zXJNOPN5R48vV5c=+cwt72T1M>YaUKZ#)H)f?EIHjF4qTi(FFt};tG5ad631^jkL~J zx6b~u8T{R3mZGz7XVbEw{Y65-*VDBhV|#OB@t7U4M5XSkfeR)mu+BP3KLx>UzHWEA>zJI6$*l<8V#nx z(J^CW*~7SQ3PXuwyl06_ksk>`C%*<0a=?m(<8aQHmH8b}-C*_OsF6UvB^0W zdJEyU?a@779J#@GZ4}Vg-iOZ#p1Ryrrc(9ho;&&NHRS)FMiMB598G3`)tsYMuv}cK zh_l2;8{XcPv^Gxa#{i`gUie&=X6c2Kg*{wDjP1BysgTtRqZoEH7*`&NV zkN>@>-|9#))l=2BDnYLGs5xw@h-2RWW{1NB39^VOS=e5wT#GZ3iA$TSYBEB+gR{X} z$4_{DTug9Sejf`4iC<^OLD)1Rrj)b0pNln0glVUqVf~6C&+!QVF>XUnZf`VAn9qa$ zL{W^aYuM1f-w2)a8%&T51)5^F#98ejb4oMnQMC+j0L5`f9?YPI+c;N_Cr2rC#q^nz zY0hPaheE9qrvhvfrAS?yHukR6l4mg_I)Brk0jtwE1LYO&CVI zgGs`;kzT+tS~2KjC0{V)tbYPu#c0n;khw;tDOWN8r)T*=;O4Q0}O_JgOCs z8nj8ad;ST2e@^ArztZd}-uGgylJg;Y3Z^mvwSmj3_5N|Kt!;JIwnrtcDxh0qwb4$; zPjBl{{NTK`LJ+b$)dfa=A^+23eLg&gJLLYi1t}K<9iBTSP6yYQ@{#FILRb%Xonn*o z1JaVf_hc_YF%n=ll%9Hx+TW9WiJeFiDt+C<2USLS{Yg#gR@yDOx6eOOY^y3~S4e|D zK5leLw|bn0z+V&Sl;~SHhL!at`bAz3lUrHg-`1+#W_y(LNwehVNx2MZTi4$NNqW0YsDx#`I3LpImxfV14KS}>K z>dRrc&Qp6f;>Y@)^2fcfangv007VsjeAh{npbQE^bsC2+-z6q*;$z1(Xe`>1 zU<-5i9w`-$VmHPpUibZHVK!c`3Hw;JMbw*G2%UTkv_!5YI$vGI ze{>;oc#PqW+-i57@Xh)bVcjO^IzMcjVy8I{19ur_0+}4aQ%QXgSW@w55-O>$v&G7B zOv!PRtyR8UmP@t%%DZZi;6xjf%jix(GYB<>Y&2?ZzDY;Qdrt7ZXgmFvOHLDnaecPA z^M_l7nG=8c+%4EP+Iyqm8(oUQ4uT~|REbm`4f z@pC@9o_UoL<*^HMMjn=NWN4Nu5>=MM`|;ltKH-oJUIplIiLk2PC&U&ivZr>X^1pR; z=J!n{*zJ>q-dRLi?)ruc+! zB9qp2c5cQ@&97li@TPB(s(&FTpP+cf)&G2Xqc0STFD`bcWY@4Uu5S?3=+8O4qS zgDUAvTi13G5eI7h_ zeT$BEcA(Tb>VLG_Tpfu=%;-XB{!soTr)DL>1{oj(n!#T1U1OaDy25FGsXruy>XSS_lskNW!{=d?#I{}QYjVFEZ!(cTFfZ$94~(EV!Bixhh4+iTSJY;l=$_vl zzR~_Ir18Dy$vpTZ)q?=rDu{T%X~Uvk(yCI$vPc?X*knu8w}~q0x*LtP@HTJ4b7waI z(9s?CP2N|!3X zTwZ1o%X-)F+0F>w$8^mzv3iqc&x*UE!t?2n`o-1^zLi&Zb5Yo-jcIASikvcAobn|y zcIx#!tXT?Z#jQ}#AUb8|rx&vZO@5o+74szf=lW1FS0-(#usroWW@j)>MyW4V_cu9h z$yBTW;a}2jtlbYd1{INR=?EC{jw+mHIbj4S%sb~W0E z`l%Ns<|{YF$kAsySt&Q5R5g8f1mkt< zG)bi=+KHaIU{^PL>mMEE?98LW{_st~^aFVVG{sL*DG=-YUdvT|#Xg2zXtQbqSh{Ps zyCl>ZQwHPJZ|;nh;07dT^e-2;FQ#_tFR$~7`-;{lYc}4ozwwOnzAUQ(7mk6%PC>(PrRW?)3H!IVM_QtAK-Vh@pyugT{zRqSuFw9MmTvj{Hr9rT#)Lyhg+s zW}-sw#LQ7kS^nh`WQXlX1nhj&5GA4#|17TT9MDHa?j|lC0`bI-M%WQ%Ze# zF6_DnaeOtbu^XVW`30HdhYdpi!ZM-ktB_4H<&Y|IbXzD|4ODI}_{ey<)xkR|n-_4g#R@oZ+;hZ!wQ$jSQU*$@GQR|j2yKq{&|pueu_$TFHh8At{tUs8 zwC+47<=pSjL|q)|0OzlKw15?~_7>9!oCg(1&>JFKELE|J5C&clZ<12X+5{ z;7}mb;5rX&UugqwID`^X3 zqAs3}HdX(#k{g--RIpW=m>+gMM+C_5?@|-J3-W7mgfJQLyOju32jA*_L>r^p%U%c` zvU#nCpo(vwMXll@j1Pr)=bt+r4qIpb#|7?etDe2hcK-{NOTQ;j zqYs3LAmI0JLs$D}3zY^)G_;nJb?1?gn99IM>&tr67Q3-iWH-diSW^;CcBfZg|HbM6 zf0BMr1xYJC0`d9#+-jgmLNTy$Eq0W#*KIGm=z?sy%7kRF=R zCI75@z4e?HWEYx{5y{7Nrhb!IsJ7zNa5(Ul;r;YwOk}XsPk@D#q5B(ID=#mj>5+48 zb+u>@`f1Lggt7WmQs?&;^C=gXYBoZ|a&VnT5+OU74jWuyE)2k|umGY-*(1vPM^+ao zki8L4)WbW{C2W??^chaMt+$zDil-B6b^+{hyl$HiZ+E(p1Zx!<%I^Xa)Y~p5P2^ww z7-uo#_i{ifZ!pdzPKY=S$)RZ1H!gjwca6v`;b0yHT-nge0$!~{h*$=7-|p(8IZxlk zEewc_4In;^f(V<3=6 zC)VF00k8yvcg4xA^%lpUm33-7Vceb1BSgHU1j-)EA{p&>1}SUq%=k_?qw!UWbPVO@ zPs!#9{NA?{y^n5pgWI9@KT7cqx~r@Wj7aWd(l;xKUSKE&!7D%Up@!ksAKKt*JSJ3` zWa$Ep>K{|0`2~S~8w!L_kIV=B4@#HcYecn3ov^N^R>zO2(Rc-E z0RC4fuyNTpeK4EFHdm2Yka4zTT%)|WD`|RdOvrh0chx6l>ehO32`SE;(`+h(r(}O` z--DcUS;_OTKZ8`Nf!d7uY^~F#bd|!xwzH-OKqERapXjJckt@ipTEQ>uynGpda&NEG z>vr1(5gr|%^wHQP&wfBl=IUO{LxWEp#Lm7|PzPpFNlFlWRJlvp2IYf@yyGx*V7(Rrr^#wp%v(CyZtq(Y2=Ha06^~K zQ&dq4(R?4%m z$(WjjF`dlOhFj^dXKpOkcYPU2n84hhgd-hM7T#1qyY{#{;s4Zd_-Zz$pz5Li1!u+0 zD2QK0Nw|j&Y|6vT^!sxO`1A9&XzwJGKX3qbAp#*OgZEA~BBV8FTyl7K;WufUt?xTF zPSEE!{<&VBHJ)d^f){XcWEuQk3dR85ayr;^_;r}SV1@^{LA@h`$B* z$VTy}X3vLnMKvZ<(Aqj&{#uk{)mNm#bX34r=<$JM%+oW>jH^(#rkXZJrP!)mVM|H_ zrp@!S4-x5br^}oHPorCi4vXXCx2s4Wz*Iwef%1L|?Yhlzk_WIua&WhsFD-I)A%mj$ zYaJ1To7aYx>y=V|8iy+*g1Y?>Mc;pC%H*)AG1&7wyLZnkUd|r>Qfp`<;X*j`&*yzK z+&mg~ewGT=3FSlNL;{WS_D-}D^+gcON34OW6uG(e5ScNFyJRg+=mhD`3Qi zEqm=5a2l?ou?TlpYmzP~y>&YUz&DCgKr4s{zr?Bc0FZy4R+P#D8wfbLdr#{=h40kg z-gn`qHoku9c>R3qC9&velU0JEv3x5G9UO+GpkC99mNMU{pTC?zAs7#{7u@lN4U4}3 z?}**d^Y(hjKTBijUy8X>^YObv!Pi9z$I(-XqxVfA(3$0Ay3*zdwrxTqFr}~yTRxZt ztY1``K$&aUpOq`D{BT_VEWjlU(qRjF7dIXVo4fPZySl2aUO}8olE?c$U{>0k2-GPK z65?2u7;F*`_v84sC?vUVQ@S?Wm+N9FeD;}%Q!5n%(*e%6_aID0!{Gqb3+#un!}yC% zi!*%Dv=L!o<0 zh0||Lw0>dz=Q}o(bS-fcB|7q`n9c;*j*T|!Qiiuy7Vf$R`t8KkyOf0fw*eyZszK(G z%K7py>2*U=zXW25HxAQKN7&<3HOs%t%IB<*<~8ma!4q+Tc06B(*bPa>cQYx8sn8_` zt1F6j+HPgb)kPGFo^Ux|4&v3G=wZkU7u%Z1elEfzSaLWkz)!G#J!U_LOo;NRv@%^} zXgZv8qJt13_=ITl+=lVuwHDD1A6~ee=>n@(MI#USg&*{kYkAYJfINOYBS(Qa>`@o)r`6mKVS;Xe@e zO8IayC=^n`3PHoEMrBoH7JVb#lZ9#_-1C)c=vK#i>~OjtJxW;E;9GY$G`coX@B|^gd_(VJK$H-CVx0_ba4|ROV8U5_i*0viF`dcEc^#99poA; zDY5JJm9m}&bB_XNcq#bNPIw4}Y_#Fh-sL7!pQM|@Pxh!v^7li+l!_PHhk?-M%F0}; zBkj{P#;j110lR$LiHK5(4VJB;7ktcXjyZX%Pm1=X5cN6#D~?k`VxjA0YdY-xPpjkKz+%18S2{lIT)``{;3h2CB6w ze)|Vk zf5^F-4PHb1*zonHSsJ*@7odMSqr1H=1(JG6{mN7R+-8@GjkDOIT~{J^!|vhM19rPb zqgqD?fVu-MFvbZ1=)8Y|u;WPtF@@R*N^+cwH*Wxp-d)tQ^HJ2wNqaCxYe?}f-RIf! z?PiQ`O!^nuh9PQBp2R zz?Y8?U$Iu(xP0rbKaU(2IZF9T0Clu|xP8Lme7U@Q8w}T|8A%0ghPh0bYQ~# zi@qRTbl#0|ES>9gtIt?zA0fUPGKpB}TJL&`a7D=B9-T$_GnpGh=|ns-9-iz>SU5VJ zkj2i`7U$V&dpN4foY+@Fk_DniH@l~PkzJ;EMoGGuS(f?cx4uW#NG4^APWHexyfdm# zrCHbn6A#l6Y1zrvM+>X8I=)|gO2KM9M^}%v*Z4ez>a`w_xPPP)q}Z*R?&Kf5-h)IJ zR4bK-rvA}DvGmZ2*mW!aDOjd>JYw^fY;}FU)+~DFw)hj4wAy^rVK!H^4c%XQmT@40 zTA`JPtxPAgln|7v8jLj(6mUBZ`X&r$5*d-=EarM5r(*oxbM#Z*9|B&SW#VTQ`(Z^S zs!R7O3Eo2;aM{j)%lvC>F8OoX!QsJ`kHMM$!{;CMjN4jI@UCvwpE2<2qW*RopFq3H z=Pce20ucH|PiF0E(Xoqt!~ZWte^E&PLQPz|O$H@2?RJc*$-GR|Os60a`Y`95ClD-v zuZ%LPxF9NyHK2P$1bU6IO@pG3hFTB1wzBW+)S~NdpViCa8&jz`qHrS3>)kqiqOgRM z>hfcRyw2mvqbWy)u``yGCwk|E;~I=qsuqEwvsVUS>d8)LoJzXm!Imw8+mrdn z0?N0VHQ|Fv1&CS)A3GOOAVN`RO=WQv9GMjEqceZeZmEx_a$%AMn_H%S$KvhOthjBl z-E&tt;Y^lQm*{ZAhv;~Ym^pt-sxKdp?g9(5NS|$%2QU^Pk>gAV5^N);f6rop)*Ri) znMYg?9fTeS-?u0cKI&rHn){?xC-GmWINc6{QI*U<1ivM$&a8AdH?@+ldb{l1P#K&0 z2PKF1g_ggH4{XjE17V~*ufm2X^7FCH5#$W|Z^Byo$%s#XN||$A;NK>`T~sJ9v=W|7 zR?=`_UqKeQU>eMW@}VMx`B?{Y=-4*26Nlxm=IcNyppt*7sRv*ySS?BF1u_9QZgkWb zRc;Lx2G<+osOu-o&(S1R0$#2tx+%ye;IaJqVcp?jBXN1iy4#`4;V9^zN)_&VM8W-Zitq)kbiK2SMsGAyADQI6Y}aEWZbUPm2GyBJwtusJ>8p;Bel@IbO8 zVtvFI$H_??KfNgrIKdmJn&rBwWmg!!8e8$jGK?h!$G7MXe(K+mnSZ*g3jE6R(;k<_ zG)-f@b$`NEAibA?gLpMjCcPm_2ghhCfpI>~t6a2Qh}5^W%A{cwC=f=oj@q$U zt_rWKzyqfTq}0ZNk2dD4VhLZYw%6v9>QJS9k9Y(5+ z{1+e`I!D=%!;;o;VlIyTe`yHHu!dzqp4A`bVrw-P*R-)WB=yMvKuMmeg2jBJ)ek1j zrhixnLg6bP|J)r4AKA57{j4=N&xlMH2aquz=f#^NF+?%oWH`uJ170@T;e>r`;(}+7 zN-juLil#r;Lbmis%zDApdhRtOBB!5cMPdu5y~+i@T*e&}Xg|^n#@?hb#eA zXLT5$=MO+@J_Hax=!{xXW8)6AG|(L{``-xu5pwEWo?O$6^}9*W`Dn8fJI)cjt%ChI z*zacHcLpG=JrqQdx`$Fk>e~(qjW`i>#{adu}fh4Fez`ka{lQDvu>^UNBg5vuaY9P*I9NHv)PRMQ(EaRY zltRA$b1?sam4UncVobH=KsW{lH(Qwizg-Qps>yWNo}JzHh~$|wFD2blMU}xOy+SMA zVuvnauU+{y_kG0%b5RWER)gT6UdF+ZH&ZfWD?%|bsi4|E5h zcKj5TnRVhVwZ>*m$R?%E)U)tEMNAM+e!K=f%YK^62xVBRohGRR5mFU&kpi!dvZ7~n@rpu(mHHRwVmjG3@rTcH={$`E!O!AJM1(cW1$^8Uo7hzJz!yc|t#FTs~AWAUrCA1&MRPk$a0&CJ{HyqbuYJ9ZCY zPe*4#t&i{*m{LnBM3d*3GOVq2xP?%^rFOq?vs2(*OH0@|<6^wd9YXQN%?1KlXMwfZ}mC3eo zUSj_y8`Y2BH0FY=_o)SO@>+NlsGy3RP>3B8ZR%oDR(OUwfl>JIrT)o)d4MM2e)GIC49TB zX;j0IrE|s>EbezqS&h0+i8!lWi-k%J0s0FgNZoORN_@+A7v5=OihC2&u4J&`;>2Us zqD4%A+t~At%-Rz?;}jgY^qcOC$FNsDn&Z0E zPt%ckFg^#81gdACHiXttE#YH&>jUU8^QQv7_d(g`zQzE=76K|a)5sKHZjJx zsrcmTcG~M*v-`*5QWUk)b#7PvU(qy4p7?|Ow+*U!G*M5XY3tMq@ zh^cI0B=OkN%_Mcv186|+#s${ipbK78i3U~(M#optwrCCPLaB$(N0Sp1unDblr8Pf#=3Gfx5qhC- z0GeVkBmUaUUcA%(9GbGp&1|w@TH?LFNIbeb@e=^2L{ZHX1J1^*+}w7jr%(M8T4MCQ z!_bI;+nLSi1LSNCazm)gwYd-i*6OF6^Qu`0FSfxKmM3FdFWmEknNuoV;nn!_jLnr< zTrY`mS}%A8%UoL4itz^YJ%xiOBJdqlI{jZB+KGB9^hiiuc&b}n^WbIatF)z@a;rm?y|vtUWXER>_lM!~RuBb7|3hZC#rW?E#K3Uy z-fS)|&*JKQ3Q2;pFm_VO*3i$p^hA=MCLnbo0!{}Fy6b2y*wUi!`fp%jk$wB+Yo)6K|c_}MC}d2j@at@_=9vbl{w<*1*r+Q8mLI)1;3Q89GSH_S!=7tuOgi3 z4D{LSYG3ksC^>fA5wjdiWiVFyIXqTa#cAN!;VIfr)#OK#tfTxGX^|}V;iDf6qd&eg zA7&*)yn+n%*xyV0bdw9(!yk3Y$wH zi4NL@qL%~Nj`6xXuNy|COFZ!vU8WZ!HFqKx2Ey)e1?ut$7Kg~@e+&tQ^pw<#$Zh5I zMmvy)kXWXp*E!l%=0JIU?-&-{tEQFL$5sR9q__s@l6}Qb93>4I!pPN_bLpBK1)~wr z155A}o)sP!0A;$;u^K(wa9~ckcqx~Ej!;v^P;vusS&A)m`Ml;RcF7lq)TG55EOxMF&g&Pb2bP*BF9Y^*wF{jG=}v z*D9^P;3QC#*5pS7RvEzpY8wg_C{o9Wz=;BiBtu}^eP^b1%(BNt+Z*qvI9YR=o{^oq z=VxI(4JNeFI7EYE%9WW5xe;H`wBypu-(?jXfMe4lRW-dpdVV9hitTns-rTS+9&_B! zaFVe)$KoW=bLl-0(Z05zmdnsy1g%D2I`Kdz?^RB0n)w(E`H_hE8pBc0Yvia3vVtaD1?Sy!-=j$1-mS7y@qP z6v|o6#Y|59X1ln>vv=gw$I4dBb<+fQ-=@5Xcsz3Nz3pJFew;td z=k;KMh(aDtcb$tKEo`9#&>Q;979 z<;L-qR{!YY&tjroAg=921T~Py73MQjH`yVzFW$blk!bDc?*lq2&d^{C9T}>v*T+V()=QYH<*K<ELIlk-zbMLP|fD5`s1#&pz9I#awEY9{Xy)ENO!D-1DpGR)GvWo7e}A8bcm=R=iN* zm>m|&^4S{$cb?kf9hSi00~9ZNmZ!nIr0lbw&@EW3W(jRbDUm=K6F+Dw$l^>RI9cf` zx)`6WwT8}0nox?I|5CRYBZ~2TwZ6gRfCi%eF9ZF%Q?0_*M#Ir434@3x>*F`Y5&GME z2g0Yee2FT!Cza+w)ULHf;p?C42A*WkD?Jyr;@l@w1)mVoe$mlVb+tQD*P6}4EDma3 zodkh#RJzqjz&|_%S`SSt=Kb!Q#pBE*Cbg&e@WfaysDTK<{)0zCVxd7%n}saTq4`?f z_GI#h#x_~rwdHhEM!U-*m1CvLjZi558VX#pW9|y3x(!fXS*SA3ldzjWgA-0Ln^69W zaMF9>X(^X)fE(9P;L+d{>)^p!l?+8+K~ABW#dfyZBsz3BU%0~ZfntL z|Fsl9=k~1 ztU~wPnF|Z??>lnd&qphiBlx&Cf%9~;^ycuyU^8@ecc&PS3IfsQdYDS}dj-YmT&-jd zX!mO7P=y`dO*%qN%Bt#Wb5|m%?l1o?kHR@W2N7o`<%S7N4P5BDa+E(8mp3z zvgKY;riJpp)2F~%7_7|rcWBj6Wl$3|-ec56Nuaz$v_mO}3Dn{@27NjV=pfab5E%ps zoZ5*Fn0v9f*=SYQ`~cF0=u|YCa<;8v5Q?-zRxk(uxRQj-czR~Yeq&)fTWPA~@S!s% zpC=tOI=#ir%=p?zOYS>WecFDwbDyCch{8!MnQ7I}voJ>_rfWiK%@2(VJQZ3zU^$-W zfu|`6JyecdqgwX`(B}nprwb!^SmL(h`j3m8f~6% zG@?-UHx!fHO0we!yZ`Jh^YG`$+Owv;i0pQka>DcVSxwe^DSj|psD_z4Z3mYertb-w zTntX_16V2!N4uG@X15A_X@F~7Vs~c$u!mA8-#`g}y7*~;FsR3wQ@QAS zzG59S&)5njRn``l-PWN!9AtmtTAM8w+NqYIU&>&Y(KnBf!D}r`9pmw8S4?BKB5ILAryHG4ik9!Mq-sKYGBejJSfS%_HTRvL0 z8LiO=TlS%i8c{CG83wpolI)sLRjx(UN9?DWvd;t1OMbCkOE^W#m9(!!Fy48&ANJ)w zR~a5_Ej6A}4SSD4-veB#!tyQ8wp%h-t;@!qhaCwH*GpvHihWc(Xy~*wSC==+>t4{1IWti8ugZRJ;T`A!x~Idg?Y?b^9ATu57}X*LXn{^i>`@% zG(6tskR3`I{ckZUepN6F;V~t5cqh4R@EFlF>QvBh*)Y>ikMXwAeO);w-VNT-oSJwv zOE##lp6wD3qw&*}hb{b>x#9^>7H?b;?0ocp_#X@UqTtoe-S%0<4o9&if1=l8a+n7P zOQsPjvcD^pSJp?Kj_5hYyH_Vn#GGu+5GU8u8oC9dBAT_EI?Q3`*I~}{SV6JG_291_GY$@x%5l-qjy-<%1MF??tYE3&Im6K>$)g)reK*RCrh`>CE2IZrrGb%VSZOOiHc#f_lujzk3C&U5A*CkV%~baf3WGC96DI3 zMy@Z{3`IXZZB(Nl!H^P+KXI9?Y$|J)de8m^Hoc=N?fKo_jib2UQEM0`e<^u5{#u?D z)%_{^01359e)Z5}KJ>#UtA{y78@x5irv8^E+oIzB_-RY78{y@0u&`w1!bfaTj+1(7IJNwB>yV<25<_$ zWZL}s?Lfcr=}8B7=s24g=s1EE6HjjeK*BeuL2jj9o*!x!d4MtrCKdJf8i@NKv@0~q zqVC{Qos(doPs?$bj*Mb?w&qUOUbAyc)b6cpYB4MTX6=+E18RjC&me;rLIl-FddK^x zU(buBrh7iKRzR|ePb@BZjt3@EFQ+{l5R?=QWh9Uzde{VgiMcKUgmyd7t3JO%mgR1-Rd7e zRp`HJU3+{y5)Q+JGw2`k=S{e&L#+sKifbWk8Ayu0U4_W`NIAyI&7p~AM8Nl5zPtO( zwG~T3uKed-`LyRKjl>4iNTuY9Ew-zCXG&IYP6FiELB4HY_F<#zYpr}MZ}~u$+8q7c zPrjb(LXCndeML?nmJ07I^x>s|IuQg4Dxd0(*~4s)6Fvet0Yo03WJm0tlvpJI)!LaQ zOU^N&h3ZaRL+qqFNkdB&VH;e+QX4;O@%oHEOc+wTCRVB8fsB*@XFl?{W(AfSE9R2f$4|ZpZb#hc!4LGTmF} zZ)}a`kH-Lg!w=P@_h^niPzVBwQs3mxGu#HtxFGiTceDzq03}MfM$|q!jAQw*pc6wp zz_~`(-y9W3t>0&txMwEZ1Bpa?UmOF_7o7gBrU34ES{o5)YUVqk=baRAx7 z9aKc}>n`ap*E2%KIj9=~`BMEp^P3yyz5C@8B z6Af>hIa3E8c@sv<@n@-C=Sqp^DpCW`t_lnqM|fCYt_mWyj{0iUq!z~kK&up~%}%E- zCVMG9&~~oaV4`0MzS;_KT7_ivo& z1wl(l77o4dxKkXW8ym8-DwYK(fbwabVHOzr>&x-ct@$dpCfmK?*s}>FSAZoa@P%r8 zXE4?*VVax5NJ4`t)EX^9ru)E>|2r3;kDV;ASW$1x(OA$vkc5%Ux|@(#pVpThDa|ku))|r_1ScW)Y>XksTfnSDz zll140^j-h)+|v4nW5|X&ZptdVdi--W*Z&|z0V(|wU?IAYWYGwr5%~{H?=OW`qO;wJ z8~~HS$dp@AXc3w8FTg5h{!T&+>;kvj8%9V-3fVZ_g?i$vcRqe2`>X8Ue_DU4;2{+ z^#43^FA7LwU2QN5_}Bmbj0!S8WO%XfA7?gO1;OQTO0M*GNPW>v!K?ZMRvV7+lXX8v zc#rN<-9sSL_V1kJ)4Y2N!;dmDM|J=G6;cuOSe52i%CV7BnKu4UT=HMv)`2es2J?n; zIAq4u(K%D=Sb+l&$5S?_(kEf%}K0cSbpKfp0Z0l^dR^z3=w{}%ho>HweC*%36BI^&MYkzKpp$xUW2>F36S)Y0NJ|pl zpzaurKm5Cz0FYbdeUp@CgpNx7E4)AkPDTp9Pa znlbVDTUB*%feQG?;QzCL1|;v38;sCK|I63-D^D|UQP_j<=>hnxx5}zQ<%%E`n)jgd zm%uM13Z>FqXt2wJSM7ym$Kd}C@t@86h3tJ^QW^fE|5M#SIneSLQN!haWwX^H@+GHJ z>@33(y}#-s3_5GU-hCcUcmH&4tDxe<&`R>&R_nhj7>ERJ&=;sYO!V&(01M;Y*}Mz0 z!DaznJp0Xi{c96=BUQosGd_Xjlg);*Ja^Zf(fs+u#>F$3EWatY$O(y||J6C64RV0l zcOwf*-}Ap8`6D_vJ-9bn+zBo*=->;IV7%787#aQj+7F;FCj*_!(%btsu`~;_qhbwnJo9Z6K*m=Y_A#oH85du`y*fT}zC#KRO{eWxo-wy|b3}8KfWySmacNn1Ojr2y|ojvTSWysz6 zy8ig6v{(#X&qR83=-*~_4C0-}a|R1+Wq$e|Ts=UWx8OKB>*{n=XrBuro7Yi!8XH^; zomI(>cRI+$dD**a$|nHMf6kdHIWu4i|1+|XSZBK|IZvY#cIgKyrBWqGO-aYUa_B`8 zi>LYcQf4Q-zvtNL)v_{oD$qa5Nn{1eq!3Dead$`@yK*HKx=--opBWQuK?3b0w$w(% z{`+dccOqfP_F_-v^u&A#h!7rg4fEQPGBSIAy|)W^_5LUX2DJQ4jke)J4EtdB4R;iU zKJwLPNm@cg3DDh3AXS75CbX!L3>=y7Pas0_0!kZ@o>q$r8s+;zlk@t!Cb7b(2pJ)` zkkH0C^MD+kZgV6{RI?Dl47iDmU2nZ~$y9$m#*)(^%8_U8J0MetAR8*STn;WgW)Fb4 zr`sH?LY0TGf4N3A;`wKYzZ>BTBrwibNE56tkiQUosvkP~RXpw(+&@L?_z7?zQ+(~{ z*uBh$A3(~rAj~ALv|0qENoGZxJ!0w1j2id-y9W?=Az94kKCPwWC9=A};sh*o+4@?< zYW|e^Yc>9U41n%^UvZmXJI#oJz2N>0G|(6Cm6&%WBZIYt^m-HRacs}nv8qyAT}wEj<}D9dOQ~x6hjc!d+;U5P zJEWrNeD=RuNC}#D`l;*qfw@$(8L_Cebkd^JMsYS*>fdb#C`bIea-4i6NBN&}w13@L zio4n&1Js)3+olwwj^W$qC%4(%WJJg>8zTwy_X%wAZ?8}Ela6H@y3P5s)5B7yW2X~J zn!qV6rFW26WTbu?^pTmY`}gn9(#sC$4F9SI5Y4cGdj_#lNBRAC=L50_0xgU+9h=IU zz5s;tO&-H2wen%>cO>+bzxP$C$e?N=WRpdP|2D#38{239L>U z6E~V6>wyq2U1yM1s{zI33Ii&;9u;t}{`+2nBSgfIqrxloA*ugaCuT_4^L02=8JsR} zq;2^cD??@tF|_wb1ZIdLxwJoV#8f|C709A)6{58Aq>Ge&9Q^*jhY9t30jWz1L=pXO z5Od^2&MBm2_A*iIWgWBmMLKgjA(Ovmn&GE_pak-Wz5pJ7t8Dyi+Qsg=*O(cp^8b+o zqRV^ZvP(}kCg%UW5gdGuE+s;wrXbsO^%GzrIl*c3-WyoG_ev;KDFOM!Ez`+wDkG=$ z8ZK{-#ANaBlLC~}))^RF_RFWDb6{BpdJCzz<=!yVhrTT@&h^0UGe7#EPL;x$Xd@p?u5=9 zU}Zl{I$f9n7*>fa4$tRCg8$=-2*d(T>Ji05)llxg?*<{;I6EsBaDTY`D8sy}7sJ-( z{r8DpF;K&T@&AvjuMVr~iPqL5h)B195+c$dNC*fVq`Ny6kdQ9vFaT+gke2T5MiG#f zmPVRGcf&V_1HXIk=N~@$oV{ny%&K>-d1pptb*lC~OB9P985QyGl>c7x&Vs{nh11=y zqc_s^>qFA4w9FsYD?hNJRW36Od`lT{We3l0OG!zUy&J&FB>t+ud<)Iq|A-ntF`YH~!rr#!w!Up&0^y1H5+d82Wuo zHxYkd_^Q!cw>3)N2JOLyq|Ao%vR%<%eYk457}^&6$OxR1xYNnfC=L#Zq>}xm@n1)L zkQB${b+YawS_B=*ZUX0p3Nt0D7wHRb@9jmvPxOtALc99lV=B==Y)?gg@u~4UJK& zLR!~#onvLRG&AuP-6FTd&GYx3B;N9d{VH9Qy_s7SZ*UGk=WxE3(O8*je~wD-iwGzo zTnfd=is_Qny1KfGS@M*o0~tZ!yt9#FK@+~~+4!4w%iAo@jZyhNTxejDBbxps3mL8S zr?crd-8%nxHEYlbB;`;|0g^3tQxH((6xR`-$C#3~l>KDBKK30P=1) zFASXRDwVHU{tMJLJnpCCU-i#)s;)W;Gtb5nJp!GgK zJzP!nF#A>$dqCxEJ5{iycNhGIodl_O*$>B|68#OPDa~rSVOxXU=vtn)AU1^J#7psp zH|mBos{xmAZ$gews;i5O#tMj ztAPHfE-Wox4@sZyPlgOBx7}hOqh5oyuIIT!r_>r1T#J5{HvRmJ)-pY<=YvIF@Nwu7 zkGtp3#EygR_!I_8q5PwuB27ghwFzyk`2QWl2 zsLAp;uF--GEWFIuuXuS4A>C8f3Q8os`XU9;Tm9uf$u2n|x9{8)zq~k?$#fQhw;-=$Mp;l$V0(77jnqKZa{byZ2o4%{R`B1?r8j*Mhm^kU#^vJTGM%g`myY9P z3ft(gxRSUu#HbVO)`YzqDOhPSx03{}*rA_pDDQh^vrM2#G~Vt0Y`-1=Ft{eP4k(IA{QOwi65E`tQA)S4xJCdy= zCO4GkqWT=tT5&P{*JDs@5Q`M?vV;h*Zbj$tvO5S2TyLp{Y<5$4&2X}Pu8g1CuF=Ah;grOZcLmxq2BxsNAmr9FMOcKW=84SLu|o) z>Y87?2tb(7!b#FXsMriG#O$xrwI2Klpy83MHKhR^(cgq{m<|5%tc>Hbvcy!)l#J|{ ziZeRg-{e2RtF^j=frvR2dB80eJ$^_9Yy^+*eZa|UAuuHmCG@8o++F@Ug*_IrFQU>Y zF$^Ab8}kPf@VCCnm)wKaZn3K`fWE?VGL}1c{o+giErd+XmAs(k7W_nJkx0m9T=~>o z1yj-=m{KL~{Vjl&k40Kk+F~k}0vx+a=ER~~7Yfclk-RuNew4Prjt4g-O$eXM9=3ph zKnytPo{aK2_)6LkRP!Tz;(;Eh5`j>%F9t<^D5CG@N-kzfzYcdgT@YZ{k#$W)QK$Lt+juwgV$xRo?%Aqc& z2kW!{{!ub%mK9yz?Sov6Nux;L`{?wv^LH3kOG}Fg@DqOj6Bbz@?Q`k)_pi0+HWDM! zT`V$2AW}AyKNKtDm4z(?jyUj%Ueke|yBEqQf3SxQN^n|?8E$qbUtC+1J{2%ai+MqL z48$ac`+LP!g6FbbEY04(KTPXq9P@_v8k-S*mj3p^K@fk46KK085Fy4pjcI0XF7@Le@G4*7qHb_I4tM-$`Uu{>7va&-T!;~XHZBEbgsSrb zH=*8%*gJydYfRnsIJ~@YL7?wV;H5Exg^+NVd0oMm87M@^7c(=(x7B&$PQ4G6j z_?>e8os1k7D@6Fj)tmYJ;PP-*WOn)+Z>uCliYE=Ka+`nXuj_`fRiPM%Q^f|pUEf%R zV!5oy4s25%l224B)s$Jr(jLw}y?X8+a>EIXVS9f+64XEy;}a6n!F5yr@NE8@cfVJ_ z*709xUx#+@EabZwBxytgBrlzcK&+vG{X(@b7YiY?1yBo#KH&{q8%}Yulrn?aEzIg! z)}xrTo?cvBoJ`*#$g}KA`FuX3poCBZ?4lZgGjatW^l;&T8FRm%MK%a#d;ra)y1H5; z9|xF6@mPH|EBsfe9~8{xesnaweu3zCmcB!GEQb(o(N)xizb{PL^u}Ld#v1inVIC98 z+bJBbj>wsf6fh+wCgRY{UV#y>X4I?}`}(@NCq^9+Ku!MrMD~6D{m0FuaRDu}v7@^7 zl9&ud#=T^Z+-Nn+Xa&uD&kMbL;65n~O*vOpW{{l%z#d#RH8tn6Bb)gz)mM^(_XP<= z#6^LdFyL406%rCsW-+d+m?ob5anN~pF~~+2_c8p*7r{-goAoq$p6BHNgsGwAKFVAu z!@nNFEIsr)Pl4?`Li?J2AnDdQCA%sDq@;_IovT*3p$}mvfM_zL1-XDr9N zG_^wA;&w^R>shvj-X>JCZ}OG>dS$5CegR&Z58eUL#b&oCeZ19-HK&N=eC4bkO6!Bz zKD)F;WA;z0ylNJBgh$d|i&q^50+0q?xlP&M9x+ufikd*^*mp|>H*-V*RK{C`BqIhl z6TkeX_tWk)K;~>_gS0j_Hes>}JS`RX`fqT?oqNyreeb+9jP<(`dsF8I zKCbe(xbaSW5#6i4qY!KXU*J!`APIVFG}ju0nFB3F3<63)4+N#KVts`PZrw&9Qm?j6 zM}P1P3KzFjG_!*>M(kC>fUS0$hK7cZjpqHvgo{l}s#r*CIql93#=r&OJwce9u^XY@ zGe*dcn^XC&3PcR5(jYI=t$49>)$JEmP+;UD`*{t{yBV0Odi5?tKSWIUuN{f{eeA4? zL4ERTxs^2mXJ8-7BjS%gyRwqKg#Q^~P2SN2Sz46^WXX23zfctd_DcRkm`{O-*d?~$ z@dq9UG%21vlSj#B(XOHZVvaH!%ArJr2xuKDT#wGq&J2HtQuZecp=f}M0_q~yO^_c= zzJGPW@6^>7E8~h&lmo4%FBCt98RcoBSO!BC;lor;Vq8ZIlOh-bfLw{)l57l!+wAX# z8H3Y95i{xi8yR@`ypKa>b-l|035c*aTUV- zN8;wpi za3FPj+u9ZJ(L8GG2M@Ab6?65=VG=s!W-9MgKA$4Yn_{!!w7_k*o2vvMW)L?;ZnCI{ z(g5*%U7!1s$IAKbZaFuspD_4fe}bX`b&-Fa)&ivHfOTp$1&gkw`({rbWXFwo`v6UFcUpXRZlNn@psW%z~QR3%s1G8EP zdEalCm)BSuAi97p{_u9jh4@3`4TR(NlDP-Z!|q}}qy6_8W~W6j@+ba11!2Z3Zwf4?enUFB^wv0#drQ zsd}xqy@^bljh7AGLm~GO%7&fb6?@V=IjO^GHO06-UKzs7J#y`+$S72=;H-E z*L)66q(}}#KO0V0fj!d0&@kI70jwyNKcI@|QTL~{tbgV2vP4Jgz7ybnWn?xMpH*zs~|$XYe` z;G=6d@&WlCu^gn5*Zfk z*X@V_x3iulm*{vU>n@yQM`Ur*?~Gi2gA zDyKadP%t0<(Hnl6-P-p6F=7ALRfMu5pB!$o@6eV8j z@POAb@96lrz0P@8xLc3!ZP&LaV~Vlx8X|BO{qVSFpTvX!@Yc6(b8s!q{P?(#UpW;H zTrw{-!1VEThtS~8Z+3H&h2CTxr1Cm7_FrC58sBK}Q{-A2)%eHGy^I;{S_8k z5=s2^;81DFjfv_V2NUnS_kEGvM&8H-B1 z>ww71DDFZ)p>T-2!QIWxUe0XDF7Vo3kv<>+^ptVwUT3!yOy-P_jeQFPG2DB9y#e;- zbrM&`L8!CT?I9ua&8d2}_Z!;IoArl`EX6nVRv#$vtRgaZc6tRam-RVluf?K&3eV~; zi4}wp((i(R;#xGh$C03D-~&oInZ z#cv$v7rLg0LtnDH>k&`;r8gb#E;)qq2>GD%h$jG7p&kXby$(b25O|5C#89HXY8HG7 zdXsFE;(Zq@58}hlL5#;sVg^LIljgmJ&gg~AB#&?q_hXp1Z!lvqzPz1sZgv?+t5YxC zK{}9&&!YP>Dm`?91ae5m|8*Km&u>4!lc0lAM5p8z4>|Ew&+@2%Ch=@cARK|W2{Kr- zU~)a&h%9Wp(42BVoHQBuipYI$Bc%{J0i5noR0DkOqJB9^`LDO*5H-umuN8Qn{tnbM z?uue39?MmFdz%8p6bzL~xQD&4Ut_`OKwrE#kbwP=z$V~T5P)V(kWThjY2es4j7m(h z(sD9IqTA?tI{dPm#;qEe;E@}o7Hnz`(tYB_YLJVOO^ImJ#tyBI_4B=+ch+(?lz}uK zYI%_j1)rap@vs`BCDyFAT7cn09TS=c}H1Ah0`SsSA=wECP#sqQPEGGYHDB_TRXtK^rk{_ z;q#@A`qJ%8`)wXS?SSe=G)hQPN2TcZSd81$Mee7L@cL#1y^7>ZK0d)~-SI{ulwXBr z>F)!ElYLPQsJEiJiTPLhG?`Av&F)S=+RZP0D@#*I*lonG%1gsM73#J6zI(}kzWecK zTVqc0+=SDR#P(#*^MyY#xwf1C4CugyBxT0f=h`*$F(0s)b*ia66YA+{SaiSA^d?GO z#LMnzY7o)IuxM4y6J69YExB36^p3cfe}5wNmVc0!H;`;VE(sZSURTYwVhbq{cs6FE zVmak%F6(ys!{B)CkBGZ=xw3A11i$?Q@s_gp7wc5MK{e;^cN;9FMn7<;)&2X*sV;r4E9VW<23*w-)*g?!gsn9W9Rc z2ISy1{3++9giqBt>6$W7WM?hX0M;qbJ>?NfnH>BQS5;TKNC8^$H3OuuRg9f*UaZ9B z)zSF=cK*4Vg^I&ZjrNH(y3CqLbuuHhji))%M`;ylG6WtGX|Oalk9Ovm`U5#s^nn45 zq16PE7DL^})^KVAGbNi6w@$i*Hv8Gukn<-p-*CPa=oNg8dr!9VL?=!aKZMMxeV8hG z{;Mge^yuDdw$#YNWS_wD`MDjh_~?hx?h%!ly2&<Yicsw3apB7^GZc`f|PN4mA}9 z=yu;FaW}S4FBQV(z9hWsESQTnYHlc9TwRAnM_Qe5v5}X;ea$arQ1X#Z7YDT!R(KXO zsO;?xG+y)fb^*Whjxb8LySuv)@Gy$m;d2GDn@(~`gjWDWU5f&Q%{TW|Yinvcz`9eq z?Lx9dvtwlAvw=28X<$1NR^dgj7 zNyl(oiyGZpi|UKU>5@oG=9Z1NSr^NCVp)SC7|hXI7{aw|jbw;8v-$Zg>P!!-aoNpT zYy0q0UWLV%Tq=8+AFrn1_k0zBsnFAoCiIJbi~c%{BTw#sxIRu}cEnm=mA8vtb+v}# zWq^jyq^XROy9y#eKyl`lbkmliFX4!01?Q%P7TE^j{U28ao6@Cg57dGM^DlcdXs$oZ{rVq0_SbPS5kE(n7S0WFEm}@{1n|5umt?NJz0>hl28cgAJOnuXMhP{+d2#?Ecz zq-C!%x}b&C?;$yze-8>*mFSDCMsp|F9-YGBw>^Z?KruzC-2rY=K9G!LC$ks2h5}Jm z2ZP#{XeeBfWvyE;2RP48Eq11ZzFTc)7QE2MRLouFo63C>ltai#O~8R)`0tMqP7hHg zZg#TMK8ka|eJ3qT(9rXdhv-~-@sh?Q{v&P^W(xJmcy_h!n_ud@nyFxw-|Aet z95+R$)+RHWRNnIJN?A|NxpkSql;{-~7uOZftpr~c)pOhayifl#;TmK@(cJd> z#IDMb=VEc37V^akxA&_lb6H4%P3U zlrq#pW1+oIU}wn#w_zey8fExhz+s7T82Jn%%5W!V)jie z9Z=T{F>K=U{Ea;~e%#)9@NIM3zqsV!NY*MgWZjfjx7FTsPrk06#@*=9M$eA)gxG3r z(7IZ(#dg&$M}|D`oKo5P;$v(}x@q|v+oe7{jW>ILk5v zENtGt9|>TP1#8CUxWW01f6^sB(=N#b=eU^p|7ajrB?uwM{3OYFFsG?lY|cYWH;_Ar6ie$hz%s9Sn~!vAXE}pP#n?bF$G$ZVtIKB#sijGk-%jD%%p-# zzH*5%cB)J(=2Mju*{WHn7XkiSnTaD(BY-isVU3y`k!1S%8O(_?ZZ&c`|{G#QcyMdT|qBy zp)2`ab7kwtD9D(F)iVlloH#9VP%2{OcyhWoPu$&)Vcp-=l*R_LXUV$U3qJI}PT`XK zAV3w<5wfycFD55M-5mz@^BD{X4$mCt$+TU5HVs*Evot8Mgx60f8jyaTWd{_#p#yo{ zLk%c$#K31_Hlkswlw>FR?z)DcpyNM;aYUO%ZbIzaU&m5hbhRv5iT#aZpUdtus|sx? zWBGo=Y_XG!=Ij?iV_L;YI%NoKk(1U6W?V58UnR*9I?u_FB%a=u$>#kdMV_nskT>{P zlq8GHY)_KQ0x4Stci5HBby%IF!9vT%*0x^|7SG*$I8m&-UiJJsm;8nl#i)_;aR?+1 z!)Od_1bz~&4SIzWZnW+}+|kXJsX-ieFjZQLbhJ+wb^qd-Qor$k#dm0bETs9_!-}jT z-oNFes01Oq3yGHHZ7!82-sRYpI#_5bAzPrO>&afVUrPjung35Q!upidj>6%?;)(du zw`DKcGZ;`cKk*Gp%rz~UEhEoJiTsJvT#zv8`$B6@{1A?eia{CTkrpdh0fCl5Sf1mv zs14h*73xu$lGQbRUwJE&L%^zQ0=L0}3OUBAT_yc!vp`&W?(+u&O~dX9wJH*_m6|U^aQ*=KW2YqX_EX>Y1{S&Z$cS z0s^K&*zOQn=zd?ylUbByrT5GI&|s$IJuR1E8Ok&;{)uE=oi^mF^BCXIcDls=zQ;Rp@-eX1UF*QMR2G!Te2V5Z`y8c4SBi zlH;7jR5^}C_Jt(DbWE8KB16EHQ064_7AL{OQd;dz^#VK3&#azj87=r+WG{^ar`-4^ zUVUj?6R?Vr>A?4|vJ;6_PF^LU$3uZf4o2jZOBiz$a`;1#Fgj^?s|9kI5%>F$kq{kp z$F7j!V6-STZ%YfHTw-XP2`@+#B|Bx@M{aC&piS(sC{X zS&=pbJm_FP$Q;*iG3I`Zxzv;@7{bMegH=mEMxzc!IZvdFK|uXxh3s`H{RY@{uT z(U>>5|GnEYZ?M25G=6`d?@?REM+fq-zs>qUXf-!aLnZr8qQM) zDD*7058>7?+&Q=k$loT?q$s8v`Ztm2>?Kf}`>dMTf~w$s~Agp>6jV<5cf8K5Un-bHz$(@8%V> z`k+#Q+iL$5$0BE>{N#0M0~GKvz^N)6@ap{WT4`^d9;F%3^s6L_6*mhI0Wej$O#6jV zazS=aAH()p?*WVG=P+7bBtjBFy?wk%)%I^b2EJli32TAHw6jIWTHPA=7kVDg@z&9K zdwZ9x%s6j1;;<9#cMYMW+eH4{5j}R}vFPNXHu_N9+V%PJ1EH9*QT%>A-GcvoM8252 zq0v^EyBX_|c}(C`N6beTtz56ILAll6rv|F4w#v{NM^CxzLo-U}owaInezzZWgdvh! z**NT67Aij(tzXQZTG*b=a@e#dDtL^(mpw@EcS&5!i+SbZc4-heeAblJ98?=(C|Y>3 zW7enL`eC0VaNp8x-ik^*pR>}))#!Zcxni|28TY2AmLu6ND@wuh3@N7Pu7g}ELGZN^ z3PI!}7=xfQ9Z=>s4A1QLlS(F?nvKW8wa~Rl_qqwW$+2Au*g5cf%VKg^8z6JxsViAMG5h*-pk_9sgcH5K~Lp>)3vUo$CfUaojE9f zTYDfQV{n=0TunjAB}-;sR+saHPr=)wJiDYp_r1DH*WV`$74d^hgDmBk!D1nmU2SHW z9t*GfSI;{Z?Q)*?$hpdP7K(-#4Sd*!%~BX_@3pF+#SLUTIP-8aTk)$NH=G}{803T| z@Q6<~(H`cjp~V*ZPMXSR%Y$=T&YcA>*hOP=dg;tMW2p1bc8nMvedDB7Rzl4wcUQhO z&YmmInQDBPl58c_pzX$LU<j0#`H`I&2JxVg<7X zRK$f#C7KhbdG|f1N?|gWf0@D_FQpEz48m{fNqgPwQ)2z^=cjt&bi_K07xV+F{l~N$r*dzcGH=i= zu*RHxUaOIhoARMDN{h#U@|anM;de$m`RBMy&Gfh(Qwt5gf*4Fgt-T+8BKTZ{wAwG0 zEr<_oWI7-3U*zfT&ws6C@Kf>$UAeF|k|?}0o!ja`5fhl$6K26Q#Yk-di$rzcHmFhF znV0x}tUWAkI=_U-l2LpJPc{ zQjO2$19G$2M$Ltw7#Nn*4fPRpN}|e5pdN+@e`{J$nSTV>*c5;=6$#b_IzSa{tya)c z@$_+z#{9*}ZJUC1KTZzebK!EPhl9WFk#K!sm*C7~FR?%3J*gqtc4}RI8Q%P@q)>jN ztM!CO!B>@p7-01fn49g+tb_CGv!sq_Uab@H%}mj#o;3}*!+w63jo37)udgIErNLpH zJCowOIc2*j>`^QN*#;Avu`iZ>Q-~y#O8$jbEBPp&kZmLlab!HNrdA7Fk=t2v#9esA znZM9|&|_)k{2R_Gz_x-YTyg|ej>}`i|5x6$)cTB*W1_1J!%Ra5C3|R_(STNWfh?eV zm(z2lUJORsJem{HWkKdYauSfVS7!tAqYPW=LIsSLxbwLro)}ZwM7od_4tP`)#4iU< zCZ<0Z{M%t&08@`W;r_m6q2lsu7qdRX@<`@ToyHvQ^d&5FM$>^b+yTC`f@w64GI?s{ zW^A!CXQm0*zZz^04V@XV5(Qtq06+|vg@vU(=J{-7f4;85HJ1lq9^7xwjf3GUGfPRH z%so9ld3-nw^=V-*ta_wVJl@nANhV3=ME2n2n$zp+6q`&;iRB!Z$kj`G!!g-i>nYE z{P4l7v@+T0^R8i3yUwcLg7!!bO0`iT^1hF~?B)qu@<80;@s{*0dW(HU<`N^9ICd@U{wKxFJUeZYh$vo`sIuv+n$ zzfzPU`(E9ed>t3Ve6*{87+*x8L8=;1uZqzD`IkdJQ{>9wJ*^PB93p3!GF@6(u~!PL z9ounrI>mqQooo5GlT80>aIe52GaEqdWyqRmw z(o@J!GF>!tS?&yC#jT?O0*2UdfewQiW~ZFM`QQPAG{((Ri!9_b8i4kh@b!euQ^g-< z99I9O9=~KL+z#ujB89HW3=ehn0XK^c@JIlW_G=)yz|=p!8#YKb+F*`~6iO~wp<)7; zlZ6qn=swwhI>3)GRb0R>aQM)t_zaF5yr@tab!wCW<(V0SlrsVujqruyfax_s7?bHw zjtWBztAUUCbb}5cN|AhQMflV$TtJ=Q#&x=8V?a^iG}Ez4D;7XDCXkbplZTh@!RG*h zYH&~wzW%wm_!Pc;RM>FR@dL?(8UF2C5Ed5b_mc$-y4zRJdLx6S&s0}AqxNtStn=Qc9hOP@`xg4HGpJ%*iHhP4sK4UWS|fl7sAY&9Bw#1mHJsFCW*S$m z#v`5rkN&*Reo~b9@UMsX+u)0*Yt+Hl>f7{+BE5yb7BMXp*k8P+MTST_gaF5evNuT} z8aOYp@bH1_pW)oXg!j{-%BD)~y}FH(71|nUI-4P{m`iThg>P@WD96$LMVLLs8*eeJ zB)>RT!^6dLt?)*ZhX1v)i0vo*{WPm*$K@kz_Iw|!i*6SQYv*&&<$#AbydF5c z@S44mf_>!j6Ea$R>{Yfh`ii97jlnmW?_t1n{arp6tJa1AX;yEh+xB=qXyIDDr{3%qJe>GR^9+VXHFHPvH#b(FXq z^mm&IoGLW*A3+_|_rNlGgPbdT-BT|^%)wq=Tde6Wd@t0DP_iUgZ-HX8kH0><)8*q; zf4}ZEdOPXp0Y_Z+MJh^2i$#&|M4h7)|K-~>e0qhZw=hwW8zWgkwSV|Wv~lxj+jEur zU2tB1NvEqsVOZJq;`IZ#FQUY6$1W1_e$;jr2rUm`QVrsv)Q^U22)*rynOjQhU7Aem zASGBU!BDw(sKm938|!nLv?AoC?cWFR3@^rlog zZ7;$!Fd8e;a3Klx=g|Ga&P0S5n0OAP{i|8C$u$R;l|X%aT~lMTUV((8kj5eq`d=8L z2>k|;)nNPmC)>S+PD<2Dv?w>v{FnyIvOe053CW3CeWUaNd(5=XFPowSPv^ z``bBt$Ft8FhCYlVXf|G6sc86&iF$WQ`Eso$tEiOjc1O6rCwswzx_%{U>u10rVIDM@ zUIh!NUz6TxaOWOQ>3+(Wj}s@KwDoUD2-rm*py?xkDfoqN;Vc_%mfy*Tz^sYs+7rU* z1*@ZW##a-;aQ{emi_2%YCp54Xdz^B^{fW{dx@sOq2g|?8+OL1#MW>^5KiCw2@}9+5+~dJFW`-UlCE8wC(CKY!-$G{ zP040Lhi2Wy2iM^NyNK93S?N@A#h=<}gJ`-lvHR2_LsuC=YV!Vs4LCd4IT2SI+w{Le zIvaJUR--KL%jvzqzUb$u7&$I7KonT=}e3*^^8yV zP}mK-jU24T&k2RGU1^fJ=1gII+k7rgX}a~74{U$)Lu5uGJN|3yHL|1{Bc)Krexz6D z<(E?mEL1nL2b`#Fh$;MA50s$R^AkHF($RZ&E}aKq**}M?F!9Us5JUS8Y!%yr|61du z+j2RF^}8z@bK5LONE5R+KV211N0`e0-AnMdq^bEd;o`Do}in+zHy#(hH*wnr)XYSAZ%8N|*9T43@; z{OUPkzW>zYh4L2=71CiK_dMG$ZRYy#Dt1LqCqAh>Xl7B(Wa}j*! zUd4%F)|Fp@<{>j9O$uM7kG#+V#--$jUh_VoiOkn8PZmF9%(-txB@FD+Ag~qw0w`{iZ z5!agd+VB2-vU|c&*7jDtya$xoY)v*R)Q6Jalxv2w8Re)ie36-*^IVVi#oAb9B*+<# zP|j9td&&#ShPQ6l!uc=nLjvz#1^iD0%Z7VoJhlh^?D-1H!<&4h2W;2+4XU@{L=Of_ za?ak*HFTY?gwFTG3i5BtKX0$(<|jRx-xgXK-CkEGsJ1y2X%DAT%iH<&bURIfZ=tKP z%|2?Mw?`s)qNliB7QTO>X#=j0lT@icuf!AuabGl#(mP1{+P7}jVV6pS?rkRZ=->wP zg&|D{ z%AAPXW1qhdTowHpjrXm@Jjt!`pP#*%qOMQ1YV?;xSKR`rEdmw+2{KuGqgXJsraeqsK)JvPE8a^M2m_%-e=~3J|x2K(Bx7V=rit z*Xh#v{I6-x*1rZRZMfN&jsXucLo9w0NG-N^P}@0FY%38-rXFgxfjt>*g8C~~G=^v4 zI{j5vo&gHbtblCR%zdP>yMzLk#JPssw zBw}OSMtNGhGXEa-7b{ddSskPVd3IiU@IKwp#ha}+zaL2My=2-iPH?z6m-$Rm!XDV` z7g@4KE@}6r2K>{1l}Ahw{j)MNcgQ{IVtE77#@y(qI?5QcJlMjL80CsWyguGLvsq4Y zTini#U7KR3I2z0i#=Rfge&a{pha?PovCqRFn zh{sB#UTItG_I%E++NDK6IU~M0-YBI_F~^o;{` z_2|UBRQ)p*bj{tlml8`Mfpofo(8@7>lTisDIdm9%sb$OKn3n}1{L{s!*eg?)cJ-Sk zjTSSG2>+XO1+hBZlb`KmW9c*$ytiOcUBseyW$W zYdbWpMn;?#g?dq*O=9 zdgs`H^^I>wOifjl9Ex8D zJgk)}a2HoZUeKCM<4IgTFgtmm=J*RYoL*fJ=WMFb`umDed|`z)+j zpTdIXXUOsmm!dFEG_y+x7FI9M>FGW$CBB;~il3-*6ww=Z-zmtNA~F|*xQ3ic1(o#R z*xEblv558RggKv+Nbvb&i<+8sZmBH;@b@3x_*i z*FXGv7i`ql4}l1au5D*4fE>UtvWcLvvp;hc#kN;HPuRjNIfBUByXLT&%Ah#RbPh!_ z16?CcqLlOGWRq02J=(x!#eH9DU$V+zUgel2o{u{9t|NY-!zp!rv5ACX-1w2lB85Zp zV``(t!__zCP)WlDo}b4H<8mF}F4sdvK28xv#}jwUZ1B03o|Rn22b$_Bjd9VwVwf8dslJD+3Nab+fa58HteEl2NMTnsmrf9J=Jv~=AA9>5Znj;{d;8S)0^uEn zL##$V-1~BX^YC$&YCu#Y}K}}ItATn=#@1WJFn2&=Ny!(;eo+q6d z_5A%A23Dng8|#hV4n1}2g*S6mO1Rcuvb?)^LD#r`7%Arx^ha=mz4Z+ZZ^frHE@Skm zkL7$3(jhR>1B{7h=_$7GtM%LOc+WsbCd2Q!vCGo9gx_OEw`_VL;sJO>) zMnj!i<)^BkRR&T|UAjt3?ExkCZ(}(bhw@snWh+LDKOIs;H8m~itYE+J6bjJz<9L4M$eb46E1pR)}7Y&9|PBcUC6)ZIRHV0*c$d z`|;`wbHZ{nr_KWJCeu!BA63ufkD=Z@}@Bg)*sw z0;nrsVHv2+I?b+xr1amyt38g`vl#Php4EI<6rR&WVlft$al3 zD=6z-Ty@q@XIU^NmhwWeG(A%?Q5AsW;eE2|d5;9utfvcZu=-0NAzeSat>_LWLwDwQ z*LY_kE3v;w)Ps1mus)BKQ1a#OrVX89T_Cp|TRk(Z_N3}V_fT!>R-oaQZjbQ4tA?N*u%l^*;Jndn3YJa)*G1~3a zR$8r`C&Z0(F5iAMU#o#<7`{C!j9f5uaMVZTyhSk|@aa{{EB@4Fq#up859~B@^AG=g zap8GYE^wI3x-&ENs63a8pvqF2sqjK$Zy$faRQD<;fumlo5PyGwdbN??$`RAOQ)9#? zFfUL{1;+mU+6e@(h7h*4pL8pH(v8s zOi0AAMFu9k)MMi@>dPb|-c`}#QiT`?=;|L=c&13t7oT|9+lRPWZqYusyWTT{gl!G= zmWz#uRtl*3?Z;Ut;Xf~!I{er`{h$ri`Hb=8IE<=u(`0Fvfc7!9Wb{b4$JugnnVGU+ zY@Mz=KZmFVKSu^OyTwSs@#^GTiyHB4HQk|8;|sow0j{jY$1-LFgd7XLV}tuHGOsdd z25Fkl6bF3krb|-!UC&{jC4U|RfcmV?Vb$lt`Dj`2Tl_c#c!g zW6V*Lka3zmJLoh1kmGSbH=nP=-<*B+#U_u_X2<@O0l}00k0^(R2ZIW)zuG+XZt`cM z9Pkjf)=1K<-cMsEsd-f>O|C%#n73DnIr!_-y)@}CZP5yM8Ck~V#|eQ=4u<2+G`+(0 z$UEXd zQ2_=oC;d{HvDupSGecf?udNjSilx0DS52(s6BAJ9%4H&+$ir&Z)MmT7=-|$z(~2${ z@>w>V-hrgr3MoMbXe1eac97lc$~EqtWu<(cgK(9p-4;0d+ewWsGe3=q$d(_9KTCaD z*pa`gwzqxPK$k~MPu^yPRe;-LJ9d<&F%cseYBmea4R(b`(8Div_4NpDL?z%Nc6jl`YH@(-myz*@+630A8#~+AFoTVLgc+y zwIc`gK0Z?ck)_&-=cqfYc12)J>F*{5rS#1rmIX7f!ii@t!p5C3TDHWgTvR77fuHcU zHp;@&2&6cQYQFqjJKUxIN8ua+=)egvX0GnGgWw3JH;nRy+rEos07FJqz9S4@-0$qj z#`BC(8|9Sg4Qo}uReb4}-5PiBT_9wD20iLTMLSJ5{bhgc059&*Sv7NRThsO;8`YSC zpt%ysQ0<$sUxLYtckAqU!6cK!wmmJGP#0#c$?ru`nV}h)fr5k#-5o=BhXN7$K1N_w0J&5Q+KwfW4+L{OhO|_vB-d#5 z#FS)|i1{5qp%HH^IS7e)z`kKx@^CTCwm3ZV7`c_Kx1-cyBivoqvJWqj;0j@@YXfPA zuyBPxi?iODUBY~)WU@G$BG|DQ1$rfY9VS?P()N3v;+KHKJ6^eRMC0EF)GuXVu*T)k4V76pv;S1NYPn`XAoe-WFm( zAdWCjLeXY}1t5CfFIGhcOZG1ybu|mc19L+6TjdWPcCTt_@K|U&6Bm33bk;+GUeyYRnD8Hu>8dp*9S?6_|6 zHkfTx=NhlfY_yxuQ{ys5WN(7XZthmH?7W!u0F-C0{)#8w-^M>ZX(-j<@)X~8?4>3u zy%5mg=?G`XcO}lM4zQ9U{XKp;4`Q6|1W?xS^lXa2yK~kT7Phvf>&L%xya6kxtjT^O zXd#kpWdpur(HDd&(N!X^GpLl-S!hf4_eA+@bE30|8*6kC?f5(vJsdk*dc=-QeZSYP z|8@)6-e~mt)O%GW7793vLq_{iNTgzlDB}Yhy}5oQQDOD|+>cWN4&WdK(2A?pG`}+B z-f)rhFsLjKJ7qQcQU2I>?)~;|&3upd=uz7*XVppL>IgYng`5TUUBW z$UQN_7u`rir$5?9{$+k>i9wh=ulQ25ErRxry+Da_2_i9Yd+{+EBy~{P>o~4w^04K0 zX(ZsXvwwCq0`iP;jyCEk-RzxbeJrMaC>58s=MN(lCxeM&)T^9hjM^*&iirqZhBC#*@pPgb zHnJrh3T??5oiK#s^_YTHHqD*cXLS=b9PGp77o|p}@xS$1XA({sE#AmiR=~+@*gC6C zYw11|RD01wzDIHLHqhZ!7b?*>t0RsZA& zS^b}z3!jccR!*H6yh~&|+iR1LXaHgW^jez)l)~|Ggi;BTKNaZC)95>vR>)x_%B~jR z>MCXCK?B!hH8N}V9c=B4+sMqYdz^z6bpOhv0_!5&6)}y~Q$1L|WYZIo2I-Mo-j#JZ z!foLn&27o9JWlh|xbU@uoNWhYD$}ELB+YY~MR#d9XE5Q$EiP1)se2UvG_PPN7d)ze z&xQ>yIO12}ZkWk|0bcJUv8F<9spam6_6o=6H_o028*J4@)Wa&t>>oyFnJhY2 z%*Nt#33bI^o0TZ`*acG&v_HrqB|d)Y5R=|9n$jjO?73;A@BaE~56{FM7jA8nFG)l0 zxAKjIb6L zrapp3v6^<+^oe=AED;6oX|T2EbsmoJ9k$38Aem1>xs$N+#rTJc-8`g>+A((DNuD?_ zZb}^Vq++WTuVk_W*-RD}QMUIVWK`NsiUk}0NumJk8U(+d@Gdu+rFiniTtz^UV>qo? zxe}>A2qPhe?Ah9`keQaqbp=@;n~B6rxMnmSm|f|PW3tU#<@SkS8MQyE?cm3`uBH0Q zeX9>VsV@pNZi#2@$#7k$Ww1|J4}RE}E$?HF{4Tbm7duMdhyc=7UP6G)<6a`S*r>q1 zeiyg4$HsxxC;w*MhZ0{&bJEx+&+-1$`&rXnTrdL0kRbO_yXD=r;>)QOH+#_9pD(Dn z-sfB9KdBCGUDcA5Q#kmv?Ye6#klgNS&xzYrQl$0v5`Z>#LEVB9MIk#Hgq-Ypc%GDIQQwcbtZIrR#_oTRP!K4cOga4MZQsme|2larRb6r>hbr=Hu!GlS7p*JSS!GkO;6`Diwj30`vBCq0W6%?B%)~@l4=SOD zt)-(2CY;fSX5MCvWRDla5OG4vd8;v3*w)8~I_FpBShORFY?&Im@5P;?AM}+#WEkGGRq%Lo$w#}#WO`HPmx^?+JgUme zSXf>>tY=vO%<3Bc4~cqWtT5v^ReZVlr(x?<xE_ z#A#&qvIsjc0BAx+4jsAWAY&}>4}#;v~J-MJThT2J=Z^uDX?6v51P4^ zg6%S=jFQM}PT!;ie(Sneayt-~&SO}qnEV)tGN;%#{??=;jX9HA*gUl_S(~VoKNVP` zo(t@79f_CaI;Li@tziALu)j;Ys|5vM-JSB$XXZcx2-@Uq(^ccvuF70YFOU&sA4 zP7v0xHnZO#TI(;LTcWYT=UwA{#T7IZ);VuRgwQNn)Jx%qyrPl-YlLs;tW+#?F*RLQ zi^%Dlg!@Bc=W+GfDji+&s%$GbzcZk48Z#sxz7Ue0+S6|v69?x6!5USz4*h)&F3{+f z;X9pZNJ|qkr5}Xdu^#jx*N~E|Povkumy*g>CV88iHmD=4Ab&|VIPJk;6dM!p7Uipo zS8vbu^;*UP{hP@w=o*gtvHU*psk(#H;?225K*_Z53NuZk@AssS_Xf2g(_2l~tI>4r zF_ApF6e}~E7ro2+RxR6tUZbLP;Y)D~o3Z^QIPan*$(5<0!2%`EtH-lVT;;ffP~w~^ zw*jfOMIf)yf0{`ZI0HsaY?8;L2&H8LF|fm9rr42Wuy}-#Q8;3lMAyT2&R(1QIGosm zP3h~a-g2jXM(fcd&KftoUP7qNrSlJW?t(a zSH7uzi4L3312D-o2b2ERCYPZBe?JsaO-ip*O;FP;%Of|97D=vn0of)u|NGt0T$kE; zwUU+>VBy{qmy?U!sRX13zr||5`WyEXTDjS38(mg7R<jP&R!O(Ai`XHo_XtgLv+9gKSi`b}r75m4mGJ z^vflPPZO__%Mv%^q=2J_calV`S|6GvvAk&6ji1ikw zwk0EFOnrZM$L?9z4*rm;A0{z>530fgby;}~5v`YhFcRDlI6ql|4Q+i(J z(CAi`al6mXCI_)qF>wkQ;r{=4-np%h+wmL^Vqg@ z+E?uqs);Gr%a`JhdFVGZ<*qg%3sC!jkgIc}4RmRW!>K>r&(_J%(Klz9h%`wL`T!Q5 z>X;=X=2^HLMOdGu@M+?N>2&NLMW;6HTFX8rZq24 z3UCJ4;Iwjs8ji9RFp`_U*oqWGCf_wq*K^_rN8P4PHGf0{TSfc!Td7mW?p+A5e2|&4 zJz-dQkh{7_xN(R4kU^*``o9YD?bp3sGm9C-g~3vU2=kQ5e= z6V@ow1~1!I$FR-?CERz%>iZ~%pJo;%X6sGo3>JbO8Hjqq8b{W}COxRfHpU&_RQfcP zjPD!6)wB))O&2-bg=g5sR4P0`z02;gyGy|y(<9Pp22iZM5x?k&WEBoG-|L|$Tmy0x z_g`y5HB2-Uz^~q)g#gamhQQ*@tL+b-9dKEq=iz>8n3S0pc*6#~CZXGco@>ivmBGv7 zaaTeU=z&l!@>w*YDmhL%YnRm1+qJaKdXkT|1h0cN+X7o`nHNuE86#v%U{GmUa*y6) zZOfErEqPQ2Qejd(H<+~@D~{&g?uT9D&4^Vzqogm0>5T~?ME3}*mpOvnN!J= zC@W)`D<*a7wv)$35}h@|0?{Z4$bK@(s3{I2YVqN?O-g){fBhI2e5BQ=b9C3V5L4Q_ z-r}gyWM;`*mZ2Kwmen^}J)~6M(l~;XmTA4uQn>jw6t}J28x0|em zQyYZIc*Y-Uu8~|@Ng5Vj4|15K$g1Yo$=ujw7E{atk`akCE1rGeele%B3CCk{+8&B% zm$@cQsLHbz!y>sz1}pJy=oth}wE9-3P!n|tCey}29a%V;?6*0!U9iG2To&>o?B(=| z)n0oYu{X9FJ>}Zc_6hNQ8co^>TG1V@xI>yJ4&mtXxNh$yhfx}}R(K8{Pi772*nas9 zG+CzZlCHA-LVag1i8A04C!k>=B>`~|7r&ZyK&ac4a}cNAP0y0D%(OCtCuVA&0cuAV zNX}Cd19Q-P>ihBdV+_z$a$@I;0r$?%!Y+@4xddNb5IUPOLU9mA=OC=%qoEoxSm(|; z#9*{U?^ADQQRlQz8!zA&cvBmC8`S9Z)hy$qqt2<2n2Jh^v4=LVJ@)XZ2bS%cRYPI9 zWOHQP^b(=hb{L%h-D~>zCX`VT+C#{;bqqg_W_H3MGgA9FW^8rt8!b874i>)7FJNIl z;?5o|=vVYczRq;sB_JjuW3JG#`Djfd9`TG6=r8HjShRA0va>4AwtGvVRaLz~^vR^! z!7vxptKXF&1RdV$=>5v5TEACDK<&05zRu!*Z+nK)raWPd?Q9AgDX01-!^dPzA3C__ zRhPaVe*Ku*yXJG|*`mZ<4M$rHeQzXc6n3c6xGO52hs!+^}|9}A}G_I|Q}H%c9)btDm2-Y%{{ z>)aOUDPj~RN~n^QC;tpBiu|n4J+9`t9-O@k<-2V^Nn}q~kzlMHJDYbr;86jP)~@~9 zbOAgzWphHJ5Vc%fm@L9GV?1^X6R2&1&mga!7s6=B7O^oVPmgHb?o zcbh_w)+mE~?n-2oTmz$L#)V(ma=If+8)9yy#FVi3J?IxljqFV!#Y$K{daLij5tWV* zo~B@T+{7(VlxKZal&VBgIStKRO-kB}{n#^->D}i$Q&rX11kwBstlR=c53cOvSdAbi z>=<2;a59(76&O74m2(@pVdq#*fe38a?&StbSf@;Gnt8C?j5WIdNSQbOeKU75)&M0> z7OlMDZ!?zW{%*?RDui|+&EMNmL4C|;d-pOnU9jNeJp@bGfqLvf64qvmi6n1?eTX?M z>Ifzi;g0vjuLTH=qg_Y~5+(53-ODCG{ozoqrtI}v^eWCe=~87fwI>c-I=)u2crUo< zX+Jegi&qN(NX%|b(KdW4rLOh4l=5D(OgfC)Tz#HCBY6)aaAhKJ7#vFgYu@GK-J}(e*;(RVRMI9i$yMI`OaKBa zOe(K)f+YML>pFkIWZ;kjl;$VHOx5?RFi)ZjkOCp;>0CFvvE&Cwc0<*`PsKkGA}r|D{_4~Ox38&Cp z1bK<6m3lRJhQEAmhA^EF``LmxZpbhYWrd^?D~qnk<-isN9FC#O0XlQzxftCH&bLnF z5{;5%?t0gf`AefyyqNo&pD#AdAHC$9SP~j(P|H7Ysb5a1nUfb<6daoDPZBO}nK#!- z+x?aHw-kLaM@73B25g+5`KBu-oHSO7=xa%UM zG^jGgT#MY6#syx!fG=91q)2Qu0IUEkJ>YXJ@}amgZ<($Y?O$e{t?G?ZX1bONqgrIU z)Qc91qr8xm_uR#iyx~mLT`AOw$UMXq?MubRqRP^3#ft0TzTPfXvG3^!eLp7@7prDY zhRm*%R18XrieJvCn~X0N{yh(+zvj6*YaascP`#3L#O|p;fqXG0{6(vub=NcmQAOSK z26Q6^>Q75FEi#^v@d8=1t5Oxt%o&CEmfC+w2(lJ2?#aS4I1+`S=-8le?_?B{h)6N! zdD7>Yx!KYD@$NU-Py!Ryw$Ade9Y5%wj80l(~svW*K3>}|Tr*||p4VOeR<)j?! zyBh%27Mo;%(xRt?=>|bS5OL`#pV+aKuwyxwaKGDLB;7=*=xRjcu9Z4GoQ>9HuOAdG zqDt7^;L))|j;+Y$^cPc4oN*z*?qf>%R-GiV>jSasa!q50R!+Y8!JtjelGWHJlkgOi zxs?-6u`7YEIfcgeif`G*z(d?a6pi?o1hkjvdD#Y z37J=I;#TbL6pv%?unu+oI2wB5d05H~o?m)@fQn#p&HaEm2o6TB#1dCwPU3WY zx?2=~&@?pu^+l2tbVYAZyIm^$#cW={W%wS+)}mM zOm8?`u5W@ea|QS-tjp0CA%hB=^fy64ird8fW1BA42H_saFk!C}?T24=!sKz{u_BJ2 z<6x!dVr=h!5rdI*y0#@6MCxw&zhKvjcjGN!X)&%yqnRWyH0B z&fAE;D4Ly|JFoaoScALz+f};J_KBRi8LaIw_omgEkoCI-3q;KBDM6L_`oJtl+M33; zY2h1~t*aiF>_P{;&_!#t+z4~*b=J2ZIo#Q)V)Hb|m=9^2l@)(<91K1(D)V}mPd-ZM zIb%gZnl@6AXDaBUR5-f61}<#dA3W;B!QWKOjsJ=HL~LDhQ%9Gz`M^K5Vfz|+3=f5EAhL)7BX;Pbw__cIZZlAvAw#(n& z^L=P9ydUMw%d9l&ftjzfr>lDJTV`k_`kTr1yiW;} z0%wd8xY}M7f}{O3EHwAZrKdYKgpdMgLT_2t&AZ}{zS6WiCr_%YDjhDehc3vHMj<}& zww%;EXV-UTGcQkWRLxYk6%0EtW_C!&|MCkvo#GQI9r&U@E5j8JD7m4B7Kzp(w8`J@ z6Oa^2P~32r#SSx#pbGeLdrddjrq$R+Uz#tjd)FzO@~s^ETJTA684Ggi!X(%`!vOzn z$p6J+&I)cLD2Ke}bb7%oBxOimVs)-FkY*WUZx5fqHnt^Uc)6WIf#OL8|PMmPi`2)Ij!@b__q^T2)fp zX<7>sK9DSO5Cj|FtT3(4p}i(tI|->UIEAcG<%4)$t0Zqp4L3iDacubRKu9(J}*FU?1iH4+_08%hyLzAj!A;BK2{GU`{0 z9hL{^hH|P-uF_4CU-b&xc6cmZ#Oc)RI?t`d?~v|&Gk~}Ok^^ICISXPC{FyhF`cI5| z^zzJ@DLna|pQ|W=AxYadE-qrkZpY*=RE$EF)bu`EOWYH5DHp}5kOs;45Zq;0H(h-( zX!!B&n65a@evD%BBQ&&*0Q20C)wnRK*LXkinrMIWY5q;`256x%zY&i`A00`vvDqNj zAbfq;@726sfNL`ICU#+W_zn8e!To9)k|EDg@wP0RqOUwM7$=%z1UuWq)1vrx$l?!E zChr}c>Na4!Ot(^OnOJlZ!qw#tVw+78;2GZ9+19Iw@~uuW;=IM!0h07p(0sjlPmBy` z<+ON3vO=#fx5BKCr^2L0$Yctx%1sM?YW}e%fHuZD+mr+-LHU@X|QePvrf^2jeO=hKeS) zIySTgdvbv=s%ek=hH*Ml6X1n!ygP9yDBD#HV*12y{G?^o;sUyjW>=i(7 zDCz|)zBp;K?Cjg`+T+Y5+#YIhcxTcsiWjZd9vAskuf~qGgi}ZD=&lNjsVsFgxU2*! zP7|krz^s44(QDb#UMB-?8OiiLCLe?^0ZG2mSSlhgm$_DX+3%{v+R)yxc&tB#BnD4MiD5wS>|he z<8a;19Vc}5(wBSt+iu=IvtB}eXvb~!R4M!c*9nNg(MBu-VNwx1@pZOq_yO|$$nKBS z?b8Q}$#@!PG=j@*Mr>Pt8oqwoaX#h!>?t6OP56eTf1v5UahWL}JX?o_8W$A2N!A~w z@6)_=`{gL<2(I)ASAp%o1?Te$c*(Scv&#T5C-rO^Z9z;np@N11!e6hsyQfN6^2W<$ z=7+djNO8}J^#{ZPP&7Lq8fS9Ce78X4<5hD4teIftBw_C-BgMw%c*%}?G#OyZJC9UX zTF``)2UUBUGfKjZPYovW$j2(JhXAWjK9M3(Gx4sl!(*mA+D2*povHZ3}ujNDG0{*Vbc z_nR5gT*;QWb$*x;d?+g>xI~WOrkp!n7Ab7;e}dXTgRf%{peOjob8I813qN2>J&@_4 zI*ozcxk4D{^@8vM*(<5Z_>-9J`l0?zuw1{+v( zV>JDoo3)}`GErRyeCjoQpxW(=-~N;ko*MpFbpv_^)FyKau5qGKXSoHknq+6U|vE2T0R)7v|5IuYD6x<)c=yFLD`en(-+u(wADdO6&$A8;f0T zi3HHt+u@sYXZ#vPEZS802&}_q^;hLo9YPr@G3S(P#zrMo8x@@Mz1=MftBSM<-+jOh zkPC&=rSZNz?sdL9-+wVE+G$xo^m{F;ogF5mLP$N;;FEYyI4_70P4fU>S4~uZa|rC9 z-vZY!wtuH`8CS*rRfb@#X00IS9UTy~{g#0}zyI%BUJ&Eq`AM{N&o2~AZ>-Op1aeEf7lGLoV=Ht-0qU|{#$8peIh}_fQ~jq3?>Uk0+Z2tT8Z=s#6vb={I}hI!7?wFrijGBj$s6XU)um7y zAFy?0`?`e_o9sD>;mWul-J*8Gh+nyRrlLB#6YmGDUhqH@;vc^-JsGAYGK9r`6*Bdw z_g|kvM>wjF4&^^3>r2T>F;mMQT9(DioIB}14JV);4kW*OrPn9f9~n5lY*Qt51b!bFN3f5mvw*793i)$ zk|ONbl%){OGeNUyN-gS<@Z?QUgKy@_MV>n2Q4;pcmVI1povQlO@929MP_;yC3v44% zU{#LYW3L}#&V7I^_%eWFNPx8sfr4E^;&0GWz;CYe?NE$lj=sq?kM^kh?4|Q4JK^DO z=!}d-Z$t|7gCd^}qY$-xt#UWn=!V9(?j)$c*p+s+H0|wZ4kOkP;hFlO=&8XO@aCgw z{m}aUNspD2N$?Yn^YZS}^S*Q?pMm}zjDi8(B=UJtWTv&gDBm+=3Q-g_l^0u-xKpQT*Wf741r^uV}rk3?t_DxRNG`7q}iP(4>AmcL8RrKhyl zX~?GU_|jzlkFKLXhX>H@LL{HpR4QfuWwL%t9s)PpZ$GPj01)D2AHe=BjQHmpl-!zj{G`} zzTqEY0RJ3?lO{0A5DlHb5d!}>Y``d-!hu6`P&>%>mz==gKJHZrFiMj*8NWBV`P=V) zqj>lLWxNfY(Rx1_uzw!r-(ML6#L9L0JIH?y?vM4DI&V}UR$Td;w9U^Y_{)UPW=Lw0 zul|>*B!1tDuun9fXeYWh?@*`eNc8P&8n}*IU4et-vr#eE^Upy!g|yNd`4i#Ne|!2? zEiiZqE^EZEeAroPybq0Y{IxfJ&QNQ-B$8N&(GF>Z@F@FzA#2VTg`GQ;0YHML;ps-$ z;|&QSE6G`+8z*J!`uT>UoeU1qi~sRmTS?A`^h0|tf#^MPo)&$q#md)fDz~=;17PSD z6s)5hJmv2Cvs2w4+wd<}!fe^Kn|3C8PyB!O==k$Z=dZj=YoGq@8_qtu`uJ|?@}s1; z+V@|z<}~F>0!C2adM%$uvKh*(Zf=f&T$!|bS~}P@mhs>~LV1H0@(C5>7Q0n=&mK$T zLDyy)#PmfX)Yxn6qFCPP_a^yl(*~(>FRTh4l4ftJR_E^@=6p5k!9kR<4;9M~s;l?! zAD=mPz)7X>E?k;ooCOFD&5O3)u3euNF;rYA0YM#`Ju13!|7kiYPD;VG7af}S85vdR z9p!}`%>UuS-2Z%|a(zZC%O}w4mtS{qtMa8Onmmt1w;_q>sjI#B6=lBIH~19?DSvis zj@=6igHM|)F8HowxKvu(*Tph^C{im%CNk~Dm_+W4a0(eTG}&YI(E_^zc5FYi@!&HSF`MZ;!v&__W$aC%|5yzx-aQoZfh24KSUe3(~Cbol>N4Dx*gW( z)bW{QL9geI%TrAAeiPVar+DIluxeKdjlBvrbrx6c@|B|oGrntZ;`>LjCEE7JmCn-* zEG|n3lR9>r&x+S{m!|i3Xr4kanR6(du^N>S)4ER+qrPtQx-;8d$k7*4>+!mLA?0t)nUOrIN zt`P6qwaPxbzcyoZ#2Ru&KZAf-a$DR_X>h5esyqgV2)$%{xK{|R~_$c$wG zpjY1h$#dQVc87Rxu!7k5Va<4o-yFYUdSJ_>xStF?PY&N?@lqG#g1nk$6KqxCX|*J4 z>B}lA8(d*hq4vO9%Z?5tf50}&U_=6sH#e5h>Q+^eh*2%6v$8PZ@&k*4sK*WZeid?v zWN~hWT*614tD$u_<|7Y;Z}Ey%Y|ea$LXFsc>SmYFl=@MqsmdJL&Cn8N7>2d&8@yGP z!tfNo)7>T<9STcuy~Amuz_WBUfd6|{2{~sXAGZM zPZ-YzY3(P8r>pips`>2{D&a?}_oS2*WJ5g#a?KJdQkvkwF(aK&;w@brZ$=6`TCH|jxD zuPZvvL{73sDtqNfUil_dM4|X}@fXE&jO97bBX2K)J%1>tl$xkG5XGv+5?feD_4*4n zwlaFHjt4FkJu192LOnDmlpp?@B#@ zCUIt9=xcA(4J)n>4s#XhL-U<)SDEbVb+j^!HM{j6mvAJN>!?wSFytbwR`#L^iY)t+ zbslUg3Zn_6A*|#R|`A^jJM z_TN3}!%e*HVtI^z3)4Y$1iv4fLA7hKkGk~gKh4nZJ5Lgi_#Ao!$W{NA2Dwdp&M@xG z{lfQe386puS^t+uOeJK3Kl@}eite2P9ihXbT7BuX>bLJHQhK#{J(*GBsK7>lx;!1o zsQle_d*Hq8pY;s?*ey<(uZ(l@TW}j@xvmoKGt4A1g2v>>R(cP)+ z70St@uk9xPb-&(Li2KzrSYeb&S@U_HD40KqTyE$+{_&lO$*QbiN2LN zx?*5kn*oNcT_l=2d{;#ZXmFof_m-BI0YLu+ePLG9!wmb4T(0QdTyBMDBfsS1eBJX8o~u@lY+2}Osp zjqg|d=?wh+RlEF!nw-;b0#q)a)lbJaxpU8UVi}As&Nq~GFw*zR_wb7?4A1Q~SGpXT zD+T$n>>t9B6)`$=!nv4se%<-r^kJ}51G1oV9$IJ*0G-{pSFj4+CQQvop z-D-Zp7A@*XH+l5|Tu%=Bc=`^61HS_@H)JYt46+h6Yaz!0;VB<@hWHohP0 z1}+G7gVaEiv_JnKtoM7F(8O|JelX(4lXD>0ox+mnMm=SKND*aes}Iu?s^_e$ZAr$x zR-xFAzN2$tqxhm}bQVA(owUE){dqb_mC{NrC3=BZzdIp&bzxi&ldU*;x!h&O#^40W zQ*rj=h@Gc#@-C}hPHCf9b*xUFt{U^r6b>VSwTD=Bz}_?&HM2K&`(+o@L#P!}A9_U= zf@%?KOIQ9SZ+Y7ke`+QZ4F6%AA+eur*qIqAZX1VVj*YG1>|@ne%Z(DqIpbR42}+zR zzf;Qv71oUucBG0?@s`PQA6;~=j%BhUTM?+vEt>gw)(180*NxRbu6{Jxqr=LI6S*1f z$jtfabPf#f&dg;}PfyZnmNQU{68r=d@X7}6)9^Rt7llR@=_!F$Mn?)4IpdTkbo_K7 zr>FYJ)w-D8WAo-=N01T7zb13sR1LYl?<*0{>m2a$&?Rl$1FM&HGkLE#OC@=#kap}I zBY?>YksgLMQs;ve^rNY~E0hKX-!i&}{n8!DTQM$iQ1LdB)#%mr=LbG~9 z#P{mwHyD<&g@JW&iszuPh1dKBbe6Z<;;q_+3}!u*sKc3yGzS|7c>N>Po(`yf4HsU~ z{KI@n_jkrXwR_(|h2Lr9E92sWLACgO2>27__n18@VT~jqA7-Nl1KpAp^KQfFmCCvWD_ISu~7c0KcHrUlO?5wj;<VkIl(|b#cjyhp4`3@Db#vvOn^^5Div-Xn+lsCZl6*#vG5wfOw)DXA z10S+jTaUrLFQ7*`ae$$fmM%H&iA-m*fPXQLWJ_k!X!2gyGvGUu$6CHHQG;n&E{h`0 zC5};}_Ry{_k#gHrQywMNu)AawW?^ErA`0S^{Hlo5=9l|t)aLQ#kK||8s;m7qkMcYn z##u&--}DR*@Gi(5s*cpHUZcPbDG#=1i)GlBjmJHF?QS$fAh5oB z<(Au*>-<)$jqBW0xeBh4xi{OGBvj^?1P%rPj_D>7;BgE2paVayU_bO^`VC+D{=zoV z?_aKoR_L8tl>4})QQBr@C3SBqEB5+4chK1Os zYok0|miDRdO)vSg-A+-+URPjIHFe;qO3--}RJ}5zB~Ht)TQ9KA;z`fY!ChCfcW>Mz zd!ow@u4z(er+IJuMd39@D>zj3vfqBSd2bYg>g@Pc7?1b-=#Er>S>#hp@qYg-ck-1+ z4tG!<*=Tk=F23C2G07vY)G96J(38PSaOD_o=47lV54PZO-h*UrP;{M*%h#2;#kf^= zZC%xkLl2SIiCp4;u`*j(mx%N4kPbvTKIM|lg}nr_O_iFdESkk83dJs(D3^)V39uMl z?1>J6HR)rud)Cu?sVxkS@;%XTI^sB?rlIP+_7C*}u97)eET(!XxF#kYIuorGjePP+ zUxvv~(lECZ^|6+dZnRj0YP0e=2PJ0yJ%rVlsqXj*_pvib)KeCJ0=J?_O&xUJgmaeY z>dj?Am3;@Sa`royVS{r~ZZfUNCBrP|wn%MJq(Q=`hWnJifz$H>^}J2k&40==Ro6104rw`Br|+q zf&~c=H0iAp|KQCk-GApVUiKexT!25_?HA!_pv^RVyu#{C7E1M7>x@^P=WURO8yq7V z{&}|p+1B>4%;QIV4bzXV;Z~Zj1;G b{8K^-wv%tiO%3p0fPadzYEO!#p9TIO>b9UB diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytriton_utils.py b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytriton_utils.py index d9ab18d9..1a568169 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/pytriton_utils.py +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/pytriton_utils.py @@ -20,7 +20,7 @@ import socket import time from multiprocessing import Process -from typing import Dict, List, Optional, Tuple +from typing import Callable, Dict, List, Optional, Tuple import psutil from pyspark import RDD @@ -28,9 +28,6 @@ from pytriton.client import ModelClient -DEFAULT_WAIT_RETRIES = 10 -DEFAULT_WAIT_TIMEOUT = 5 - logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" ) @@ -38,11 +35,11 @@ def _start_triton_server( - triton_server_fn: callable, + triton_server_fn: Callable, model_name: str, + wait_retries: int, + wait_timeout: int, model_path: Optional[str] = None, - max_retries: int = DEFAULT_WAIT_RETRIES, - wait_timeout: int = DEFAULT_WAIT_TIMEOUT, ) -> List[tuple]: """Task to start Triton server process on a Spark executor.""" sig = inspect.signature(triton_server_fn) @@ -78,7 +75,7 @@ def _find_ports(start_port: int = 7000) -> List[int]: client = ModelClient(f"http://localhost:{ports[0]}", model_name) - for _ in range(max_retries): + for _ in range(wait_retries): try: client.wait_for_model(wait_timeout) client.close() @@ -93,20 +90,20 @@ def _find_ports(start_port: int = 7000) -> List[int]: def _stop_triton_server( server_pids_ports: Dict[str, Tuple[int, List[int]]], - max_retries: int = DEFAULT_WAIT_RETRIES, - retry_delay: int = DEFAULT_WAIT_TIMEOUT, + wait_retries: int, + wait_timeout: int, ) -> List[bool]: """Task to stop the Triton server on a Spark executor.""" hostname = socket.gethostname() pid, _ = server_pids_ports.get(hostname) assert pid is not None, f"No server PID found for host {hostname}" - for _ in range(max_retries): + for _ in range(wait_retries): try: os.kill(pid, signal.SIGTERM) except OSError: return [True] - time.sleep(retry_delay) + time.sleep(wait_timeout) return [False] # Failed to terminate or timed out @@ -137,6 +134,9 @@ class TritonServerManager: >>> print(f"Server shutdown success: {success}") """ + DEFAULT_WAIT_RETRIES = 10 + DEFAULT_WAIT_TIMEOUT = 5 + def __init__( self, num_nodes: int, model_name: str, model_path: Optional[str] = None ): @@ -221,7 +221,10 @@ def _use_stage_level_scheduling(self, rdd: RDD, task_gpus: float = 1.0) -> RDD: return rdd.withResources(rp) def start_servers( - self, triton_server_fn: callable + self, + triton_server_fn: Callable, + wait_retries: int = DEFAULT_WAIT_RETRIES, + wait_timeout: int = DEFAULT_WAIT_TIMEOUT, ) -> Dict[str, Tuple[int, List[int]]]: """ Start Triton servers across the cluster. @@ -244,6 +247,8 @@ def start_servers( lambda _: _start_triton_server( triton_server_fn=triton_server_fn, model_name=model_name, + wait_retries=wait_retries, + wait_timeout=wait_timeout, model_path=model_path, ) ) @@ -252,7 +257,11 @@ def start_servers( return self._server_pids_ports - def stop_servers(self) -> List[bool]: + def stop_servers( + self, + wait_retries: int = DEFAULT_WAIT_RETRIES, + wait_timeout: int = DEFAULT_WAIT_TIMEOUT, + ) -> List[bool]: """ Stop all Triton servers across the cluster. @@ -268,7 +277,13 @@ def stop_servers(self) -> List[bool]: stop_success = ( node_rdd.barrier() - .mapPartitions(lambda _: _stop_triton_server(server_pids_ports)) + .mapPartitions( + lambda _: _stop_triton_server( + server_pids_ports=server_pids_ports, + wait_retries=wait_retries, + wait_timeout=wait_timeout, + ) + ) .collect() ) From 3ed95bd3fdd1c7f0a350f5fa0149a1342ca8fa9a Mon Sep 17 00:00:00 2001 From: "Rishi C." <77904151+rishic3@users.noreply.github.com> Date: Wed, 19 Feb 2025 15:54:49 -0800 Subject: [PATCH 10/12] Add Qwen-2.5 Notebook (#496) Adding Qwen notebook, mainly to demonstrate how to leverage system prompts/chat templates for batch inference. Also serves as a non-gated and faster alternative to Gemma. --------- Signed-off-by: Rishi Chandra --- .../Spark-DL/dl_inference/README.md | 23 +- .../huggingface/qwen-2.5-7b_torch.ipynb | 923 ++++++++++++++++++ 2 files changed, 935 insertions(+), 11 deletions(-) create mode 100644 examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/qwen-2.5-7b_torch.ipynb diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md b/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md index 451256c2..ea696c48 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md @@ -43,17 +43,18 @@ Below is a full list of the notebooks with links to the examples they are based | | Framework | Notebook Name | Description | Link | ------------- | ------------- | ------------- | ------------- | ------------- -| 1 | HuggingFace | DeepSeek-R1 | LLM batch inference using the DeepSeek-R1-Distill-Llama reasoning model. | [Link](https://huggingface.co/deepseek-ai/DeepSeek-R1) -| 2 | HuggingFace | Gemma-7b | LLM batch inference using the lightweight Google Gemma-7b model. | [Link](https://huggingface.co/google/gemma-7b-it) -| 3 | HuggingFace | Sentence Transformers | Sentence embeddings using SentenceTransformers in Torch. | [Link](https://huggingface.co/sentence-transformers) -| 4+5 | HuggingFace | Conditional Generation | Sentence translation using the T5 text-to-text transformer for both Torch and Tensorflow. | [Link](https://huggingface.co/docs/transformers/model_doc/t5#t5) -| 6+7 | HuggingFace | Pipelines | Sentiment analysis using Huggingface pipelines for both Torch and Tensorflow. | [Link](https://huggingface.co/docs/transformers/quicktour#pipeline-usage) -| 8 | PyTorch | Image Classification | Training a model to predict clothing categories in FashionMNIST, and deploying with Torch-TensorRT accelerated inference. | [Link](https://pytorch.org/tutorials/beginner/basics/quickstart_tutorial.html) -| 9 | PyTorch | Housing Regression | Training and deploying a model to predict housing prices in the California Housing Dataset, and deploying with Torch-TensorRT accelerated inference. | [Link](https://github.com/christianversloot/machine-learning-articles/blob/main/how-to-create-a-neural-network-for-regression-with-pytorch.md) -| 10 | Tensorflow | Image Classification | Training and deploying a model to predict hand-written digits in MNIST. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/save_and_load.ipynb) -| 11 | Tensorflow | Keras Preprocessing | Training and deploying a model with preprocessing layers to predict likelihood of pet adoption in the PetFinder mini dataset. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/preprocessing_layers.ipynb) -| 12 | Tensorflow | Keras Resnet50 | Deploying ResNet-50 to perform flower recognition from flower images. | [Link](https://docs.databricks.com/en/_extras/notebooks/source/deep-learning/keras-metadata.html) -| 13 | Tensorflow | Text Classification | Training and deploying a model to perform sentiment analysis on the IMDB dataset. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/text_classification.ipynb) +| 1 | HuggingFace | DeepSeek-R1 | LLM batch inference using the DeepSeek-R1-Distill-Llama reasoning model to solve word problems. | [Link](https://huggingface.co/deepseek-ai/DeepSeek-R1) +| 2 | HuggingFace | Qwen-2.5-7b | LLM batch inference using the Qwen-2.5-7b model for text summarization. | [Link](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) +| 3 | HuggingFace | Gemma-7b | LLM batch inference using the Google Gemma-7b model for code comprehension tasks. | [Link](https://huggingface.co/google/gemma-7b-it) +| 4 | HuggingFace | Sentence Transformers | Sentence embeddings using SentenceTransformers in Torch. | [Link](https://huggingface.co/sentence-transformers) +| 5+6 | HuggingFace | Conditional Generation | Sentence translation using the T5 text-to-text transformer (Torch and Tensorflow). | [Link](https://huggingface.co/docs/transformers/model_doc/t5#t5) +| 7+8 | HuggingFace | Pipelines | Sentiment analysis using Huggingface pipelines (Torch and Tensorflow). | [Link](https://huggingface.co/docs/transformers/quicktour#pipeline-usage) +| 9 | PyTorch | Image Classification | Training a model to predict clothing categories in FashionMNIST, and deploying with Torch-TensorRT accelerated inference. | [Link](https://pytorch.org/tutorials/beginner/basics/quickstart_tutorial.html) +| 10 | PyTorch | Housing Regression | Training and deploying a model to predict housing prices in the California Housing Dataset, and deploying with Torch-TensorRT accelerated inference. | [Link](https://github.com/christianversloot/machine-learning-articles/blob/main/how-to-create-a-neural-network-for-regression-with-pytorch.md) +| 11 | Tensorflow | Image Classification | Training and deploying a model to predict hand-written digits in MNIST. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/save_and_load.ipynb) +| 12 | Tensorflow | Keras Preprocessing | Training and deploying a model with preprocessing layers to predict likelihood of pet adoption in the PetFinder mini dataset. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/preprocessing_layers.ipynb) +| 13 | Tensorflow | Keras Resnet50 | Deploying ResNet-50 to perform flower recognition from flower images. | [Link](https://docs.databricks.com/en/_extras/notebooks/source/deep-learning/keras-metadata.html) +| 14 | Tensorflow | Text Classification | Training and deploying a model to perform sentiment analysis on the IMDB dataset. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/text_classification.ipynb) ## Running Locally diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/qwen-2.5-7b_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/qwen-2.5-7b_torch.ipynb new file mode 100644 index 00000000..faf5c51d --- /dev/null +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/qwen-2.5-7b_torch.ipynb @@ -0,0 +1,923 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "# PySpark LLM Inference: Qwen-2.5 Text Summarization\n", + "\n", + "In this notebook, we demonstrate distributed batch inference with [Qwen-2.5](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct), using open weights on Huggingface.\n", + "\n", + "The Qwen-2.5-7b-instruct is an instruction-fine-tuned version of the Qwen-2.5-7b base model. We'll show how to use the model to perform text summarization.\n", + "\n", + "**Note:** Running this model on GPU with 16-bit precision requires **~16GB** of GPU RAM. Make sure your instances have sufficient GPU capacity." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The dataset we'll use requires Zstandard compression." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting zstandard\n", + " Downloading zstandard-0.23.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.0 kB)\n", + "Downloading zstandard-0.23.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m66.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: zstandard\n", + "Successfully installed zstandard-0.23.0\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install zstandard" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Manually enable Huggingface tokenizer parallelism to avoid disabling with PySpark parallelism.\n", + "# See (https://github.com/huggingface/transformers/issues/5486) for more info. \n", + "import os\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check the cluster environment to handle any platform-specific configurations." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "on_databricks = os.environ.get(\"DATABRICKS_RUNTIME_VERSION\", False)\n", + "on_dataproc = os.environ.get(\"DATAPROC_IMAGE_VERSION\", False)\n", + "on_standalone = not (on_databricks or on_dataproc)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For cloud environments, set the huggingface cache dir to DBFS/GCS so that executors can load the model from cache." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "if on_databricks:\n", + " hf_home = \"/dbfs/FileStore/hf_home\"\n", + " dbutils.fs.mkdirs(hf_home)\n", + " os.environ[\"HF_HOME\"] = hf_home\n", + "elif on_dataproc:\n", + " hf_home = \"/mnt/gcs/hf_home\"\n", + " os.mkdir(hf_home) if not os.path.exists(hf_home) else None\n", + " os.environ[\"HF_HOME\"] = hf_home" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Warmup: Running locally\n", + "\n", + "**Note**: If the driver node does not have sufficient GPU capacity, proceed to the PySpark section." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "352b738e1a2442b0a997467aaf6eb0ad", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading checkpoint shards: 0%| | 0/4 [00:00system\\n\"),\n", + " lit(system_prompt),\n", + " lit(\"<|im_end|>\\n<|im_start|>user\\n\"),\n", + " col(\"value\"),\n", + " lit(\"<|im_end|>\\n<|im_start|>assistant\\n\")\n", + " ).alias(\"prompt\")\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "<|im_start|>system\n", + "You are a knowledgeable AI assistant. Your job is to create a 2-3 sentence summary \n", + "of a research abstract that captures the main objective, methodology, and key findings, using clear \n", + "language while preserving technical accuracy and quantitative results.<|im_end|>\n", + "<|im_start|>user\n", + "Epidemiology of hypoxaemia in children with acute lower respiratory infection.\n", + "To determine the prevalence of hypoxaemia in children aged under 5 years suffering acute lower respiratory infections (ALRI), the risk factors for hypoxaemia in children under 5 years of age with ALRI, and the association of hypoxaemia with an increased risk of dying in children of the same age. Systematic review of the published literature. Out-patient clinics, emergency departments and hospitalisation wards in 23 health centres from 10 countries. Cohort studies reporting the frequency of hypoxaemia in children under 5 years of age with ALRI, and the association between hypoxaemia and the risk of dying. Prevalence of hypoxaemia measured in children with ARI and relative risks for the association between the severity of illness and the frequency of hypoxaemia, and between hypoxaemia and the risk of dying. Seventeen published studies were found that included 4,021 children under 5 with acute respiratory infections (ARI) and reported the prevalence of hypoxaemia. Out-patient children and those with a clinical diagnosis of upper ARI had a low risk of hypoxaemia (pooled estimate of 6% to 9%). The prevalence increased to 31% and to 43% in patients in emergency departments and in cases with clinical pneumonia, respectively, and it was even higher among hospitalised children (47%) and in those with radiographically confirmed pneumonia (72%). The cumulated data also suggest that hypoxaemia is more frequent in children living at high altitude. Three papers reported an association between hypoxaemia and death, with relative risks varying between 1.4 and 4.6. Papers describing predictors of hypoxaemia have focused on clinical signs for detecting hypoxaemia rather than on identifying risk factors for developing this complication. Hypoxaemia is a common and potentially lethal complication of ALRI in children under 5, particularly among those with severe disease and those living at high altitude. Given the observed high prevalence of hypoxaemia and its likely association with increased mortality, efforts should be made to improve the detection of hypoxaemia and to provide oxygen earlier to more children with severe ALRI.<|im_end|>\n", + "<|im_start|>assistant\n", + "\n" + ] + } + ], + "source": [ + "print(df.take(1)[0].prompt)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "data_path = \"spark-dl-datasets/pubmed_abstracts\"\n", + "if on_databricks:\n", + " dbutils.fs.mkdirs(\"/FileStore/spark-dl-datasets\")\n", + " data_path = \"dbfs:/FileStore/\" + data_path\n", + "\n", + "df.write.mode(\"overwrite\").parquet(data_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Triton Inference Server\n", + "In this section, we demonstrate integration with the [Triton Inference Server](https://developer.nvidia.com/nvidia-triton-inference-server), an open-source, GPU-accelerated serving solution for DL. \n", + "We use [PyTriton](https://github.com/triton-inference-server/pytriton), a Flask-like framework that handles client/server communication with the Triton server. \n", + "\n", + "The process looks like this:\n", + "- Distribute a PyTriton task across the Spark cluster, instructing each node to launch a Triton server process.\n", + "- Define a Triton inference function, which contains a client that binds to the local server on a given node and sends inference requests.\n", + "- Wrap the Triton inference function in a predict_batch_udf to launch parallel inference requests using Spark.\n", + "- Finally, distribute a shutdown signal to terminate the Triton server processes on each node.\n", + "\n", + "\"drawing\"" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Import the helper class from pytriton_utils.py:" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "sc.addPyFile(\"pytriton_utils.py\")\n", + "\n", + "from pytriton_utils import TritonServerManager" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define the Triton Server function:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "def triton_server(ports):\n", + " import time\n", + " import signal\n", + " import torch\n", + " import numpy as np\n", + " from pytriton.decorators import batch\n", + " from pytriton.model_config import DynamicBatcher, ModelConfig, Tensor\n", + " from pytriton.triton import Triton, TritonConfig\n", + " from pyspark import TaskContext\n", + " from transformers import AutoModelForCausalLM, AutoTokenizer\n", + "\n", + " print(f\"SERVER: Initializing model on worker {TaskContext.get().partitionId()}.\")\n", + " model = AutoModelForCausalLM.from_pretrained(\n", + " \"Qwen/Qwen2.5-7B-Instruct\",\n", + " torch_dtype=torch.bfloat16,\n", + " device_map=\"auto\"\n", + " )\n", + " tokenizer = AutoTokenizer.from_pretrained(\"Qwen/Qwen2.5-7B-Instruct\", padding_side=\"left\")\n", + "\n", + " @batch\n", + " def _infer_fn(**inputs):\n", + " prompts = np.squeeze(inputs[\"prompts\"]).tolist()\n", + " print(f\"SERVER: Received batch of size {len(prompts)}\")\n", + " decoded_prompts = [p.decode(\"utf-8\") for p in prompts]\n", + " tokenized_inputs = tokenizer(decoded_prompts, padding=True, return_tensors=\"pt\").to(model.device)\n", + " generated_ids = model.generate(**tokenized_inputs, max_new_tokens=256)\n", + " outputs = tokenizer.batch_decode(generated_ids[:, tokenized_inputs.input_ids.shape[1]:], skip_special_tokens = True)\n", + " return {\n", + " \"outputs\": np.array(outputs).reshape(-1, 1)\n", + " }\n", + "\n", + " workspace_path = f\"/tmp/triton_{time.strftime('%m_%d_%M_%S')}\"\n", + " triton_conf = TritonConfig(http_port=ports[0], grpc_port=ports[1], metrics_port=ports[2])\n", + " with Triton(config=triton_conf, workspace=workspace_path) as triton:\n", + " triton.bind(\n", + " model_name=\"qwen-2.5\",\n", + " infer_func=_infer_fn,\n", + " inputs=[\n", + " Tensor(name=\"prompts\", dtype=object, shape=(-1,)),\n", + " ],\n", + " outputs=[\n", + " Tensor(name=\"outputs\", dtype=object, shape=(-1,)),\n", + " ],\n", + " config=ModelConfig(\n", + " max_batch_size=64,\n", + " batcher=DynamicBatcher(max_queue_delay_microseconds=5000), # 5ms\n", + " ),\n", + " strict=True,\n", + " )\n", + "\n", + " def _stop_triton(signum, frame):\n", + " print(\"SERVER: Received SIGTERM. Stopping Triton server.\")\n", + " triton.stop()\n", + "\n", + " signal.signal(signal.SIGTERM, _stop_triton)\n", + "\n", + " print(\"SERVER: Serving inference\")\n", + " triton.serve()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Start Triton servers" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Specify the number of nodes in the cluster.** \n", + "Following the README, the example standalone cluster uses 1 node. The example Databricks/Dataproc cluster scripts use 4 nodes by default. " + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# Change based on cluster setup\n", + "num_nodes = 1 if on_standalone else 4" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `TritonClusterManager` will handle the lifecycle of Triton server instances across the Spark cluster:\n", + "- Find available ports for HTTP/gRPC/metrics\n", + "- Deploy a server on each node via stage-level scheduling\n", + "- Gracefully shutdown servers across nodes" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "model_name = \"qwen-2.5\"\n", + "server_manager = TritonServerManager(num_nodes=num_nodes, model_name=model_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-16 11:49:25,237 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n", + "2025-02-16 11:49:25,239 - INFO - Starting 1 servers.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/plain": [ + "{'cb4ae00-lcedt': (3490378, [7000, 7001, 7002])}" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Returns {'hostname', (server_pid, [http_port, grpc_port, metrics_port])}\n", + "server_manager.start_servers(triton_server, wait_retries=24) # allow up to 2 minutes for model loading" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Define client function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Get the hostname -> url mapping from the server manager:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "host_to_grpc_url = server_manager.host_to_grpc_url # or server_manager.host_to_http_url" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "def triton_fn(model_name, host_to_url):\n", + " import socket\n", + " import numpy as np\n", + " from pytriton.client import ModelClient\n", + "\n", + " url = host_to_url[socket.gethostname()]\n", + " print(f\"Connecting to Triton model {model_name} at {url}.\")\n", + "\n", + " def infer_batch(inputs):\n", + " with ModelClient(url, model_name, inference_timeout_s=500) as client:\n", + " flattened = np.squeeze(inputs).tolist()\n", + " # Encode batch\n", + " encoded_batch = [[text.encode(\"utf-8\")] for text in flattened]\n", + " encoded_batch_np = np.array(encoded_batch, dtype=np.bytes_)\n", + " # Run inference\n", + " result_data = client.infer_batch(encoded_batch_np)\n", + " result_data = np.squeeze(result_data[\"outputs\"], -1)\n", + " return result_data\n", + " \n", + " return infer_batch" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "generate = predict_batch_udf(partial(triton_fn, model_name=model_name, host_to_url=host_to_grpc_url),\n", + " return_type=StringType(),\n", + " input_tensor_shapes=[[1]],\n", + " batch_size=8)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Load DataFrame\n", + "\n", + "We'll parallelize over a small set of prompts for demonstration." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "df = spark.read.parquet(data_path).limit(64).repartition(8)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Run Inference" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 10:=====================> (3 + 5) / 8]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 10.5 ms, sys: 6.63 ms, total: 17.1 ms\n", + "Wall time: 23.7 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "%%time\n", + "# first pass caches model/fn\n", + "preds = df.withColumn(\"outputs\", generate(col(\"prompt\")))\n", + "results = preds.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[Stage 16:=====================> (3 + 5) / 8]\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 8.1 ms, sys: 4.47 ms, total: 12.6 ms\n", + "Wall time: 21.7 s\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "%%time\n", + "preds = df.withColumn(\"outputs\", generate(col(\"prompt\")))\n", + "results = preds.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Q: <|im_start|>system\n", + "You are a knowledgeable AI assistant. Your job is to create a 2-3 sentence summary \n", + "of a research abstract that captures the main objective, methodology, and key findings, using clear \n", + "language while preserving technical accuracy and quantitative results.<|im_end|>\n", + "<|im_start|>user\n", + "Oral health promotion evaluation--time for development.\n", + "Increasing emphasis is now being placed upon the evaluation of health service interventions to demonstrate their effects. A series of effectiveness reviews of the oral health education and promotion literature has demonstrated that many of these interventions are poorly and inadequately evaluated. It is therefore difficult to determine the effectiveness of many interventions. Based upon developments from the field of health promotion research this paper explores options for improving the quality of oral health promotion evaluation. It is essential that the methods and measures used in the evaluation of oral health promotion are appropriate to the intervention. For many oral health promotion interventions clinical measures and methods of evaluation may not be appropriate. This paper outlines an evaluation framework which can be used to assess the range of effects of oral health promotion programmes. Improving the quality of oral health promotion evaluation is a shared responsibility between researchers and those involved in the provision of programmes. The provision of adequate resources and training are essential requirements for this to be successfully achieved.<|im_end|>\n", + "<|im_start|>assistant\n", + " \n", + "\n", + "A: This research aims to improve the evaluation of oral health promotion programs by developing an appropriate framework. It explores how methods and measures should align with the specific nature of these interventions, emphasizing that both researchers and program providers must collaborate to ensure adequate resources and training are available for high-quality evaluations. \n", + "\n" + ] + } + ], + "source": [ + "print(f\"Q: {results[0].prompt} \\n\")\n", + "print(f\"A: {results[0].outputs} \\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Shut down server on each executor" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-16 11:51:42,365 - INFO - Requesting stage-level resources: (cores=5, gpu=1.0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-16 11:51:47,609 - INFO - Sucessfully stopped 1 servers. \n" + ] + }, + { + "data": { + "text/plain": [ + "[True]" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "server_manager.stop_servers()" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "if not on_databricks: # on databricks, spark.stop() puts the cluster in a bad state\n", + " spark.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "spark-dl-torch", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 13ff674d96d7e967fab4997df27851f64b808a04 Mon Sep 17 00:00:00 2001 From: liyuan <84758614+nvliyuan@users.noreply.github.com> Date: Thu, 20 Feb 2025 15:19:17 +0800 Subject: [PATCH 11/12] [Doc] Update plugin versions for 25.02.1 [skip ci] (#499) The PR is to update the plugin version to v25.02.1 Signed-off-by: liyuan Signed-off-by: liyuan --- .../xgboost-examples/csp/databricks/databricks.md | 4 ++-- docs/get-started/xgboost-examples/csp/databricks/init.sh | 2 +- .../xgboost-examples/on-prem-cluster/kubernetes-scala.md | 2 +- .../prepare-package-data/preparation-python.md | 2 +- .../prepare-package-data/preparation-scala.md | 2 +- examples/ML+DL-Examples/Optuna-Spark/README.md | 4 ++-- .../Optuna-Spark/optuna-examples/databricks/init_optuna.sh | 4 ++-- .../optuna-examples/databricks/start_cluster.sh | 2 +- .../Optuna-Spark/optuna-examples/optuna-dataframe.ipynb | 4 ++-- examples/ML+DL-Examples/Spark-Rapids-ML/pca/README.md | 2 +- .../ML+DL-Examples/Spark-Rapids-ML/pca/notebooks/pca.ipynb | 6 +++--- .../micro-benchmarks/notebooks/micro-benchmarks-gpu.ipynb | 2 +- examples/SQL+DF-Examples/tpcds/README.md | 2 +- examples/SQL+DF-Examples/tpcds/notebooks/TPCDS-SF10.ipynb | 2 +- examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md | 2 +- examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml | 2 +- .../agaricus/notebooks/python/agaricus-gpu.ipynb | 2 +- .../mortgage/notebooks/python/MortgageETL.ipynb | 4 ++-- .../mortgage/notebooks/python/cv-mortgage-gpu.ipynb | 2 +- .../mortgage/notebooks/python/mortgage-gpu.ipynb | 2 +- .../mortgage/notebooks/scala/mortgage-ETL.ipynb | 4 ++-- .../taxi/notebooks/python/cv-taxi-gpu.ipynb | 2 +- .../XGBoost-Examples/taxi/notebooks/python/taxi-ETL.ipynb | 4 ++-- .../XGBoost-Examples/taxi/notebooks/python/taxi-gpu.ipynb | 2 +- .../XGBoost-Examples/taxi/notebooks/scala/taxi-ETL.ipynb | 4 ++-- tools/databricks/README.md | 2 +- ...for Apache Spark] Profiling Tool Notebook Template.ipynb | 2 +- ...Apache Spark] Qualification Tool Notebook Template.ipynb | 2 +- 28 files changed, 38 insertions(+), 38 deletions(-) diff --git a/docs/get-started/xgboost-examples/csp/databricks/databricks.md b/docs/get-started/xgboost-examples/csp/databricks/databricks.md index 400e0166..1334e2fd 100644 --- a/docs/get-started/xgboost-examples/csp/databricks/databricks.md +++ b/docs/get-started/xgboost-examples/csp/databricks/databricks.md @@ -21,7 +21,7 @@ Navigate to your home directory in the UI and select **Create** > **File** from create an `init.sh` scripts with contents: ```bash #!/bin/bash - sudo wget -O /databricks/jars/rapids-4-spark_2.12-24.12.0.jar https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.12.0/rapids-4-spark_2.12-24.12.0.jar + sudo wget -O /databricks/jars/rapids-4-spark_2.12-25.02.1.jar https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/25.02.1/rapids-4-spark_2.12-25.02.1.jar ``` 1. Select the Databricks Runtime Version from one of the supported runtimes specified in the Prerequisites section. @@ -68,7 +68,7 @@ create an `init.sh` scripts with contents: ```bash spark.rapids.sql.python.gpu.enabled true spark.python.daemon.module rapids.daemon_databricks - spark.executorEnv.PYTHONPATH /databricks/jars/rapids-4-spark_2.12-24.12.0.jar:/databricks/spark/python + spark.executorEnv.PYTHONPATH /databricks/jars/rapids-4-spark_2.12-25.02.1.jar:/databricks/spark/python ``` Note that since python memory pool require installing the cudf library, so you need to install cudf library in each worker nodes `pip install cudf-cu11 --extra-index-url=https://pypi.nvidia.com` or disable python memory pool diff --git a/docs/get-started/xgboost-examples/csp/databricks/init.sh b/docs/get-started/xgboost-examples/csp/databricks/init.sh index a4b475be..b3435512 100644 --- a/docs/get-started/xgboost-examples/csp/databricks/init.sh +++ b/docs/get-started/xgboost-examples/csp/databricks/init.sh @@ -15,7 +15,7 @@ sudo rm -f /databricks/jars/spark--maven-trees--ml--10.x--xgboost-gpu--ml.dmlc--xgboost4j-gpu_2.12--ml.dmlc__xgboost4j-gpu_2.12__1.5.2.jar sudo rm -f /databricks/jars/spark--maven-trees--ml--10.x--xgboost-gpu--ml.dmlc--xgboost4j-spark-gpu_2.12--ml.dmlc__xgboost4j-spark-gpu_2.12__1.5.2.jar -sudo wget -O /databricks/jars/rapids-4-spark_2.12-24.12.0.jar https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.12.0/rapids-4-spark_2.12-24.12.0.jar +sudo wget -O /databricks/jars/rapids-4-spark_2.12-25.02.1.jar https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/25.02.1/rapids-4-spark_2.12-25.02.1.jar sudo wget -O /databricks/jars/xgboost4j-gpu_2.12-1.7.1.jar https://repo1.maven.org/maven2/ml/dmlc/xgboost4j-gpu_2.12/1.7.1/xgboost4j-gpu_2.12-1.7.1.jar sudo wget -O /databricks/jars/xgboost4j-spark-gpu_2.12-1.7.1.jar https://repo1.maven.org/maven2/ml/dmlc/xgboost4j-spark-gpu_2.12/1.7.1/xgboost4j-spark-gpu_2.12-1.7.1.jar ls -ltr diff --git a/docs/get-started/xgboost-examples/on-prem-cluster/kubernetes-scala.md b/docs/get-started/xgboost-examples/on-prem-cluster/kubernetes-scala.md index 3c007a36..db945e9a 100644 --- a/docs/get-started/xgboost-examples/on-prem-cluster/kubernetes-scala.md +++ b/docs/get-started/xgboost-examples/on-prem-cluster/kubernetes-scala.md @@ -40,7 +40,7 @@ export SPARK_DOCKER_IMAGE= export SPARK_DOCKER_TAG= pushd ${SPARK_HOME} -wget https://github.com/NVIDIA/spark-rapids-examples/raw/branch-24.12/dockerfile/Dockerfile +wget https://github.com/NVIDIA/spark-rapids-examples/raw/branch-25.02/dockerfile/Dockerfile # Optionally install additional jars into ${SPARK_HOME}/jars/ diff --git a/docs/get-started/xgboost-examples/prepare-package-data/preparation-python.md b/docs/get-started/xgboost-examples/prepare-package-data/preparation-python.md index b43b3801..d4976234 100644 --- a/docs/get-started/xgboost-examples/prepare-package-data/preparation-python.md +++ b/docs/get-started/xgboost-examples/prepare-package-data/preparation-python.md @@ -5,7 +5,7 @@ For simplicity export the location to these jars. All examples assume the packag ### Download the jars Download the RAPIDS Accelerator for Apache Spark plugin jar - * [RAPIDS Spark Package](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.12.0/rapids-4-spark_2.12-24.12.0.jar) + * [RAPIDS Spark Package](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/25.02.1/rapids-4-spark_2.12-25.02.1.jar) ### Build XGBoost Python Examples diff --git a/docs/get-started/xgboost-examples/prepare-package-data/preparation-scala.md b/docs/get-started/xgboost-examples/prepare-package-data/preparation-scala.md index 582cc06e..7359630b 100644 --- a/docs/get-started/xgboost-examples/prepare-package-data/preparation-scala.md +++ b/docs/get-started/xgboost-examples/prepare-package-data/preparation-scala.md @@ -5,7 +5,7 @@ For simplicity export the location to these jars. All examples assume the packag ### Download the jars 1. Download the RAPIDS Accelerator for Apache Spark plugin jar - * [RAPIDS Spark Package](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.12.0/rapids-4-spark_2.12-24.12.0.jar) + * [RAPIDS Spark Package](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/25.02.1/rapids-4-spark_2.12-25.02.1.jar) ### Build XGBoost Scala Examples diff --git a/examples/ML+DL-Examples/Optuna-Spark/README.md b/examples/ML+DL-Examples/Optuna-Spark/README.md index 7681d375..6f3f605a 100644 --- a/examples/ML+DL-Examples/Optuna-Spark/README.md +++ b/examples/ML+DL-Examples/Optuna-Spark/README.md @@ -147,8 +147,8 @@ We use [RAPIDS](https://docs.rapids.ai/install/#get-rapids) for GPU-accelerated ``` shell sudo apt install libmysqlclient-dev -conda create -n rapids-24.12 -c rapidsai -c conda-forge -c nvidia \ - cudf=24.12 cuml=24.12 python=3.10 'cuda-version>=12.0,<=12.5' +conda create -n rapids-25.02 -c rapidsai -c conda-forge -c nvidia \ + cudf=25.02 cuml=25.02 python=3.10 'cuda-version>=12.0,<=12.5' conda activate optuna-spark pip install mysqlclient pip install optuna joblib joblibspark ipywidgets diff --git a/examples/ML+DL-Examples/Optuna-Spark/optuna-examples/databricks/init_optuna.sh b/examples/ML+DL-Examples/Optuna-Spark/optuna-examples/databricks/init_optuna.sh index 191e1248..e082025d 100644 --- a/examples/ML+DL-Examples/Optuna-Spark/optuna-examples/databricks/init_optuna.sh +++ b/examples/ML+DL-Examples/Optuna-Spark/optuna-examples/databricks/init_optuna.sh @@ -41,7 +41,7 @@ fi # rapids import -SPARK_RAPIDS_VERSION=24.12.0 +SPARK_RAPIDS_VERSION=25.02.1 curl -L https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/${SPARK_RAPIDS_VERSION}/rapids-4-spark_2.12-${SPARK_RAPIDS_VERSION}.jar -o \ /databricks/jars/rapids-4-spark_2.12-${SPARK_RAPIDS_VERSION}.jar @@ -54,7 +54,7 @@ ln -s /usr/local/cuda-11.8 /usr/local/cuda sudo /databricks/python3/bin/pip3 install \ --extra-index-url=https://pypi.nvidia.com \ - "cudf-cu11==24.12.*" "cuml-cu11==24.12.*" + "cudf-cu11==25.02.*" "cuml-cu11==25.02.*" # setup python environment sudo apt clean && sudo apt update --fix-missing -y diff --git a/examples/ML+DL-Examples/Optuna-Spark/optuna-examples/databricks/start_cluster.sh b/examples/ML+DL-Examples/Optuna-Spark/optuna-examples/databricks/start_cluster.sh index 1290ef32..a1a4c8c6 100755 --- a/examples/ML+DL-Examples/Optuna-Spark/optuna-examples/databricks/start_cluster.sh +++ b/examples/ML+DL-Examples/Optuna-Spark/optuna-examples/databricks/start_cluster.sh @@ -12,7 +12,7 @@ json_config=$(cat < Here is the bar chart from a recent execution on Google Colab's T4 High RAM instance using -RAPIDS Spark 24.12.0 with Apache Spark 3.5.0 +RAPIDS Spark 25.02.1 with Apache Spark 3.5.0 ![tpcds-speedup](/docs/img/guides/tpcds.png) diff --git a/examples/SQL+DF-Examples/tpcds/notebooks/TPCDS-SF10.ipynb b/examples/SQL+DF-Examples/tpcds/notebooks/TPCDS-SF10.ipynb index a8c19f3e..115828b9 100644 --- a/examples/SQL+DF-Examples/tpcds/notebooks/TPCDS-SF10.ipynb +++ b/examples/SQL+DF-Examples/tpcds/notebooks/TPCDS-SF10.ipynb @@ -30,7 +30,7 @@ "outputs": [], "source": [ "spark_version='3.5.0'\n", - "rapids_version='24.12.0'" + "rapids_version='25.02.1'" ] }, { diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md index fc75cc71..7b43f592 100644 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md @@ -186,7 +186,7 @@ then do the following inside the Docker container. ### Get jars from Maven Central -[rapids-4-spark_2.12-24.12.0.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.12.0/rapids-4-spark_2.12-24.12.0.jar) +[rapids-4-spark_2.12-25.02.1.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/25.02.1/rapids-4-spark_2.12-25.02.1.jar) ### Launch a local mode Spark diff --git a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml index f9305ff2..be07b07b 100644 --- a/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml +++ b/examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml @@ -37,7 +37,7 @@ cuda11 2.12 - 24.12.0 + 25.02.1 3.1.1 2.12.15 ${project.build.directory}/cpp-build diff --git a/examples/XGBoost-Examples/agaricus/notebooks/python/agaricus-gpu.ipynb b/examples/XGBoost-Examples/agaricus/notebooks/python/agaricus-gpu.ipynb index f92af8e5..d01b96ee 100644 --- a/examples/XGBoost-Examples/agaricus/notebooks/python/agaricus-gpu.ipynb +++ b/examples/XGBoost-Examples/agaricus/notebooks/python/agaricus-gpu.ipynb @@ -73,7 +73,7 @@ "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", "2022-11-30 06:57:40,550 WARN resource.ResourceUtils: The configuration of cores (exec = 2 task = 1, runnable tasks = 2) will result in wasted resources due to resource gpu limiting the number of runnable tasks per executor to: 1. Please adjust your configuration.\n", - "2022-11-30 06:57:54,195 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 24.12.0 using cudf 24.12.0.\n", + "2022-11-30 06:57:54,195 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 25.02.1 using cudf 25.02.1.\n", "2022-11-30 06:57:54,210 WARN rapids.RapidsPluginUtils: spark.rapids.sql.multiThreadedRead.numThreads is set to 20.\n", "2022-11-30 06:57:54,214 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator is enabled, to disable GPU support set `spark.rapids.sql.enabled` to false.\n", "2022-11-30 06:57:54,214 WARN rapids.RapidsPluginUtils: spark.rapids.sql.explain is set to `NOT_ON_GPU`. Set it to 'NONE' to suppress the diagnostics logging about the query placement on the GPU.\n", diff --git a/examples/XGBoost-Examples/mortgage/notebooks/python/MortgageETL.ipynb b/examples/XGBoost-Examples/mortgage/notebooks/python/MortgageETL.ipynb index 9d7767cd..51550e78 100644 --- a/examples/XGBoost-Examples/mortgage/notebooks/python/MortgageETL.ipynb +++ b/examples/XGBoost-Examples/mortgage/notebooks/python/MortgageETL.ipynb @@ -9,7 +9,7 @@ "Dataset is derived from Fannie Mae’s [Single-Family Loan Performance Data](http://www.fanniemae.com/portal/funding-the-market/data/loan-performance-data.html) with all rights reserved by Fannie Mae. Refer to these [instructions](https://github.com/NVIDIA/spark-rapids-examples/blob/branch-24.12/docs/get-started/xgboost-examples/dataset/mortgage.md) to download the dataset.\n", "\n", "### 2. Download needed jars\n", - "* [rapids-4-spark_2.12-24.12.0.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.12.0/rapids-4-spark_2.12-24.12.0.jar)\n", + "* [rapids-4-spark_2.12-25.02.1.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/25.02.1/rapids-4-spark_2.12-25.02.1.jar)\n", "\n", "\n", "### 3. Start Spark Standalone\n", @@ -17,7 +17,7 @@ "\n", "### 4. Add ENV\n", "```\n", - "$ export SPARK_JARS=rapids-4-spark_2.12-24.12.0.jar\n", + "$ export SPARK_JARS=rapids-4-spark_2.12-25.02.1.jar\n", "$ export PYSPARK_DRIVER_PYTHON=jupyter \n", "$ export PYSPARK_DRIVER_PYTHON_OPTS=notebook\n", "```\n", diff --git a/examples/XGBoost-Examples/mortgage/notebooks/python/cv-mortgage-gpu.ipynb b/examples/XGBoost-Examples/mortgage/notebooks/python/cv-mortgage-gpu.ipynb index 3e441712..66d53270 100644 --- a/examples/XGBoost-Examples/mortgage/notebooks/python/cv-mortgage-gpu.ipynb +++ b/examples/XGBoost-Examples/mortgage/notebooks/python/cv-mortgage-gpu.ipynb @@ -63,7 +63,7 @@ "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", "2022-11-25 09:34:43,952 WARN resource.ResourceUtils: The configuration of cores (exec = 4 task = 1, runnable tasks = 4) will result in wasted resources due to resource gpu limiting the number of runnable tasks per executor to: 1. Please adjust your configuration.\n", - "2022-11-25 09:34:58,155 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 24.12.0 using cudf 24.12.0.\n", + "2022-11-25 09:34:58,155 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 25.02.1 using cudf 25.02.1.\n", "2022-11-25 09:34:58,171 WARN rapids.RapidsPluginUtils: spark.rapids.sql.multiThreadedRead.numThreads is set to 20.\n", "2022-11-25 09:34:58,175 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator is enabled, to disable GPU support set `spark.rapids.sql.enabled` to false.\n", "2022-11-25 09:34:58,175 WARN rapids.RapidsPluginUtils: spark.rapids.sql.explain is set to `NOT_ON_GPU`. Set it to 'NONE' to suppress the diagnostics logging about the query placement on the GPU.\n" diff --git a/examples/XGBoost-Examples/mortgage/notebooks/python/mortgage-gpu.ipynb b/examples/XGBoost-Examples/mortgage/notebooks/python/mortgage-gpu.ipynb index e64ba9e0..53d03ee0 100644 --- a/examples/XGBoost-Examples/mortgage/notebooks/python/mortgage-gpu.ipynb +++ b/examples/XGBoost-Examples/mortgage/notebooks/python/mortgage-gpu.ipynb @@ -84,7 +84,7 @@ "22/11/24 06:14:06 INFO org.apache.spark.SparkEnv: Registering BlockManagerMaster\n", "22/11/24 06:14:06 INFO org.apache.spark.SparkEnv: Registering BlockManagerMasterHeartbeat\n", "22/11/24 06:14:06 INFO org.apache.spark.SparkEnv: Registering OutputCommitCoordinator\n", - "22/11/24 06:14:07 WARN com.nvidia.spark.rapids.RapidsPluginUtils: RAPIDS Accelerator 24.12.0 using cudf 24.12.0.\n", + "22/11/24 06:14:07 WARN com.nvidia.spark.rapids.RapidsPluginUtils: RAPIDS Accelerator 25.02.1 using cudf 25.02.1.\n", "22/11/24 06:14:07 WARN com.nvidia.spark.rapids.RapidsPluginUtils: spark.rapids.sql.multiThreadedRead.numThreads is set to 20.\n", "22/11/24 06:14:07 WARN com.nvidia.spark.rapids.RapidsPluginUtils: RAPIDS Accelerator is enabled, to disable GPU support set `spark.rapids.sql.enabled` to false.\n", "22/11/24 06:14:07 WARN com.nvidia.spark.rapids.RapidsPluginUtils: spark.rapids.sql.explain is set to `NOT_ON_GPU`. Set it to 'NONE' to suppress the diagnostics logging about the query placement on the GPU.\n" diff --git a/examples/XGBoost-Examples/mortgage/notebooks/scala/mortgage-ETL.ipynb b/examples/XGBoost-Examples/mortgage/notebooks/scala/mortgage-ETL.ipynb index b551df7a..ae76ec92 100644 --- a/examples/XGBoost-Examples/mortgage/notebooks/scala/mortgage-ETL.ipynb +++ b/examples/XGBoost-Examples/mortgage/notebooks/scala/mortgage-ETL.ipynb @@ -20,14 +20,14 @@ "Refer to these [instructions](https://github.com/NVIDIA/spark-rapids-examples/blob/branch-23.12/docs/get-started/xgboost-examples/dataset/mortgage.md) to download the dataset.\n", "\n", "### 2. Download needed jars\n", - "* [rapids-4-spark_2.12-24.12.0.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.12.0/rapids-4-spark_2.12-24.12.0.jar)\n", + "* [rapids-4-spark_2.12-25.02.1.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/25.02.1/rapids-4-spark_2.12-25.02.1.jar)\n", "\n", "### 3. Start Spark Standalone\n", "Before Running the script, please setup Spark standalone mode\n", "\n", "### 4. Add ENV\n", "```\n", - "$ export SPARK_JARS=rapids-4-spark_2.12-24.12.0.jar\n", + "$ export SPARK_JARS=rapids-4-spark_2.12-25.02.1.jar\n", "\n", "```\n", "\n", diff --git a/examples/XGBoost-Examples/taxi/notebooks/python/cv-taxi-gpu.ipynb b/examples/XGBoost-Examples/taxi/notebooks/python/cv-taxi-gpu.ipynb index e0e1372e..6554ce3a 100644 --- a/examples/XGBoost-Examples/taxi/notebooks/python/cv-taxi-gpu.ipynb +++ b/examples/XGBoost-Examples/taxi/notebooks/python/cv-taxi-gpu.ipynb @@ -62,7 +62,7 @@ "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", "2022-11-30 08:02:10,103 WARN resource.ResourceUtils: The configuration of cores (exec = 2 task = 1, runnable tasks = 2) will result in wasted resources due to resource gpu limiting the number of runnable tasks per executor to: 1. Please adjust your configuration.\n", - "2022-11-30 08:02:23,737 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 24.12.0 using cudf 24.12.0.\n", + "2022-11-30 08:02:23,737 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 25.02.1 using cudf 25.02.1.\n", "2022-11-30 08:02:23,752 WARN rapids.RapidsPluginUtils: spark.rapids.sql.multiThreadedRead.numThreads is set to 20.\n", "2022-11-30 08:02:23,756 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator is enabled, to disable GPU support set `spark.rapids.sql.enabled` to false.\n", "2022-11-30 08:02:23,757 WARN rapids.RapidsPluginUtils: spark.rapids.sql.explain is set to `NOT_ON_GPU`. Set it to 'NONE' to suppress the diagnostics logging about the query placement on the GPU.\n", diff --git a/examples/XGBoost-Examples/taxi/notebooks/python/taxi-ETL.ipynb b/examples/XGBoost-Examples/taxi/notebooks/python/taxi-ETL.ipynb index fda40075..5c9339c4 100644 --- a/examples/XGBoost-Examples/taxi/notebooks/python/taxi-ETL.ipynb +++ b/examples/XGBoost-Examples/taxi/notebooks/python/taxi-ETL.ipynb @@ -19,14 +19,14 @@ "All data could be found at https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page\n", "\n", "### 2. Download needed jars\n", - "* [rapids-4-spark_2.12-24.12.0.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.12.0/rapids-4-spark_2.12-24.12.0.jar)\n", + "* [rapids-4-spark_2.12-25.02.1.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/25.02.1/rapids-4-spark_2.12-25.02.1.jar)\n", "\n", "### 3. Start Spark Standalone\n", "Before running the script, please setup Spark standalone mode\n", "\n", "### 4. Add ENV\n", "```\n", - "$ export SPARK_JARS=rapids-4-spark_2.12-24.12.0.jar\n", + "$ export SPARK_JARS=rapids-4-spark_2.12-25.02.1.jar\n", "$ export PYSPARK_DRIVER_PYTHON=jupyter \n", "$ export PYSPARK_DRIVER_PYTHON_OPTS=notebook\n", "```\n", diff --git a/examples/XGBoost-Examples/taxi/notebooks/python/taxi-gpu.ipynb b/examples/XGBoost-Examples/taxi/notebooks/python/taxi-gpu.ipynb index 6903547c..05bbd6a9 100644 --- a/examples/XGBoost-Examples/taxi/notebooks/python/taxi-gpu.ipynb +++ b/examples/XGBoost-Examples/taxi/notebooks/python/taxi-gpu.ipynb @@ -73,7 +73,7 @@ "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", "2022-11-30 07:51:19,480 WARN resource.ResourceUtils: The configuration of cores (exec = 2 task = 1, runnable tasks = 2) will result in wasted resources due to resource gpu limiting the number of runnable tasks per executor to: 1. Please adjust your configuration.\n", - "2022-11-30 07:51:33,277 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 24.12.0 using cudf 24.12.0.\n", + "2022-11-30 07:51:33,277 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator 25.02.1 using cudf 25.02.1.\n", "2022-11-30 07:51:33,292 WARN rapids.RapidsPluginUtils: spark.rapids.sql.multiThreadedRead.numThreads is set to 20.\n", "2022-11-30 07:51:33,295 WARN rapids.RapidsPluginUtils: RAPIDS Accelerator is enabled, to disable GPU support set `spark.rapids.sql.enabled` to false.\n", "2022-11-30 07:51:33,295 WARN rapids.RapidsPluginUtils: spark.rapids.sql.explain is set to `NOT_ON_GPU`. Set it to 'NONE' to suppress the diagnostics logging about the query placement on the GPU.\n", diff --git a/examples/XGBoost-Examples/taxi/notebooks/scala/taxi-ETL.ipynb b/examples/XGBoost-Examples/taxi/notebooks/scala/taxi-ETL.ipynb index 20d6c32d..4a7a2c2e 100644 --- a/examples/XGBoost-Examples/taxi/notebooks/scala/taxi-ETL.ipynb +++ b/examples/XGBoost-Examples/taxi/notebooks/scala/taxi-ETL.ipynb @@ -19,14 +19,14 @@ "All data could be found at https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page\n", "\n", "### 2. Download needed jar\n", - "* [rapids-4-spark_2.12-24.12.0.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/24.12.0/rapids-4-spark_2.12-24.12.0.jar)\n", + "* [rapids-4-spark_2.12-25.02.1.jar](https://repo1.maven.org/maven2/com/nvidia/rapids-4-spark_2.12/25.02.1/rapids-4-spark_2.12-25.02.1.jar)\n", "\n", "### 3. Start Spark Standalone\n", "Before running the script, please setup Spark standalone mode\n", "\n", "### 4. Add ENV\n", "```\n", - "$ export SPARK_JARS=rapids-4-spark_2.12-24.12.0.jar\n", + "$ export SPARK_JARS=rapids-4-spark_2.12-25.02.1.jar\n", "\n", "```\n", "\n", diff --git a/tools/databricks/README.md b/tools/databricks/README.md index 95b222e9..e5638850 100644 --- a/tools/databricks/README.md +++ b/tools/databricks/README.md @@ -19,4 +19,4 @@ top of the notebook. After that, select *Run all* to execute the tools for the 1. Multiple event logs must be comma-separated. - For example: `/dbfs/path/to/eventlog1,/dbfs/path/to/eventlog2` -**Latest Tools Version Supported** 24.12.0 \ No newline at end of file +**Latest Tools Version Supported** 25.02.1 \ No newline at end of file diff --git a/tools/databricks/[RAPIDS Accelerator for Apache Spark] Profiling Tool Notebook Template.ipynb b/tools/databricks/[RAPIDS Accelerator for Apache Spark] Profiling Tool Notebook Template.ipynb index 0c058638..f260b8bf 100644 --- a/tools/databricks/[RAPIDS Accelerator for Apache Spark] Profiling Tool Notebook Template.ipynb +++ b/tools/databricks/[RAPIDS Accelerator for Apache Spark] Profiling Tool Notebook Template.ipynb @@ -53,7 +53,7 @@ }, "outputs": [], "source": [ - "TOOLS_VER = \"24.12.0\"\n", + "TOOLS_VER = \"25.02.1\"\n", "print(f\"Using Tools Version: {TOOLS_VER}\")" ] }, diff --git a/tools/databricks/[RAPIDS Accelerator for Apache Spark] Qualification Tool Notebook Template.ipynb b/tools/databricks/[RAPIDS Accelerator for Apache Spark] Qualification Tool Notebook Template.ipynb index 3d1894ca..1d4a594d 100644 --- a/tools/databricks/[RAPIDS Accelerator for Apache Spark] Qualification Tool Notebook Template.ipynb +++ b/tools/databricks/[RAPIDS Accelerator for Apache Spark] Qualification Tool Notebook Template.ipynb @@ -49,7 +49,7 @@ }, "outputs": [], "source": [ - "TOOLS_VER = \"24.12.0\"\n", + "TOOLS_VER = \"25.02.1\"\n", "print(f\"Using Tools Version: {TOOLS_VER}\")" ] }, From efd0ed65c5dd537ef289d5abf28c1f6823523d66 Mon Sep 17 00:00:00 2001 From: "Rishi C." <77904151+rishic3@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:19:54 -0800 Subject: [PATCH 12/12] Use dataset streaming, cleanup diagram (#497) Use streaming to avoid saving entire Huggingface dataset to disk for large datasets. Updated diagram for clarity regarding client/server interaction. --------- Signed-off-by: Rishi Chandra --- .../Spark-DL/dl_inference/README.md | 26 +++++++++--------- .../huggingface/deepseek-r1_torch.ipynb | 11 ++++---- .../huggingface/gemma-7b_torch.ipynb | 11 ++++---- .../dl_inference/images/spark-pytriton.png | Bin 42644 -> 41091 bytes 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md b/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md index ea696c48..0286a97d 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/README.md @@ -39,22 +39,22 @@ In this simple case, the `predict_batch_fn` will use TensorFlow APIs to load the #### Notebook List -Below is a full list of the notebooks with links to the examples they are based on. All notebooks have been saved with sample outputs for quick browsing. +Below is a full list of the notebooks and their links. All notebooks have been saved with sample outputs for quick browsing. | | Framework | Notebook Name | Description | Link | ------------- | ------------- | ------------- | ------------- | ------------- -| 1 | HuggingFace | DeepSeek-R1 | LLM batch inference using the DeepSeek-R1-Distill-Llama reasoning model to solve word problems. | [Link](https://huggingface.co/deepseek-ai/DeepSeek-R1) -| 2 | HuggingFace | Qwen-2.5-7b | LLM batch inference using the Qwen-2.5-7b model for text summarization. | [Link](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) -| 3 | HuggingFace | Gemma-7b | LLM batch inference using the Google Gemma-7b model for code comprehension tasks. | [Link](https://huggingface.co/google/gemma-7b-it) -| 4 | HuggingFace | Sentence Transformers | Sentence embeddings using SentenceTransformers in Torch. | [Link](https://huggingface.co/sentence-transformers) -| 5+6 | HuggingFace | Conditional Generation | Sentence translation using the T5 text-to-text transformer (Torch and Tensorflow). | [Link](https://huggingface.co/docs/transformers/model_doc/t5#t5) -| 7+8 | HuggingFace | Pipelines | Sentiment analysis using Huggingface pipelines (Torch and Tensorflow). | [Link](https://huggingface.co/docs/transformers/quicktour#pipeline-usage) -| 9 | PyTorch | Image Classification | Training a model to predict clothing categories in FashionMNIST, and deploying with Torch-TensorRT accelerated inference. | [Link](https://pytorch.org/tutorials/beginner/basics/quickstart_tutorial.html) -| 10 | PyTorch | Housing Regression | Training and deploying a model to predict housing prices in the California Housing Dataset, and deploying with Torch-TensorRT accelerated inference. | [Link](https://github.com/christianversloot/machine-learning-articles/blob/main/how-to-create-a-neural-network-for-regression-with-pytorch.md) -| 11 | Tensorflow | Image Classification | Training and deploying a model to predict hand-written digits in MNIST. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/save_and_load.ipynb) -| 12 | Tensorflow | Keras Preprocessing | Training and deploying a model with preprocessing layers to predict likelihood of pet adoption in the PetFinder mini dataset. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/preprocessing_layers.ipynb) -| 13 | Tensorflow | Keras Resnet50 | Deploying ResNet-50 to perform flower recognition from flower images. | [Link](https://docs.databricks.com/en/_extras/notebooks/source/deep-learning/keras-metadata.html) -| 14 | Tensorflow | Text Classification | Training and deploying a model to perform sentiment analysis on the IMDB dataset. | [Link](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/text_classification.ipynb) +| 1 | HuggingFace | DeepSeek-R1 | LLM batch inference using the DeepSeek-R1-Distill-Llama reasoning model to solve word problems. | [Link](huggingface/deepseek-r1_torch.ipynb) +| 2 | HuggingFace | Qwen-2.5-7b | LLM batch inference using the Qwen-2.5-7b model for text summarization. | [Link](huggingface/qwen-2.5-7b_torch.ipynb) +| 3 | HuggingFace | Gemma-7b | LLM batch inference using the Google Gemma-7b model for code comprehension tasks. | [Link](huggingface/gemma-7b_torch.ipynb) +| 4 | HuggingFace | Sentence Transformers | Sentence embeddings using SentenceTransformers in Torch. | [Link](huggingface/sentence_transformers_torch.ipynb) +| 5+6 | HuggingFace | Conditional Generation | Sentence translation using the T5 text-to-text transformer (Torch and Tensorflow). | [Torch Link](huggingface/conditional_generation_torch.ipynb), [TF Link](huggingface/conditional_generation_tf.ipynb) +| 7+8 | HuggingFace | Pipelines | Sentiment analysis using Huggingface pipelines (Torch and Tensorflow). | [Torch Link](huggingface/pipelines_torch.ipynb), [TF Link](huggingface/pipelines_tf.ipynb) +| 9 | PyTorch | Image Classification | Training a model to predict clothing categories in FashionMNIST, and deploying with Torch-TensorRT accelerated inference. | [Link](pytorch/image_classification_torch.ipynb) +| 10 | PyTorch | Housing Regression | Training and deploying a model to predict housing prices in the California Housing Dataset, and deploying with Torch-TensorRT accelerated inference. | [Link](pytorch/housing_regression_torch.ipynb) +| 11 | Tensorflow | Image Classification | Training and deploying a model to predict hand-written digits in MNIST. | [Link](tensorflow/image_classification_tf.ipynb) +| 12 | Tensorflow | Keras Preprocessing | Training and deploying a model with preprocessing layers to predict likelihood of pet adoption in the PetFinder mini dataset. | [Link](tensorflow/keras_preprocessing_tf.ipynb) +| 13 | Tensorflow | Keras Resnet50 | Deploying ResNet-50 to perform flower recognition from flower images. | [Link](tensorflow/keras_resnet50_tf.ipynb) +| 14 | Tensorflow | Text Classification | Training and deploying a model to perform sentiment analysis on the IMDB dataset. | [Link](tensorflow/text_classification_tf.ipynb) ## Running Locally diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/deepseek-r1_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/deepseek-r1_torch.ipynb index 55f0d879..d59a6b32 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/deepseek-r1_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/deepseek-r1_torch.ipynb @@ -6,11 +6,11 @@ "source": [ "\n", "\n", - "# PySpark LLM Inference: DeepSeek-R1\n", + "# PySpark LLM Inference: DeepSeek-R1 Reasoning Q/A\n", "\n", "In this notebook, we demonstrate distributed batch inference with [DeepSeek-R1](https://github.com/deepseek-ai/DeepSeek-R1), using open weights on Huggingface.\n", "\n", - "We use [DeepSeek-R1-Distill-Llama-8B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B) as demonstration. DeepSeek's distilled models are based on open-source LLMs (such as Llama/Qwen), and are fine-tuned using samples generated by DeepSeek-R1 to perform multi-step reasoning tasks.\n", + "We use [DeepSeek-R1-Distill-Llama-8B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-8B) as demonstration. DeepSeek's distilled models are based on open-source LLMs (such as Llama/Qwen), and are fine-tuned using samples generated by DeepSeek-R1. We'll show how to use the model to reason through word problems.\n", "\n", "**Note:** Running this model on GPU with 16-bit precision requires **~18GB** of GPU RAM. Make sure your instances have sufficient GPU capacity." ] @@ -261,6 +261,7 @@ "outputs": [], "source": [ "import os\n", + "import pandas as pd\n", "import datasets\n", "from datasets import load_dataset\n", "datasets.disable_progress_bars()" @@ -330,7 +331,7 @@ "source": [ "#### Load DataFrame\n", "\n", - "Load the Orca Math Word Problems dataset from Huggingface and store in a Spark Dataframe." + "Load the first 500 samples of the [Orca Math Word Problems dataset](https://huggingface.co/datasets/microsoft/orca-math-word-problems-200k) from Huggingface and store in a Spark Dataframe." ] }, { @@ -339,8 +340,8 @@ "metadata": {}, "outputs": [], "source": [ - "dataset = load_dataset(\"microsoft/orca-math-word-problems-200k\", split=\"train[:1%]\")\n", - "dataset = dataset.to_pandas()[\"question\"]" + "dataset = load_dataset(\"microsoft/orca-math-word-problems-200k\", split=\"train\", streaming=True)\n", + "dataset = pd.Series([sample[\"question\"] for sample in dataset.take(500)])" ] }, { diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/gemma-7b_torch.ipynb b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/gemma-7b_torch.ipynb index 9d6d43a2..63cbd1a1 100644 --- a/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/gemma-7b_torch.ipynb +++ b/examples/ML+DL-Examples/Spark-DL/dl_inference/huggingface/gemma-7b_torch.ipynb @@ -6,11 +6,11 @@ "source": [ "\n", "\n", - "# PySpark LLM Inference: Gemma-7b\n", + "# PySpark LLM Inference: Gemma-7b Code Comprehension\n", "\n", "In this notebook, we demonstrate distributed inference with the Google [Gemma-7b-instruct](https://huggingface.co/google/gemma-7b-it) LLM, using open-weights on Huggingface.\n", "\n", - "The Gemma-7b-instruct is an instruction-fine-tuned version of the Gemma-7b base model.\n", + "The Gemma-7b-instruct is an instruction-fine-tuned version of the Gemma-7b base model. We'll show how to use the model to perform code comprehension tasks.\n", "\n", "**Note:** Running this model on GPU with 16-bit precision requires **~18 GB** of GPU RAM. Make sure your instances have sufficient GPU capacity." ] @@ -200,6 +200,7 @@ "outputs": [], "source": [ "import os\n", + "import pandas as pd\n", "import datasets\n", "from datasets import load_dataset\n", "datasets.disable_progress_bars()" @@ -269,7 +270,7 @@ "source": [ "#### Load DataFrame\n", "\n", - "Load the code comprehension dataset from Huggingface and store in a Spark Dataframe." + "Load the first 500 samples of the [Code Comprehension dataset](https://huggingface.co/datasets/imbue/code-comprehension) from Huggingface and store in a Spark Dataframe." ] }, { @@ -278,8 +279,8 @@ "metadata": {}, "outputs": [], "source": [ - "dataset = load_dataset(\"imbue/code-comprehension\", split=\"train[:1%]\")\n", - "dataset = dataset.to_pandas()[\"question\"]" + "dataset = load_dataset(\"imbue/code-comprehension\", split=\"train\", streaming=True)\n", + "dataset = pd.Series([sample[\"question\"] for sample in dataset.take(500)])" ] }, { diff --git a/examples/ML+DL-Examples/Spark-DL/dl_inference/images/spark-pytriton.png b/examples/ML+DL-Examples/Spark-DL/dl_inference/images/spark-pytriton.png index 841862a4dcb1a80fb2ece9d59f383ad406fadb4c..f5b4e21a45d8700ae4ed7cf740c3cbcb21dd110e 100644 GIT binary patch literal 41091 zcmeFZ^;?ut*ES3Y3Zep{AdR%tNOy?R0@4k_&^1bT*tCFjgMdhP4WZI0-3-#*UGwcR z@ArA`Kj8b};W*~vnk#p%b*^)rd;Fg$N?*T9dKC=~?YgYYBNa3>%oa2>jBD7J!I5p7 z4pKBUbY07b51+_Bd`SJo9%g1~V~U2x6z254R{qsu``SWm(JYLRKt8d~xgtj|eo?;!RE5LTHYLx)-M&OuGv;dki{X zURA2;f$_DR%qyqX9r7Wz-MJ?&v2E$p+f35eKT0@1<|v>!eXgW;kMuw=yfy zy7Y~RHTSDH&J8+~R-^l=ggq%3w`boGFyd#)?UEtu^gCFRF$l-Ssrz=g$7*>WSsPTq zm(cG&MO##TT=XG1@M^wA0Ue>}p@!dvSC5B2<_lW!*{R*BoTOoX^lC~!+(W`G9TRh> zUGYHB+8U->vStbjXv|M(?c`sXE%)*>`o3QwpX!t70{dD-u?-=`71 zN=;2IY;R&FsPaha_wC@D2#tlK<1;}B#Kpyh-Gz%CW^WGR5D*Z6+~>Ox+J><`G9+^5o?Jb`^!M*qIZa(H|Gml1;diw_ z1tF+25DxbHkiX9cw+f?<3O=!PHMP-xWN8bU2fRa+hm%|Q=k@>V%)dAO$DLaL?&RR) zW${+E03w{p?iIC>1+Nkjg3IjF_$gFOECd+@~@NT z8QVBE{#V&hMeVuzo_u#LBkRY1s~4!$mH$znpI`s4Qh(a=|9a}rfd9YY_jjiJ-<Eh}&{P^~kctr6~K}LqDaGeYWC$QI@6s`Q2l|@{= zUV-o?gS(}UuL!&h911>|&wh5kvw35q#}oJBY+iRs?NdF(k;JEd4(aj&DLSY7;h^59 zV!~d2`g#jF#ti})Sy`Op!}MmNFZ3$gZ<=4dKBu6_b`3j-Jm0LDK7FxD%%n0>s~6|` z@-*VKr0|q!%Z~6zk70_8k)8AL?%|>sQlRkbjij8lkJTVRVE}8Daoz&wkjT1vTGF0e@5y*^(OEHFk9SN2Z3*Fd*?&g=`0g~h-`AO@qoX5?U~Fk& z_34ymj9?txQ`nP_Gm>M`KJ;?Chi(@J$Eteup16t3YX_h4gcG=4`%uCg*R&y)i#{xzF8KNh#aK&bB3HjWBC z*)hNWf*zh&`L$!FTB#uHajbdqKMjY5u|XgsFCP(1#&;7^y%^qi^VhvJ?9-p- zMy9{KG#$#-4v&ny{rryNzY#Po?AO@%C|1BB)5P3ds99!QXPCW2VteSI%iYe*9Q1h5z2KrZ6D*j;=Au+xl5EpJM%%e{e95 znAUITDL(hJxi@ShtE#HPYO2XeeiwjFL`46HBY^BnHU4SxLUzE&JpS zgrc_pF~t84_ILTd>Fyy4aoqQ+brSx}J^&ksFf1)C%k!xsAO61d6bBf^pUv(of~Aqmeoe zSKB>F5SNg6p@pvWe7u+v)y54X7>$7&RZ*pweloZg@K00~x1>chV#C7j*0^j&8PvL0 zH6=S!KS{-KL3mXx%|`NA59NN}wdVxyjcSPDfF`ILSq1tg`ySly?(SwV*yA#~>DPyi z4~dcrv?bWe_Z^>+l-?(oS5%DZfYeXaxKOc4rzBvAo$o~mUmVTO3tz4IWn!Khq6ZUS zxj}A$xT?a1X~QlKx7fvfv=6FGgApPZBKgSoCCxO4-(s>)((@W&=Ppb3Kdt?Q6n&`hiX(G zQReJDmH5Emp_F1Z__?nFhVXrZT!;B^d3a#4vVQ3l+Jw_yx_k|PolQec7MPdHN`6@- zCA_N9^Fss#Pk76B2@;Xo?y*_R$T#uq;U{(U4hF>prt_11mSR!)DAuri{hIa{ShyGm zrkBw%C2*I}azN)?uMN+@9?@$!O1|EUhlP$Q$0!b$fooL@y~(R(PuWiMo=}kqshXGT zyMY-%;CjA4+=T3v@OriVQxJuv+yHDzOb+=bQufaJ8})h5lW)kNv<S2f=;M30OW z^>VkHtaiFo?y=V^eW|=x+90Ujz$>}jc6?#w_`G8vTh(hd&-Nu%&e5`EnBVAQ;B|1q_%xCQW$N5A@7TuSQlb{Zjs+$sMVm>k0PGfh|P z_xV%hQC8Nf!U0&(%Eu{%PZxnhsLIo=xFJJC5=yo}D)Oas?nC6OzRhWl7GBsCc-O+! zk;BuUwzkKL_o6=opl=AXSZdoH4zmI zgXcX?ZS>JcLma>-r(#t*txDOma$lE6L55*q92N}aW|};4Gx3?SeVYSMp3ArST(lk4 zOQXuAn<5oVy0el~aG6^Lv2o|@9kk(ceOG6ah|u(Qld7VCWRQV&DlZRJAh&zQ@zDBI zt*-aQvAKgc^5Q%#RQH6K)@WXAHJ+0HSqeqfK>R#_-k!WNVu(D}T3oUvs zuKu_=vOkkmoLa`$Rr!@G$5Q4~B6K`#zz{S@X8L0v;htR_))UVbNg*Y7Mxdc>U<_S~ z!s@}qYw&3%F~@|>#22XNM3q|fe&s75WYNs$UgWczxKju>>x%2Ed_L=II`nvpBX|D9 z7VcHt;M-AMyZxrvHKVefWWt2fML;6(K1!*A&H>WR>SemF{yRm4;a&XF>G#X z8ar;?!-1SZeCzb7QY3>$V?lPh$e&Epc+7t}CKw=i5<>DKZ!djpf^g~D!q3n%;+vq|#7I-;~$(|}3Rg08{Wec6Lv+*~0 zdf{%nzD-|5+AuUoxxTE<&dhWwjFnmv7B2f;y7GuBEEo8X2K&T&pQEEihn5W;ky**! zV?G4^KPMCAt?yI405q~51h-IrOl`ydAm{5^q^4&Rgobg0_?eo0=Z_&ZDs)|(iRUd8 zI|t*ij$+deIW@>^a5(ySo_@^`xlrdFJ+Y@%-Af$7UyxoM!G32Ms#%H)@DTLOP<<9yC3al5UpEsTIx z3X@fA6TbTqyra%k1Hx(jMIKUXqHt2@vOY!+t2@qwiMmR71SC2QX`=@se30k9Zi#NI zc@-Z)F*zNVp17@KXPJQJ!zwpI|Kr?z#UFqP{DOfMKpdB=AJ&iowgU%g&5Xs$JlR(< zLh!$%--6YjAKV-)lZ#}w?^oc%vdL*HTP>(Js5{*dD#;q*LwM2>8+sNU^9PotZXAwT z7igC(9KOU8y>mY~E{^%6t6*&AEtog7#6=ssKEqI)$yG5ls5`B^=u9boRkx%LwA(L_GW_1!mfs5nbmsZ4@~)H;0P z$KbvFolgQdmFh=j{U(ERZ4ntt$)Zhm1vml}dxjCv)OH`_sUeOxT&1GqCRETl=EkSo zmV3Dtnm!7J`i{Ml=<#P;-|(<*La&o@zbD}`dw&P_z46b9h#G)Q24D)f(v|}|X^xAE z)lFL8V?ap+;*-Z{pW^vh=T|j*k{8pJ^;SIUj^&az@Hux{28O^-zqBPTM5)}Q?WWt& zkGGP9TB!jyw(4ECjjOhs$u7B9Ekk2!Jr1K@;}X9i|0s5(6}NYCw%gN1mvG>V>-|A0 z!uo{&ljsh~=ejQ~Pyh=X*LdCl*LM56t1?w~7Vs^uz~_BiZR^E5iHl#(1XKtTm}0sO!~)Qi<7x;6^1TvWd9w-A9;dPl&$G z8?G?P*Q-)-UK_a!Y~Tl8+p%||pJgAQ`Gwa=jokcclw83mnh74kK+nRdTrZLf zhbq=w%7*-!6$C`o0z^hnZ-o8{qWJY1Y}+^hY+c^+c)RioPyP#TPCwdaBfIDao{P{z1-?_OhINVTyV(a)oPOQ?Q{0{D1$ zu_v^Sl;>YaTbR)R#$Dnr3w;Z~cnMYt zwZ8)DXRjLiGUO%2Zeb=c;ggZEoTUCgF_K0Ixw=V6LVP?~>?H@a-vQTsEWmU;@(Get zQj*3*Z^4f&0I}%?8AWNzjLHu7zz(){G~7 zSoy^-OxvKpx&Q27f7gAT$KbJy%$DG#^sjf1@L6!B>9c7?kWe zE|?%rs_oa(3HVuQ@ei02ymPlh)uooDW=j7-segOD#Gp^lt;MhZE7HHcG`eX(iKLq} zTK+2f)`y0XBB;oq0cqghHhm(9qBc019nz{YXx~i@{fm#^8PM}{{k5mQwpe|AeI~Qb z!Ht3CwGwLAe)aXwAduH}>uD^Hb(D#z>8MgT=|76)k-qqqx#VBrv2-oF+MltLa2JHf z-Ia<1KL_ofqrVGIjR6P_8sGN2|E;|f?gAc%WY}Nl7gh4a2o=|bzkmP4XmR$@zh>#a z38tW!-95UN3440WD}QV8x7xnq*;vfc?x9Q@_{7AF(xH^W#@G3IeqFag{LbJ1xeu*n zsx_9N%Lc1-2>El{Q;R#l&N|~i(}gd6s!$*MaXID2uUA&;i`TSTH$)^QZ5(9wiT(}e z{W!6;Le<#yt4ptWjKhd2b4n3BsZ#%n`ETzwow$AOixh$D|D~wD=a}R?3o++Mf4Gqi z!aJg#m8YbVf&cpYE5mfTYcHU&GBPve%*--_?r_l95P!JzuQMv8|4~#)@^bRO?tNG3 zn;s5*c?B;DfIL|Z4UI-4{(oN7cNaAvO#y5PPfT2gx2r;b*X>;Z8W@$Y6C-GR#p$rX zdBV1D4mOxH4BzX0knbqsFa5SA;&lKN)M2T1-=sDO>fEOhAR^HW0fSrl)}8xL#c%a= zXA>~hh}+v22ayXDuU&bn^!xctVc)QfZ)Utbo?Ttjp! z(g}qVXRD0>-d=1Nw%*Q>dAa2$o`w|=Tid8IGQSX)#WIFMQTE^$R%`ldqGjg-%wwzm zU0^%x9UN%0zJNL1ppM}M!lf1WzL)?)86_oR@1W28u&-f+mcoBtnXMd2rzDHmdDV$uq%R)%JQfpYyRH%BJAl4t0Y zaslpbCY`6tX~zp?D~NiZUb5n!Wp222hs*5RO>BORr+`%f!$ASQG{SNxF@O*v)sk2O zy2SwtMg0LIpOEmI?_?dHcu-I$*2L?SN`r;fT+1)VJ@Usa`|@FO8zcbR5KCPmALviK zZci^U{L+=96!&Mi1Hrk!5EGrxYb*Ib%meBxHek?*8opfnZ<8e&FraSBR{a{k8di?_ zzVn(EsvjAzHa0f?Kjs}6K458F3Z9w#x|aN2%B9?Ja)-H$^u9OZc6Ry4=zkbk8b%OU z$6uaj6dv*52PUI>pylG)n44E*dT_InDQMRmQgiAFO`a zylFk)mnPMK>+Mw7Dg|i!pAq=IM}2}3vLZm_#cuV>n*+-B=FOWZ%J$t~1Ng)YH45lj z+C5ncX+cS!ls5R$wwNvibVoIX%oZBEYy!JDtggryg|2{1;!CVgrU0 z&?giu;i5=YHG7+_=`WntL#%D>?Ex2wKYfv(F{bmHDAU8_WbIB3EKM2oP!}c`_J!rUcY5in? zryc!lN3YEH9@dB9?E1WS=(d>ITX^F0w@rIU*H<)1*6&3yB4W}QW z9#%lQZQLt*yK2aGJ8d=`T6Jdmsr59`;jXXvEvzBfIIY;`*k;ayZ)1aVYeT_ayMje# z4&@ey_Bo}NI+{8>IBSW{-(5Kx`mWcvR>anrAX1@emo8i^2R}794{@B;^N}nXh8^eR zUI@sCQ1*nWk_|^CQN+pB5bdAyab@{TR4MUpjI8R4rtG%eB%F=sVO+?YVrWsEi|Nne z{%lHVUcYvCzyTQoI*DF1-H)^ zChd<^ljA$0&UUSq9jz`QPA0E}+<^!lsnoTPtmv94EFDwAtK%ocmRPg(?rlxDgvLWy zeA+!yJNkzcS$2nQBC_TAv}@X(&d1tf#!pTbV*g^|HNfM_z%33v2JY=UfC<+6{%&8b zhfCIo3d%uJi&J+jk(Al3Zupvh;Tp`*i0TiT($-OgY4mtPXV;+%vAfbiuLdcgi*0*h zm*|_17KerxPi{i1zvRezeU$vBvrv9gaBo>G7Si2H=4421B_*y`ep!(ES+as{V?d$Ljz72r98^3oA@lVc zV8h4SMu74g>;f9$fsMSt&-qedO(oYD?5V0P52nV(o)woL_xm1jdL5e`n>^2~T3Rg= znhPee?pzkg+3`@E+^f04`N@ODnoq!Suk!31Sx3aeF*N2nuoJuLAyaahXRh8bKN4>t zkB=&10CS|qFfW>{2evux5){V2qZFi1gix`oPI*awpHkM-<3Q(lrv|>U8mHBNa5l>C z;udgl^>p>NKAqE-%LwSlxVckka(~x1-hAE3vmsvn?aor)#*xUy@~Dm(r3f#n$>tsd z#*ScV>9Qr-4@y3jk5%%opT7)XXf_T`*4ZaeUN_1V8NBRhXSC5#|Hu*hQU#WBv5L_BHzef${S_hH&e%^fo%S9`C)pYpX7A z)HuQ)jqR?qm6|TtpGPySsvx~nVHby!iV>AHjP2bbZ$HY`l7<@v-PV}0kK(w9_2y1B z4K8xFn$_7kqPj$=8zw)Q5ct|=J zaA94c82sW=58wUEGgL0pVpVHx*;1jDVdu>$CmOvR;uW-|dtkVcNISU~`=Iutq%du`Yz(URTI$ z<3OfCi$yp}YxaERR2_;$DFu7(XtJu=Kc3vEKVwB~@C3DJ|0S=*OFNfMU-PaVjdt1zwSSL+o?>&M|x|fvdaYsx)ZjYqYxp7vWJ%um#F`rn> z)89)*F-Y4fd{wqxqE8vZ_JXA^=1U`Xn#FbvGoh)GWteyyhHlfI|Qr?lBaoQNc z0+B99&%(6!q1=}XD~UCDST-7Y_8Gp%eVqyS37`4Ff`rY4%e9A!5qC5@nHkS{Z^ZJz zyjIi0oPrH(hsZcwFw0uT@-5Fvhv*RC1)P2W3hkEdGzPNQ6mkRpK#uNbnj($tb?{Z} ze_R0MJ2nyZ6J-ok4hfYg1T+Zo&WDa@NHo~%Sk6jj7dtnBcWp+WNsmixJY}^REzYR2 zHCoM^2eeltU>;&{W(XM-wF0{ylCHM$Z3atO5q%GV8=1{ea6}Deiu_!Itc~<1c)^*Ut%vqxa*-kO5o3 zc$^WxPDtXm4A=A8AGE(XUhDu-0Im%oAWeu3TvN}0fFul1_E&O!Hfs+}Kx{{^7;=6z z8wxzo^wXt_E5hrg{U&Ee+kFLwVu+dI0qcQJ%BaLc0K;ak3P>kaZPuO6#hND-5pCOI zPl%c&Su~_KxwfSV9K3Bh!Vz;Ay zjki0swlcTaa=4DBrp>x0*SOZSKW-T2CEr_}=XQ7z&}k^?(UNEbAMC65th0$g<~EX? zzGD93biM3)$1mW_6bKM!62NNO8%aCs4^c_AI-%Ba43Sd#ItYL z>zKumXS+MS(p0gs%7ngn$YUh|Kx*f9eFO|`-Ey7RXWqyXq#3OG>&$i9-Et<(6`I3mZ78~%eiUwh*gUkQh!#AAlPBp?@?98S!@m_iw}KwviV7*9poNp zhYdXP9w!Q_Z&a_cBUZwPtY;ei&5Ui1wtko?@vWl8>GdbM#5vaNb_-0;dM`oMJNF0J59ArU|+U2H;A_qq^bCLGj zqgisvid|i{gdXK#jQvsVp+bkTyR<>jCF*nYavLT?QyVTqXtnX7A8p@vSF%io=2DBK zRYM;^rF>VjIDrD$!a6)3ntGwtTLSK9uRM<@iF!@q)!!I?NPf2J-7Gs~ynCAJLfiqu zfsEI^HP$P|$tK&?iRh~@%}XgYv};T&^ehMNS|bVEmKO}{%eO}sO}E+R$3(L_Ub%_I z#CK^GX;)u3g^!w1XxesvW;28Fuik->o*p0Sl|=38!+j6T2J6W{;vf(a!-iiND;~lz zmHN9YIX0PJ2SELa1M6P;6H9{Vt4^AxcF6stO)gyW#4}7(R4hkQCuWxRi5~`q(87M- zXyx1Ua^w#l5X{mWx90efh|;Fso)%>g37D%RWce{_Hu3EJ6;nq5|d}`nsu*28S)Lt zrlhSj3yc}O463UtLM0FwZXMQ!$IWwne#?v(>k_>taYQ0RKAEO5q2`JS$iiZ;Id+R^(aD8D&T{ zB4#;<-tFs6E{@FF%5Je;6U4z6LU#jXT3bN|nRl2%%}z5hK-{p2DabVWScJm(?r2}K zr{dT_%bP}GSMmYm$m3tU#I(BKbme1;cf^SveE5)?5W&NZGR-cEK?hVEAsO=01fpX# ztAW1mpYgKiEptmK6<%58*Qn(YV;w=K_RX$ckS=cppS7HCfsi^TkXZVLC&t<%a+pAJ zZJ9P4h)v#uq{vA*i;VT8Si2|NF}!vB(Ia(JmMZ#eQ~y$@tzh!dlLa!7?;DhS zh6^S(k-1*do~J{fbhPsUbQR{VjOWfK9*I!qb3b@PDFrKvnD;}tCmg$s4L)yktZz&@ z4js_BGGt&7*Vre`>!ioU=6-g!cC20XTXz_lL4v8R&a_bPh}p)9?asx-ew|Ysw(!c~ z8+or+yOFx?9m~z~m#1Qinuoa*UB0*AR+rhbBI}B>9C_Rk%I)k=)I$4;sUk%PKHUM_ z3*pV1%nK$7Sbh>ft>jOPTgCIW4j7kY?T4j=aNnZ{%D|2;ae4Nl#q7)y^u+=QLg*D!OjP zKprILC9h2TEGD{ix*>g*`7JLxP)X5~k*pB&%-I*qbr%_Q`zQOuz_q=PG`+@+bPzDA zR_L#`e=2biW1Epz!jJ;OD7V$_i?rmga+hY6n_PmPSCuZI9RNfs>l9oeLn6{{rNMF{ zhE7ZpYPlj!rL9*hFz$)tyD?nq{lz5C$y>po)?)7wnU|`P_OWWzc6qy8U%uo887-n7 ziDBK3ffx)%KQA*M`Lx=@L)Bt4RhU_7tJC8-z~wsY0TGkmAaRW{*_dF_^%i*=k~lQw z%9VkX$9Zw6y|+g7f|$ebo%r)mJ>p`rK!U>8;>N~lY%34w;cP#FVxJg#V_;(-eXZtN zdE)pRIP(cIXf0F83%}6M_k#4iz~`LBM8Wa?SpX9Ni+op#E4tNdoNX-uJX}i2@Ht;K z%xfyPn-m-^wTxKO5L~LkF94aqY_c{q>vmbPsM%-qj6Z3l5!s(fQN=^3<%$HrP_ssKAhF z>y08p4hZ2;>?zp91Ago#WbBM6!yT}1bPc{|52!n7rZLeeJ{h#E_9+?pbJ)HO;md>} zi{y3}++dsoEwtx>qAObAX*fj)pp|k96~|1wdCA1?926&y!@b<^6@#zwy|5rC>1*F>Tlf~eT#~!suz^8Y98bJ z4NMyPXLGFtv0Ubg4JpSbW^xLjmUD{E<+tF->gi>WmRH>y{*KaK$4y+FVsyRe7TV-) zE@;0^LCkPOfBD21P2sC01fKfv?d8pOVGL|FUtfm8{f7@4{89SF#y~~*vjG8S5QF|y zXI4spTsuu0hvq~SHwIw$1;y(bzE-wxqw4Uu7)W2^XM>h?=?W@muI}bV)&&Q| zexEW&0+8I*IO26Uso6P{s_C*iq%h?=d!5UT?`Z>?+mF}8oy#EW|D;=Z_wi_*R}D%O zWxVf*>vb#Yci!rWxX@H zE>tFicneLk76YvFOLk$#`2S zTqcYdKITS(Aml!ylHfAkZViCHl0y&p;zT^ktMc{>!iPG>4C*7o!#jI`T)`BGH0^iV z80A56&-si<|B%_wK7b-!%+nZjRy@6mj=9E$Ka$Zn<&@zJi0Z3;V9MThWmQJlWW(a% ztNAsa(tUgnSI|yRq~yIOI7l;OBN!HDKhQO&rS=FAFfpY-W$5VWZs~pbrRmkg!G~0@ z*!SJRWLER#Xkq>k9|4G1eyqK)CTU8yM)J)Z(~hWxr{^F}383Tlh-`MXAN&;;$HvdXdJCODXxbInLk=hNv6h2%soIh3XkXNroCpKgsM!3U@ zPCd64ayX&w(hJTaCl`w08y=q;L{ASk!X941lZh~6yP6C26T;k6cz;sfy4;`D0aD^~ zQMz_vK&BuMP|ifJxDF77%@ch$$OI{F=I>Pw=DmDdRaKxe$17v$wvB-Yl{W@3(1m`J z@~rgqP2Go5S}OBqdsqmsQ^MCVI~&`&|ayETijrDGN8E*e@`+)|AbcTSFpmF9)t^u&w-4Z9gRhBCU8m?9V?n=lIU(;ZP$+ zHUv6Sgk9asqUVewLy73|@jfF8VbV_+T(A`MUPH4F{2WsDd4j;Y8Qo zs$sZ^_U*;;p65IT2I*3Mf|{Sz+A4DGBimII4Sa__4!sQ}ajxXyK=v4(&<%Mt@tah*4&on+?eNyajN_4#awoIj78Jdkq z34>MfghAnx&nDv{i@m#;dt121%QxwqR{07wu6V*m_tKT$!k>#N+KuXbR?B8SJzS4e z@2Cr#9IcnDsbuf_Y%Am+Jl~X!{;ZQk6cMs!~CBlx@Dj%fFQ1 zAsBJZe2jQcw~1TY#tSZVd2@B4-NT=>RWrRy3@$dy7U}nq27J-_j>k`X*Uc2pA0@dh zC86c}&?6@k>X-AcU|e4Bx!5OmTcL7ktB~qjZ-DvpTeo!NU!+f*E-_`RRwR|Ro?WbV z4afNNCdTqR_&<)_OM>$q(5JiTg}{W)@tSATTV!W5uVHUKH2O+_5qGENILU`UEnlq| zX!q7!v+3yR-Lk7n2Tf5{yZHI6)JzlpaHJ~-=G8#_cZI+>yz;Ykbe)kT(+}cQYf_$c z;ub*Rr>NA}sj>M%<+i%Um~!DdX@j1k9j3Lm?r@=k8i@rTfjn{J$DyaAZ>d_TAflS( zut~io%OuZfMYqL$Vhli#Zx>7M;?gZ$6~Sit<177WK%SDEr(G9koreFaz9d|W>H4pe z!b~W0eWgGtHVzIQC$fBQC11t9Y$wZKP;hqf+4`zT_vmB~SMz-`TTy3A+r76&J{-~e zRnvzHX+U$;u0L#emux7XcX402<4V#|1zdF7p=vQK%PZ7vW2{JDO2o6kDFy?<63s@m zZp7=|E*yK`t4N#AI{mdnX2;az=uqx@9iPp}(;7kLyMM^llw4G1E|SGl3M1Pwf!to0}guUq_!DbZOP8 z4doeFcCQv0o<9vJwH@wF7@pH<;i_rR8=j-@Bl6-=-Rnct`oL z0}+fl&O_|Tm$lDRSbtHR{=KN9Gno0v5D0`LfIPVU6%OG?$v`6Dw~nsnqi6kDMu15} z3&t*`=1kKTzdb5hPd$bavMNACNJwbQHJ)6*Dj*{N;SP)C!zgc&;R?&13u2JT7?MzU z@3avYJ=bSS!V;yEgHh*ky|Oj>PXFz!JD2uOqia2!PkW?nPx_U_A^{YT1B&W-lraSw z2gyA41k^H(cW-=ewDzQ&^}b(B_<{dD`fMvLd4?XupM1<#xWUA4uahkV5}WrV=?^UA zqROhrh!R*l7ilSdOORpV7ZJ=S^QMH=aHnkZ8QXZmkNAURi_rx~?fvze4q^L88A$iU ze8l~&u4n-4c&9#Vj~-?|Mm{)sc^JHRmL?h4Noh>wx#b5|EpYW^%^sn$9(h|7pI$qkf@|k5s*SL{bg_2fxSJGDa+8v z8#5Ad{MM}C*&^h976TjenHlLln@=@>iFX6mg_QBt$&0xSfF<@u?j(FW! z@)I6LFG}CadZO3;>A`%vx}1KmT>bU%0wGGiDaB>O?ltESnDc1bak0flblS@UT}eW7 zvSA-s_X?DMdvU)=V|2j-kOG``6a2fqZG=*qJjhpCvXr&opSRrc=;Vzqd2~CLA9>}M z<^zS4uU^H65OE)$tQK^+D*HSV@jm01+mNFcuPJ4$PY&Is~#Mdn(Ae9w*&uUK(Hz9`;4oX|b3${#! zR)frLK=^L@tcSNb%eK+Vb-jMaolk%XlEx?dw9cfZ5$Nb$fgRhsCve8`puAlAYOMO;!ID>|^kvE~>|r{Gwo{s%|HdAgb=JW3+_YE?V!EP)em zA`kGtR|HvE4yV=WC4KLsj7%fK=6_6m_wE-Xp3G{1;s}7HdfgIu0%h4YmYjI~L*UA` zu`9ViKm|%f(k!@aZwAyUVCh4GnrX|xer65jeUBH~w})D8oEu*I{BlhUx$p>OY%Pgh zq8d*`-i$7N%(U&&h0dQV zAXWx*%##8tN+l%{pg_Q)irE}QLJNe0(~tZHb2NdlW$7ycq&sZft|r)(L*kJ5Edb5t zKUb2e^`P=94ij0qxvb`sJoCBhF$0!Ols^zsGoLH%k^6 zvIaK40O6s(_r-29^4*QO9H@dP(5d!*y1Mx2S0v|Wk8J_xcUXAb*4@qCqMT3xnscro z_BNyrgd9LQwYYE%*cS2EC^4l*Dkr6I>_Jo?U95KiOPSR$0n+azaC}O371&xyF(CHc z307M8c%rJgh{GClF4-{Rm8x@p=a}!2$Fs}PF3oTzl8)%s_hOUGlkR&5c?K2k8KtX^ zK%E^<*b+h+hJ*Cd*I?IW)#W{BVpFueSKCwnh@Hy57IuFY<+1?GB!jFivH0`RwH>gD zb>J$AGF>O*3%io!Dw;beqQ_F)Ji+e*u?-N4Jn-1UO)oIajY1)KKCwUB#M?dih z>`LU|nX^=~9Bs}$bbwCbRn_I0t8>{HVLlYu31ZdfR*Lc|d16<&E}E@d?+4%SD_IcDsn=z2xj{y!1mDXP zV4ns!WoL>E(ox)mYl4`X?Az|lx9_OMJ8`$`N;=5)JnH7J+6m=rhX|Fo?I|&~6+xxv zXCC--cDfS0t@}t@nQK_mzDA*uA*a48tPr1UW5@Yj=D>7)?A~aNcbNNi(#xA>-N=)U z8*b}Ww=4%f#(oUymjAjm8w;yDQ_{X~9XAJ7CF%$9O-~(Ew0u;oh4Z}+al5#uUFBFb zaXdZCNxv?lUA)DTty!Ns@ocrW@6;u_H7k3wbs{!7wgriJwJp_W7rZ%gm*vY>2`oon zbp>_+OK(>L_*a-Cp1&=17c3A1eu57Klz>cG_bxY>nu~G`n~Ry;xc@Wgd4)e>BOLWf zzX4_OZ%_y_RS5|6HOfexZ+X3`I_t}M8&JnR8=W}bRf|U<%p_)-A3fU=Le^F&Vn2Lc zvwcjRrIXlJtw32`6zTe6TWi*&s1STncOZhY!7y!K>5hD+Av z$Gj%-r@=R6Bn!p7>p}sMcS?Axe+-@Gw!KHZy+L`uXA!A2|I+1579im+^0&FYvI=wQ zhg0DhhCLC7nQR~0BXjQ9>YHvjAQc`4y)uM4Tr1y>YV7rCYYv zJ!V?>ly0s~`>&j=^2tw@>ve_GYo6?8XrUh?J6AHI4Z;BffaM7mcZ5l}B;7ZXXeh6SDc6@PCX3cK z8~}dAG_}QCG6^ioQq8<25<;ti0(9{p6a$G2TA#Nyws$Up)trIg@9g$Zi|iYW)xGMH zI$H9XTm7*i|M0|FpV(RN;ky~CHSZ=}*}PY_I>2Mjv;9%>N)Mj; z%Edutx@e~M)J|=0jzVCP#~N-=O#hHueUH}N=+(6U4;ddV#i0gL2S|c4S{0lto&YXD}RY09;t#K*lvBb8tZ$VqvOsVltMvtrRaw4tpPySRqJ~5c zYjD0sZS@)3eDaQX-+dC~iN&l!m}*AdHu#~VJ!OU2?IMk=is%Q!dWLBalZQ?m1z9{Mdb&rRZcYR*yrIj@o82k$V}$g;Szx zCnBT26|{!{ryLa#KvuFYVB0fKXPQ*>QJd!iR!~&7#68y@h)DYiuZ7+cSh!@K5D>9AHhgTj35aI-v!93xkQ1TTh1N-Y-zcu7Jv*+KN?21n?- zXTZ5HpuNK((WMrlMIfX#*$B$sY5&yHS@B$SXM&}#+$$@bQeqE&(z^7<$=bwzrRM8w z6n!T28z=SzSl_)Ml6e@r@3s*8m9QAlo%^=tw#Xq}*lqU(dG@N04F}0yQNybx-xIa& zg5MA6OL;JrKKl$6^8C0t0%$v^Patv4loH4dfEXC8$F?oSG!B|(<oKP1c_Ue>4kcu@2Tc7_rWYAQWwUb&9v4R5XVK$EI!GNv zky8oBGo-9ana;j1^D-H;Q_5+x!dJw@jjhy&#ny|wmSbmgzWImfx0$|lroOxq%xAd) z*$=#u`;dyOS!=uuTElJoJfN}p%_3h4rR@r_o09e-dr!rbDp>2sX+PJxL;l8#U<>Zc z&|=n|aBp;URJj7I3^<`@QU=yizH#nzFd!DEIGmH&Z*7d(u%K^tbpLWVZaTGM zYac2gs!i5wx2KuVFQ++Ly;Nz`r6|f{LSa3SOMl}3DBB?Y&GA9&4yEpui9;&K`qGyE z=cA6S+`oS_A%lS0fHgR(q=HQrUg=7)ZA_QX2{vj1X8f2y9L&=7wD$+v%$*JVBanxi zPY)HfmxfwM`eXL_97*n}CV}F(}#t_|wB?sJdv zetX~XJmWea&e*c{UTdy7{~f=-7WxuPuI;>hfgk8$vCm=H76nz`J486jIh|;i4ZJci zw&rh6JXY44Kns9BN0jRB-q{rIL8E(l0gdGb)Pqtw75O?2&92jI^(aMPt3&jj*&yho z^(MH<$mX!3QMNY$pZ3c59)atD>D)4I;WBpL+zYYWQ~uB;Xg_{xYi9f96ItC{_eti* z%d*+4Renb38jSD5!d_Cp(FN!cBSmIO+AH`Sgscfuc%PN zIVFNV;axsn!(^yhHcv^>K7343P|XGJwG!1@^k1SxmW({AB9lmOZ*)3tuW~9+&o*Gn zo7q1|3uH#2YOwI?!z(-`fhM2C$mCYsF&yJq4G0tC-V?K%)12GGmbzF=nIKB^J2LKB znFUpCVS0;J6S5ZHpLm?;8+oaQ<>{%80lsgu_&7oHYz=?D4}I#ywH%JrMW_k!abGb2X_HrIlQ0sR7J}~RqJ+Pt zVpv`T+`%O$U)@TP^SE}UzM;Vnl|uC-1&6Y1iJvA)eSRnrpoj0>Vlz9adi0XV@-I#K zAi!_s#iahoI`g@u=@O<_QWAA1sy&tTnHIx4&ngianLt3q(6ZG%9oYw!JvYVW0SpRM zZyqw6{#F;%cPZiCje#`PU74SG8GL^32qhLbopgbaeracW!@$D+BhWk(*x(kvS$mga5j$OT?nc!s`Ds`__A55 zS5NlCA0VZe0dA1Wf?NLu$P@ypLg0B?+15UTdIb~1`Np#o10nkp2ffQbDU$YS{w{g& zsoQ2Ej`1UW=)q@ivP8{hG%hyrm|yeyg1io(Gny0~QE;ic0!=Gt){8$L@xycDqNNZ?x6`NB?``}!R4L|QuBAl|L_LJ5{4 zM7YB=Y@Lot_e#+(i_|QBbjTKQbwz-)^u&X@5kY~A<9HhAQRIP9*wtLa7<>`%Krbl`?fT1ec680wLdNF!mlRl|{#sj2_p)zDFFV%U@OFnkKpRcd*N@vxT}=&=0YgoK%QU|ZE$Kj&zb?B$mM8o|_DXJT>s21tgODUvhb-OV*+@1v9R=0%re z$#owhv(d<#*y%u5Yrop&H{`#4^~T#ybFZ7|B&eh$uQl}=4P~eK6;Pp?Bhoob2V)xa zEyBD>ldwK9>~HqiOeeb=T`ecD9*%s-Yj_bd{v-pFlal&pl;-<#;#)y`wmCUp2bbMv zN@ynlRMmVkO>yL-rbm;V&9fE@x4tXbWZuP|x;=t6La_GeV_3Ix0?0{PW5i|6Dd4ga zEZ3<4|D>|om&Tva?rvZwP)Suk-I+TaJEf7JSeCyg2m}EEI*mmtLwE&CPSr{I(-@v- zLWfKDzkqONS&%{M#nNT9fu@M^Qso!l)3bmkveCL26#p&w@uK~=F9F6p9rH@NK(pD%Pz04hi8|^aHPCK*C)Ke8mCha&w5M9ILBZa zWeIRH7=1qvom@(FmKdzfKLzdJ@%)Od=| z8khl?29Q^}Qk^?#7U@lSfLXZ~wiABqB%s-Db~svrQp%l7lA>P;b9GcRx6}iM|FxN0 z1#n8>SKH?#=c4J)QbBnH0`08#GpwX-=~w6aV@!aAV@~|C=D-^TKfRc{S ziI=AOTB3laD9dR1)wnHkmR~8wjbV5XK#l&{AkG1^RTwwmgsO5NJ@KE43$Pi(1>n9% za~R2gx!u_C=kn+cfEl)_^x6nuD4~86rVjN0T9-sz1Wa#$9v4^m6f|4SeB4obBnn8# z<*CucE1Z1CnfyXe5i~Z$w7pwKe*gWj}(n`0!o}NUaSisAEyS^rnCtF@=a7NgHwox*|gniU~ z%z=mP--=&6f&5hWrs;mW0N%$?%jfEtXH9=hR8*q&tE0?-4D*p~A~a`*W|;_qTMhR! z4Nl=gF?1<=d#?4*;@RA7C^O`3APU&(Yt}trg0NH_ZOmyneLw4UhqnEN3n;IHz^o3P zQ7Q$Jk-0%o3Tj6?k@nBN|M_GNIzeaNWIQ4MC(Zm4t<6wV^;6TOb6hwVs2mBx1pwjw z$Vj3;c^*nAwHG|4GPQyq2tZW8-94$GcpsojgzfXe>)+biIE7!F69s@=1Ju%;aP%Lo zqyPE%XcX4LS@^4%m%isL*I7M)CJ-{Sv^cvxXoBF;YrdItzr+1Zvr;nrEY|*;LHs#m z>*XLg9D5U&m*kwJ(6Q%qji|r9QjB!)69L6GG4hHFpc4^Vq%JVCZ56DZx3hECbQ7H2 z=|2_%fi9%as%(`>YQ5&Qnfc~<;i6=f?##YBmO}Nm94Z5k1X;QrYkjVE8sFs5^a1_3nC&WVo#{}M4Z%{((5 zq0L#>^?aVvjjW=3pf!TP*$`1DAP*c= z|NA?lNiMr9Ty#)b9wf|o)IIO~9piS4fVrURP7&Wl4$n-7w`mY3fkt*pXFJ|sP|lGk zZ)7~*S)78My^}v68E6NTOq8fL6NRm-=!Kb6%lyH9fG(X;^>sHOf;+1&{rJT?|6Cn~ zBpQg%+>;cGId5K<#VVc75{)Z2e--m=e zM9d{&ivc}{Vrwsmt#94Bbw=t-_!}%8fKNQ3t zG?0t`-=l%p?bTdc=4WE8Qe%wnGv^kra99a<4p{sqk;)minZ3l2)y~J6;6q$FLTxju zbu*WVmP?F5F`K%_L@_kNig{}IfkAXouL^ON?CG-^nWKi7Wji~B?ObQCnx6YSQPvBO3#7HxW*35vd%kFwLRb%Vc6XJ@KzCmFhj1|nI|85O zXYrjJ;MZlWWh^J)nQN>UBncgVTspS%5Xri>Don}mK1`D_lO4M9A&y3Q#3R_^_+;tg zXRQ3%Wl3fxrp4`T+ZM7=y9NBp5jajOYj4S?$b;fmr2Jz&j-56th@B6}d~dO3`nulN^I1=u{CkrGTNtJ< zpfk{>LhjgHa8UArOxbpuxplBuTSHue@nZ)NMed-H z;ZeZ0`Y}?=z-xxsW=7LUOT zLSE~&v>!$)Qc=W1VM!dOFj*O+F%9?Te##1mrSPv6J5nPJMN;{ULuZmhgF;AXbeuWi z07mc|*z{loy&mpP^c3Z=dq0K@rO6OoP+dsbzg`i}&C*VOKhvgar;<$9JvuVf?M)bH zAdC8YC~hBKW>FNSm@Jd1R^$5;g@1`UqFB$9NIsB!o_JivC<^?t8tOBFat8_=U_bL{ z2M%tFP`ua9&IT)AeNlNLLFc2ZL<@m#DhA^<7_NWlqH$_0nmhtW{`ZJSg2}c$YS(8x|w9=V7sxGmsmJv%$-^C zM66BS4uU?%|NM+ql5rAs`liJ*ui5&w3=zVnx}k>gA8=)9n4mM?Pi}?;MSn5?8);@L z!{En=2TAS{sSBE#LXjGNr#NL1iLi6+9KJo}i?65tWU2Ziw{H*=Q-!BL4;jQUmIT^Q zqIR`?i&V6v&rYe6Tm9097`*+aa>;Pr*UI-%s_ExnklWU<=S2mAY8Q%<{eU?c1Ilcy z9%iES&D|_d&Y6AyT_jbJ!CXSetKZThzad%VZ!$jQxIFQ6-eI|{a{p7)Yu9cq|7b=n z>AePSa)Hj%pxH|RztZt5r&f_OzKMJwsnASxF>T-hbB27tP>c2WZYW7+I{=u?19y^{=WIn?w$4ukKLO0Em9S^?xE$C zno6x|R5f24ZDa0d{4mpKJO==*et>?fmd~_hTM$7ZQdq`` zWV5FvMZY4Q)x2PQbh;{y}LH?YN>+a?v+y3($|dUo%NcD5FCf- zrhatV$K0g3^^`VoAh9W$DYX94Q~zgpvlsz>6$z!D#IFutT}h*ewuC80+mtfy!miF* zSFGk#ty;IwXRl-*^x&SH$l57#qJb1=_RLmo93za$w9lC60vbBkI}txZc;PfCxuD() z7>Ljw`WweU_SkZ7{)0z+7MoLF(KYX_ZbvDsjMp*)bpE4uBZ-j#ErmNmnvm(qCTEtQ4@}Ew_YA(4#weF{m5)4601! zvgE66W|U={Z|GW77IOqrs^5BJCdg69!F~c0U%B1Y{>F4@sa$WbHaz}{5lcZbJi><~ z$$Imd=)+^IL++{a?{`Ezp?h17I?b9LNTyx5fX0b?owe7y=P?#|%rv?m0qYC{ul;t9 zvM&ZUfj4MHDs6FF8|?nMB3N#)`Ko9(28Vgb!*cAsyD79VIT9~W_l=**uGTe;o+2Gh zFw-8-3NuZfMZ&ZG;(hA**9Eji0BnPZ%;;fIuwLqtV%4;`%=-lTJLFKszZZ%0bjo{pA&f zm4u}P=V~%;%hf3D#`@9If<8h1n zzOyq3qK8i@&8ynyEPpn+_%b(`k}Z^D--u2h5QQGth6T*x`zQA&TGD1rmlluEO!~5M z<{NvTCpG&D^nqL)v zpI3q|`rr-LY>movU4}^()QKmAVoiZcei{8fqxAcxKi;z)Ml?=L6(pDPDb%44Y?MU! z;;&|2$+iAntyOm)rK(V?e8do$~juh+U& zYus1vMUG`fT<8?ivZ_HDtg*{QWjc|BI*dUA*6h&)M=L{r!tO273FOsPC2ndF&uUsc z6roP_1AL^~5E`l4`geHT1D6x-k2)Um44jPg2@=h;In$a}?andnhcA4*p}9%s$i?L0 z@;Gw8PrNqqew@EoOEsG#{KC4yi#hUHszvTn**Wf9*}2XFae}`Ek9HgDLhr|89bekU ziQZnojbMRA1W59)C4H@X&wSagVI~6)+F#R`UX0nLI#gPn)0nm~)x%{rbrVQ95+3kv z=mzWe^7&x4*+BgXlp#8mES1he0v!o%e~ASCOG$wElg%co-5__JR*P0F5-30ZeqeR4R34Y+$i z5)z^izC=Lusj~5LG~f7ikGT|MZ#Tl#>mAumlN_y7{RzoqSYX4?-4-jM)0g`yk{zPk z!(CFU+;88WRZ!n!V*|S)+wLx8GFQ!W&(wEax2{E&^>2O|BH}j|_c;#fk*%sxuT2L+ zrq^d<*LFVZjr%leUB!-fqjLSZB+BqE&o=nv_2HX~GNb$gyZut*O^1Zuf@398R=7p} zXXHyE_rmL7ec~v9h;>J5UB|WgXIa?<63xSvBLIBChX4c$pX~M`dF#cN*(m)<_?ZyI z;vUKS^<8gsdl<)XN=|l0Emya%`A`X4z2M^YfCf6`2niO%e_DUm6vnhTn#f`c?<3+s zc;E$ZNl%jbu9C#POKhWVj9Tme#fRT~L;&ViwtigkoArq$B37~=4-rK5@Ul6*a6pKE zdEwyZs!hL1v+K3ddB`MPJeINQ@-7DmVBbQuX>EfgLYUjkw&`dd=W6BdUA<%4q?f8T zjGDcy3Vxx-LnxLu!PEdvb$5>YF{hqUz)f^abxj^5kg%Ri!n?q$*BXps?w(Euceo;m z@{?4tmD}v$gng-#kQVi$0g~~Vb*4&uyS{oiw@j(W(Z2Xgyba--%0;bW5yy!y)q?K{ z+V5|4=lFb>{dzls$+e=1Q>js$t^3ijx__XJLO*H+YiLHV6r1vD9c;9TBgN`d$<+_h zb}mI<0qMrsSDjfC}!WK~j=~zG_OPMkviFggM2{K!*AUQfqU?>RF=%9ch_Q}^L2Dj_s4t;RKczjm- zm1~}ZxX@SdyfoT)x%8UD){zDQ6lVGcBhwhQlAM{WxTy_seY4u2JQLG;iUt8o># zc+}~o7>DatNA4Jw%bREqCj*UVhB8C~I&wr6jcK{lJL>g)7wd-kXOLx0`o7Wb&MvMb z1qS_@efru1>MeByR+Og-m0MnW(&HxMagMIhsl|U!2HpBV8h`73)d8Q zVNchmq}XA@Kr;#I3edwcjY`S^+8|^wbxt5PXc|FNeTf90;5BZynNGsGF%LHfp0(G9 z-t3cNKIP@+s$C3j-45TxX}!(t2kr?tZVp?X;#@=SBNryT(?{y|bRDj#BkNV7ESLn< zcz3_2)*cGmSrV0vtgQ{&z-t%wzD;*9Uv8vUuZqN649?Oi>j_l%9`f+eQdwIAm16pg zW2*Rpd;MP8YXlb`I=i*U8~6&lAEsC~3zNoX*?zptZ--)uH)!+0g`Y75aMb5CgE8!f z%t|P^bJB~8Xkam&zFcb?jV_n_3FI3Vss435{VW| zFjOD{DzwF;J0guiWA5=ys@-GG-zK5wbSN36|R9$z!R zI=|jhkm^n?6_b3=DBDl>bXR+PwdyNftAA)2*-qzi0k!77YZb7#&Lk#RfQaJygu&>C zEYHziM=KSQ$2&FH!q?6B#@oijF>-umwVc&L9oKtrvYu>@191T?Lk@B*2O3CLI0K7l zn{DI3dh}wu{#bdBtbcNGlEtz3YPlZCG3VVS_kMTn4;PDr_{U79_jGr^&Cnjk$G2g+ zO;Ej`$88n7I>syIUo<1v_f;LG1M+6cl(1}@XOo^8R*)5nYRu1#uk9>K)g|zJZWV*X z`~HncP>EN`kXek5S;){4;b=Adi5(_jc}OH$=};z#*z;&v*Qv5^fJ@a`&616}A{^JQ zj#mqy4}S+DGTlwX!*5Z^X&?Cc%hqRi>g zF9d1)xhb^AYEEgP3K?WxSMdy`5dDk|mechr=_)iZ1VsbfO&77(v(o^#@C_a+xz4&1 z2%w+Ey9-IUpdWvWNAX3L(tdD|EA`)`d>kI}QMdbeq*JQqkjS=CJ!F3PfNrQsU#5|z zKo$=#qfAQGGHmE&kb;o)A)a>6MsXI`M={@Ir*AM9mTcpJ)(EAXr^14|9lxwkOxvVt zqtsY}H-F#i3NC3yeg!5G7c3Y7`1aN$^0?uRxVecIqF)#ZH^Sf2FXkvQBVgTmr}gv9 zvaO~i_y!a`p1R%O(CW?C;g4CXK4uC1NC-*34s)Cc&t3b-U0d|wlsTHtv{OpzVT=K9 zYn%`_g2O_aXMR)s%{%hjo>W{IeC)A?{%k(Z9|S$OK)ysf3;6N@=o{oI2#=rykzbJIK7{eEGy-d< zGv0|OeKp!YzEe#uiq|P>Ro(f{-piP&`jrcDn)&XN@=3C0bSERvTZ%Ey!Q8njo|uWl}3z zB0z>`&DxkW^o&-yZSk_?*E1gUs>Lc4gvE60AROg1@Q;gUCXl$mzS?X}`63{X*b@U< z>&lGL?Z6Xu7=on%K0gfTe7CWH)lRk&G-8%si@F4q21P35f~cxh?#m8doosAq+Nv<9 z8|i+(%fXk)B6Q+tNrKPCl`Zg*%tv6W@VJ(%!t~b3&L+Rkt4!JAn+{Pb9vP&?nCkW3 z{VbQw;%WN3LI#b<^4qu;aYvm@oc^}I^yX{OAg8+7+&s_zX++*p=Pb$Xa*JNUZ0CKs z(&OD-pPQD+jrV-bIeB7^e0F`kqIWGSKkuMUA(b$a(?m(B5mcNy9F(V1cr~CI*)4DB znlL~*kh1}v_oUm=>Y+cOI2SSNRTC1{238erIQxr&^@$yI?`tt`PwbCIY1bzb)@oS& z2%~r23qouX1hL65O`|0JqOJ*L$5(O9e(j@3HvM6x0fsQ6onx=M13jz)>+vwmic{7g zZ?~WU!Dxe}@ckSg;9!Nh+4MQQSj|0J#@GG<+~*Uz75q1iG^O3m5mJP#En=^eEsn&@ z`3067YU8wj$WSr#3C{g+tMyB+(3{=yA0yZyA1-m_M~P6d@1R$zSxF#{*G6APJz-E& z^~LgF5wL!ut`yS4{4>P4-Vx3MS``p#fb+w_`a#4ISRH}D>gZOI37P(?2hMGp6w$jq z#w7D9p3KJ-kzC(rHx?U5)3GDqI{Uskxl?nf0ZXYHaYOdI#&wwULeCO$<(7(`)1q*a zRs@sF4$zx#^P+t_E6fyy!j5IA_PhKd=NnN16FiEdV;`fJm=#7giF9l92E>)BFVqdH zg4|WrJ@8n2`_`h)?Pg#LgP{Y?{Qw$;FHtu&TOo>niB);{wzxYE_T@N)L)v2H%Cpyn zj~t>Bm)iE2$D8#sCJYNwf<#m9xsc@9sH}Bt#+Ju2h(iLFSiJh?CE~_TMW9{$22=|y(&wsv zwYKZ6+{$_p<+=oX&B(T0%O9zhN_PI zCrROST2UozOs+ctxzyjKBEDx-Wx?|J1gsY)t>jNe7FKsqy9-&R6yj)ENVsnt3#|2- zLplTWbkj!{n(_kPmeI3rZ@q4BK<~xVj@B&k$4fB(pg299O8BO6C5o_tv|M41KUdb# zu4Zw(XqM2YPfLX)g4u~qy(VA0edUt@i8hyY#rP+24L9o0jjDidHZC{4-Xs ztsd@qX}7MZD~r8NZrbclIxH}E=4k!6{<1h^sss4BZQ?lZrp0{Rfvr|)MR`1W>|*dP zZ(2$j=nogl;k88d{FiD`CQMmMygEy{mcWkp9BHkZb}J=dLZoG6w1@?8(k^sYDMwlklWDI~IW&8wLPF61o1%0wkbh%p z=#HJVjbC!-i#F-F7%`vZcl!ztnF7#F%rY_-37mK4KVpbwh1-_24b zcHVx+sC*t*0!uGXf)wEv#uvz8M8~v!>;-#)rWyTsv4+)-M zcZs{YN$xB}V%atN$s>mQ%wR(wq+~PdN?Lk$4A4D_lS}WR^MzLflt0mfJkFbj>WKc5 zaS7Bm|K&vhT(rpI&#s@MDyCGhkGrNFR(ae_W5TLDN++W}j?ZX~;6E(6ZqNIo!g4G1 zTEJ#7j=8;1K1H|_8nrQ``;n2IYzgWB>`V%APYfp8z&9q=o`oE3Q7xPIe-%lX1@MvwneciCfTh|$Ne(DB=C7m#!up|p_ zuaod?<1?Y^cjtWy+g-J41^>8CzhcZ7`#~c!`jo%V2usdKq%#F^Pu&Qg{8B2U0nbU{ z{KJ$wsPY&)PTl#XBC)N1}mkd)DVr|{z$ zIlQgjTc6~g5;A|7K!DbrB8?4ZFA%c!qV;j2jE!nxRFD-sCXA$Ft4qp_=(it$$-xLV z;EO^kmrE;ZXTZ`$Y|zSX^Y&_7+{m{$5pADMPA^NFMT;w&qeI0CplxQMfSFb(s!RG2 z4t-8SZ2>h9YF7_Pj+R=gRulNkaL%>`+>8wHy3-P;9ZK9#Vm=A%DJ87hDaz^DyUHSa z#tzvcQKFyKOU$(a1|jioZMK-FNC61=>8mcG95ERtxi}kBFqbd3G{P(=YD#4&i>oxr z)8O@d%&)NS+zs%3md>`@63`uEpHnC|<${^L7vCpBVVk?UAf- zXbrjV{`0)KpY>HR|5}8I0pXMN2>^D&@50#=bhf0qM`&e|fp(PKioZjyze)J6CwP$y ze9!QH>5@Nh$@P@Np(zhw=b5QU#XdfJ&q4_b>0Gr!38kSQmOfYR$SErmb_JbDt?OyA zC^tv?TYP!ZbB3O52hB73y^aaS9Q|GCzwzK(#<`q;gfLiJ5#Ez^onS7l4E!(LMLz#y zBS8>S6QWcoJYAS!b>)}WfN9I@Wz?NwmvzbH{H^7RJcD%=@yh>o15Xgjpl;k4v^l4P zf4-w!5p+(ON-AIV{;M1h3U#X0H} zG@`fwIzA3MAl+1KaB|O&?e`gh57lq3JP9;Qi`o3=E1WIVnw5e#;*jOXU#LapT47`( zHkzSq%EsLeFx>1suL4+g86q3Wz@C+YiDuL(Q+zm!scG29N{V~Qe+fITK7Xb$9Yp=o z&8)mi743eEzqkH3C)iqhfjAFPwl}h?P7Y*b(w)!lIEF?h(c*Aww>_BZxc-LNd~V_I z!80*n%pg~2^fyqu#N6f@kvvv9$*-%9rdbVaB*XZVk6b=z+J6b7uPR+g&xm4<{;pc- z>Qtt(bGY6}ZjrYHCQaTBlePfmo@1u_V2~n0VgO*-AqMi}_v-1Q!g?z0s1Gb+ecEUF z;9Jce9*Y9T$Ga^v(H3zdOJk?UV~id_`-huf%L2NpXdWNzY6DsG3Cs#v6w-j6!z35deaZe#2$sTdc-o ze9JpWZrP{3W&5t#1%|7qK?L(vRlsPXEV9sK&2i}oA5uVF<5ffUjgiRtr_Y`$fDh?n>%J&(K44bdgEl(5V}ZJ@@Z482I*iQnB%#+an^(&DFz__|Hm=|?!8~!#9fP+;I ztja|($$$Lm+^V|y9E2qP`=I|HwoZf{0~kJd5N@O;K!jJiU1~WF^7UYbDZ?S(0kS!q zv4F2NiZdh8Vr1rU1Wa01C6!8+k8Y=2mKY&r%Ciu$un=+7)};QAMVXiwebvvr*;;n>*SgAm==5{n;Drrzwv zTz9H05vTEUA`U~j=w@z~EimGlKgMCyNqc<)8F0-meYaw{SX~hdfn(N6^FiZA*SAtQ zOL?00yPG@#WP*IIGxy`aEN#9zw@f-zf$Lke3}8^r-1sLJ1oM~+ri^YY@tLT%N_VwHYkBN%iL z7w{y5Wb5}Mnn3iUXz}{G&ax^0MzdI;T~e_NF2#FdFOHHQVD>Y9GuB5A>p6h`WzLS( zQ+lKuBuEekCVtW`OHX%AdX)@YOBpUwIb9Rnl7rk6y~>ORLI8H^P*9%2$5oQ-M7L6P zdh&ioP9^`P#B0j=;F=95fHoy#AN#y=keOz91+n>k4IsppsH|=XhOAQs3EYqmq5sHP z>ERs1zWsQxiz;E>F5P6(%x&3JgX=3OjRVuXm9s}QKCZgX@(-*8xh?5RUyZkJrvKz| z3ToC61?PUBrfh=M5gp8Q%s>RuFT|Q`!gT%gbulV3N96@ZkMp(-!uS9Lxa8@6{Hf{! z=!)MFS{?!DjS-~O>%b(TF zoX*<|iU+A42dVN;)AaNJl#~K*#6qaM`Pc2ab;*Y zfNA8(fJw5O^kB~O$z4r!@fv=KxeR*U#|mzfhFOx}HI8O+0#omydT8hnZENnm{A)J$~raq_I|!5UdlP0NhCiuw2zG(&ZrtPIn&z- z*ns1P`kE#luj?ENv6HIvxg}aam1tgKXwxqYHru${L%6l}o0TQ4sGrl*7#H{Jz~v?H zAv;yV3!1@jGgHZ?2+?bTg@pj_UhiGe^EqGPBkhPs5>45wz}ZNyI@!)i-b;hgf5%e- z!@$Mq17Dl!O&UM^J-wOU#CiP-Bo_33Ngvv z9LC@fnvuQaUUj@|A!QZmR=f(}9a%K3kod`Ag~qE&SMBBHnPk+o4r&Q2 z<;ut35KCH3DfTuL=D0s(r}^Ccu9n9`c_zwK_-Oo^0oIqWdOyZT`aN$&dkUPB-RgX= zwZpVLj<*N$ii_{#v(Polk0qb%uQ*hxA1o+WsWH%DF4Emo*w;Ee==<1GQn{VARXo7F zEtRgU(Ud!YP$T=L!(i2B5Q-`RUiK2J}8~rfq{7 zF3FJ!2UaB3fYdsB)shM?Z$xwIl=86|0HaH;d41VHqQGf$rbqZW0C-l4(2$VG-BWpC zzXV20wB67L-qzH}CJ=eLnFrW0fyRrcoZfY(Al}T9Co75wkLeC={llmBoIC{L zUOt=c!c{eud;PjxeSKg?HKC4mYE2#xFWmL@Dv^RN_D^i!$!36m?zB-6h->+P*%?aB zsS-aYpQJV)Zhw7ZV}yLBaVOBxM2;!T*a-xAydgyDJG6VZNrs3liifRR*2>!5vTKrQ zpGIB}P~TaOVZdLrcd}Mb(3)hD*-OJ zH>rL99RwKvdq+r8wM-%EGPBu`X@Rt!)+FJy?sR@Ykjq(E0`sO8kU^C~1gypFQuBeJ zPD-zlrve+CKK@&(!iO;ynG(yBlkdi5Jq{G!Qu)>f@~^1yl96||gvf`Jf{E<1g5h7? zfY_@u5Ze5B;G5q^j~h|^+&tcG=L>aHm*8y+`d|WaZO+k=pKei+LNq4Dqt1(LV65?_ z#;$09eNov|8Bheo_KsoKiS2UXX-c$t^Np96Ni0R4Ja~B^fw!3IbG7T$MWf_+9%B}0 z61b`zXiO~{H8wNkpTx|k?UQzkyb%l9U!C6und>*Uyvw&n%WRUc5+^Z>pkY(cVP+I3NB4k-6<8BJG%**)W>{IdEI7*cc$H&`RseH*j9?cSQ z2aU$ZeBT>;mzbN)B$`q;pqi)ag#2m#*QLy)uxb(m`52@GB3g>JTttU>ZRa{7SO#94 zMq5;Vq^VG2j310R;K@3=MTlo;iDPkQ#ClBtq_d&E#y`t$&i}RSF!5`g+3qogLPMP9 z!=(+rQ3zJL)(Q00I{{=S^K^n@AB(plwFaA;@|W6#Po5-(Mlh+$dnkwYQHck(ihV$q zeUBoM;_u=#E-HFnVyzAu3!_I4Kus}@*5P1Zez2L9UCZ@ZSWAc$y%W*w9TSCZD7UQkywHZ9cOJI8TM7(rH&P~;Y5a#SQYs4GUiR-R4?RU> zFuEq+&98rLP``~4Q4BKY+(JL5++yAe4Jo^8Oach$NlqU>Ncp5vUX}2AW^H>8~ENz2mTHBYD;9z@^5b7FT1Ji~g;QA7&9E0i1 zL%`OMnh6RO-9~P-}{4!y4L4sk&E(^RYOwz&MledwtVL%gtxTWjHLAg5d1oh z0JX7B?}KJUKaB>fRH$s{I3s{STkM%u9#^G9WFsnwwK7b=t`AWYG&$zc~*P-nQ zN2lAMR$OM)QU5GtdA}EPOg<>md!%7gBC~0c+#}e{{mPfb`bqH>1IdkKMea~_C#ej@ zlPNl3tl2mh^PN!IdReo>X;u$aBdy8ChYEt-uBHIz-*#qiJI6z?Fg14CrDWUTp473t zj3=i9Y`^~{8Q@i zi6_}hZckr?1EE;jwKPyb9D_TX+#E>mJogD_r_aL%821syWA2BU8s+u}?-(T`fSr4i z;hxvLxL2x2FJ!JlPp&M@gq%{OzjQOXltDgu7&6Pk%jQ1A84kPPrLT16Tkp3sA>XYq z_PyOS44=4LbZL5G>Nq)am>yZp!bljZf)iLWE$!-kpMN(r{^;dAWXRC7u*sDfXlNJQ zBR84*PODnBrELiszKsw2Oct_0|C<(-nE zSgY{nCD{ByK*^8P`SSI=p5{+H<=J5i*YTpl-2$}gpLq>9u^aX6z280al|^cRAF#am zVtwM(ju-vNC$y2Sk%`mmXH~0lB2gkj0eg}`WTQYc4j|->Dsv#sM2w-BV_2cZ0z>e$ zMsu?!g*)riGOFWRxe-d(_$GJ4fE&Mj;8*0x6dNtZvpwHUXIe61f5)w2U^qgl^eH1p zAeajs-BDF`&urwwj1iQ{)-SCTmkmW8?Qa&~^FZBEEn~`hJJ2s?akvWYXsAfDd@DHr!{SvFRbhI zEv!a>et1&}TCKx2-iVwNd39jG{?MSna1NlW(;AbWqzVh)Z|PNL32N-ZEaE5`*q@D$ zr>SYiTWu8Qt!sDRDKHh{d6y@!lgW3!x*lNp_RYT|>{FfE4-eLLS`oAZHTe2yn)}h! zJn^beq(VyUpbmeoSLs>Pn@Z-u%hU$SxQT%N?Pi@`Py9VS$=(^dc-MPLQwisJt2t*h zNVV)Mu)>(6rglq$b@!L~Il*1upjk~EKxuxq7uBA(N0V(}d|5=d=n^e#ro{qD24?b| zm{dRlY+?da`(SJSj&%1eLfmgN@WVC>cPjRX#;b}3XDOLr;KfZKH^jj}zld+E7F3NJ zLk`9={}H{0NJ%bPL^FJrKla?e{GEmZVLEdVX|W!%WB6Zwlm#L!MX$b}Wi9_1z4-=R zF|Uw8{>cXa3df}5zbDC1>e&D1*UvIPW#CGSS>G3^ocDJK3H3pw2-Jo4T;V&{4F5<@ zQIvoyWy^f{|6W6y5{UMwJ(&G>0%;0dVJgu~o%kP*0WaSJQeBFSbFTk$1%G)BgDca; zi_d>mV*J^n1T~PLHpYlOH|l=9C|LP>b#Mg$-ueA|4bK@Vc$KmeHvhdr*5FFSP_}~A zzt><4jv^i3R_Naw^xs2*h~mG8^j}W;2iN>x$w{Z=J@+YJT3&i}{1^C7;<3!5?1wL2 F{XZb}?@j;! literal 42644 zcmeFZWmuG5_dX0G2#Asb64G4*DAFM*-3^13h;$Ae1|8DUNC=34beDp3O2aTn!_W;w z{4dno`+lD1KHiV-r{5gMz%cvT*WP>Wz1BL{xz^*yya{}B#7TsUf^w_GN>Wl)PEwLu)ye*um5l`o%9D`zL<~)hN#ekP#xBd-wob*Wu^RXY1b+;#pI#qHzxMiZ$9&&xDy1?DWmx3gsmR1 z+kpsz76U%NdCP=H8-9M1v!K9A_;qwc3*Ih1jqv>p{NqG^EWa>PriiW|^iMq8zz+hC z@yvtO`3tWG26me{c%^v~C88s97ZWg_m=5kLY7Uw zO>YIL9?;3_lSgg;5ZETHHX-8;!< zbEv`rVta$A*t$2!pS}@}Zg~*f{NgMeq@Db{QupoKcXF1r)!{0Z*tO5XR<((>t56i- zlaqA2NBzAl=$wU)W)N@a4gHW>6K3%xPtD9 zqdEEC)bzhOh#J^(x1bk?AC)i1)EE!*i}*ub)Ea9191wTs&Dy7Q%qY@Ng(lEh#fkV) z)>}**?p6e7WMUhl=(M=l)!uasuPb$J>VgYNGDp^%X@FJ7qLzy zX;dHH#bywfcmi6INXewlNBiji@xAA&@NK)5ZEJJ{|ZbX6W^hB)HxyY1qossZ`; zG==5E^@CWY;@`he?aRl}1g|r`z&s>4#EOvC7;>p#nZnw9g(K@cw6^NHTC{r0<$fIz z>{U|;rA$^hK$81)fnTig~I9s$lHC|dggd{wcWwq zw;{m;UlV_j?^y2e?WpX=qTPBNHTMLQeh0G*EnBzzc8D`FsnSI79{){EY}v% z=4AR{S9lk7wEID&sRZ?!sR@SbXQiw&=9jy%^bL z6ze5Nqk6!?ek-UfQe93!u2w1OZC|CT^r&N!qf zE=%*1i>8(8+CJzIy%+r)-GKNZxTbXO)t-a}pGA|?d#A8H|2<2hy3joNvTmICNAW=) zpKFN0fqdBThIQEz3PoCj+x2Tq>2K86ZPJ+cZ}T1BIIyal$I z_lwi=U|AQcLHS9<9?V#3|HT{8Jz{lNj7Jr{d@cOmm8Pi>^2^S-})ta9V zDtpT}K6{&4*tA!Etf;e4syZrz*u(5DroOcW+LyzZBRNLhLN=dG+fA3a9l4p-Kda~V zGD7Mi8;`0YCGAv_WP0hJ5$?y=zp06YG)(@uqHw${QS>zU~>uLx)!?zfHs%b)?S)&6(DT$PwQW2@=X4ka)s5S-beyAL7rI$$^Jfb_04-^fnKsJnk98J#vb>hNS28 zZ9MdRIec+sbvzX`2R`q-bR8!y9KJfpxkFzYwv`2+1>;6jMl(c9sx7NoXBSHq%3xldfa|f9+CU|nfDqUQXHp^aGyz04=P@?Z^^=3UZjwny3yJ+!~FmYH&M7VA6#TTr_ z5eLTk2}iGmZ?)`Oc3ar#^AgjJ|WxUGs=s`Ib;*Zn8A_Fsdgt@gQ2;aruKma8x*lb#qP&^W%9z{ysF8xe*R** zbo$LS2DD}?pDh~P#X>q+Yi?=H(pd5N09G7d&lx`%r?ttrm9Rm{dJMLQXV+UI)4@a` zL^tBM;yJuVkL)IqCRFzDhzmPpO5sRh=tt2j=969JN~o)Kgs;*h>PEn)8!9+56dM;I zyBZfXqmt4=O=GvLgVrV1J=Zr82=WoKQ1Tx#atpfO%RV52L~yG4QXc74Ggscdk19(q zn|?)s+@Dix&W3f|>qw45+=qL;^vwE9^uh=xIGZr>XoM0r>o=C@g-guu4be`~?ybB3j=xW_>h1RU%}#y=pw;w;YIwC{Cf|1J9Vk z>v-_E2F31)y009SO-n-Eq3JNCTI3mItg#D8PWhDbu4s-hyI0Op{xV-STt9FlP#M?j zwy_Vz*~uj)bg_M#wAqie&riZv{YZ1)>f%A6W!|@jZ^dH9XWj>IR@)bQUWzSVO(45> zMt8+vmkolHyrO6N=br0*W7Vwo*6_&&L^@`&`+?};#bVHxC{`+jD7WaLZ{+#83l9=| zOW{-}yW~Fnyh-Gpd|EPrV_RyHHp-jZ{Gycpe3d9~vr#i482&XbRe9;$&yW0CZUH>&!F4a7e3aIl#>H~YnVA%SlBr~w|9Aq{yrGE z)wq?Wu8XdcqM(_*EvJdOy{QGKyRE}@7ZhQ4LExjUg^LNbyRD6#v!J^O?awO&fzQ`p zgK4RMUgBacLaVE!N-b&cWI_FilZ%s!R`eD%HMOvl`7=RvDVg8hfxkp(pS!p?2!g>- zD3lZWfYaW|63i_iAOPm#0rT*109SB0d)T>{xO3P!)BU>1U-yx+a5i(Ya&WP-x1+wk zuZgL>tBVLN?e#!^UccsP;coT!NOsP@mjx^keBA=(=HvqZxi`>N`1)HxRV#N38$Bs2 zTR=R(7^1v94~2hT|358%kNA(Cx_|fN=HcT1XV-r;{dZR_XA37uds|>k7tz1B=6C0R zHvaA?48C6aKSc3M&Og5e1TA_?82o3`L~mteeJeyk5l4}edaUV=x&g!T)*L--*^T|Y zZDM>^FPDlOF3v&2yB!OToddjE}E z^c#ir{-5NYTy8h;$0l_t9wjvh?blx@KCrLfp56AD`?1)F%-Z`zdlwB)90m2if7~Vt zepCT_Nd7+!*B2Z3qmht+K$z72DF6Axe+Wch6HX~g7WH2Z*MEw?472{9?@{Z<~Id`2W+U z-=^pPwCR^&`v24D@iVsa`kuRogod&n$US+&j3)f7_nicCt#o&H_sx+HqNw_Os}PAy zu;@uZpMjq4Ej86+%;*fP8C2LFfQ{emvegLIh+?y!?CU&z4?n-~d5<_j=FT+}o*%z& z$*Q=zs9%T_Y5?;m;BiS&v#@k+!=uOyP7KM7k zSG*YgIM{SKcDOq9)xOb_`;=H@)|lL-nV3WWWQ)?lSR~JWsQCoCIki+jWv?E2NyNqN zi%x8qflhqJF_@usc}~Q?du$nie&xgb`5li%=qJP4vZv^9Jgz6ystxx@NObZ^i11a2 zK91Z`+`F292yIL(i?qoI3Qak|C0N&VsaA595^NV7B zWZ+^*<5Yd?O?K%ey>@@MVX5D4DIW#(A}IJWhKYe87xW_g)*o*b;6UiWP^_V$5wV_Z zMyd8tKH?kgzc^Zc8Ni{M1-=aYH^&(#-NNnc5TCnC@#?n|$|Za!?BTSA^VI(Dfr1IR zcp)ym6o%jPR1yykT;voL!}GC&`u`p-PVwotaMGBB1epe19?m0)pQ=iog+>uOM<(iB zL(jydAR{9qbq6I)`VW9$PHG9pr!&7iKaELB(kr_$f&JU!)5`b@)X+Z5Ut|7Hmv}ux zymY{ye{nWkBl?#%ZlLWS{b(H>9SNwYaMAy9sW19N$Vimc8?;0#IvKBj!x#-{A3Zs= z<|S-wKABru>X>5l^Zx4p1B^#RM0A;!6J)|R*ysiA>h8WZeL2MaOHQfOPhT{wzN=;y zG{LjF@w*3SW_617U`$L5!In|3-`{w`0?$Dt{atdheqc7+HXfzgCtTT6x<#hWsqd!~yv#Ucg00;9TFgG9n=l4GwndAMl z2@KWL)N)@43;o86;%jKYp`tr`PyIAY+{LA0HR$DU8MP8#AIK4dyZd{4FZm=!{*qlx zax&|=L>Z+a0?r?{Srh)86oVFSEY5E?P|@TW9o9!!JgzQ|8{L`~V-#|<3bi2!VG`w3ewqi* zdL2qW>Xm;8rw|x!G$b{4+n(icUg>wu5M*SO3BxkWsuj=Q9yiP*dHGOx78TObu z80c6RWB6ED>5b0jJK4MA#YSvNNlC*c#?803>4|9Zut3K!XvS)GXr=Xd`Bbmx-BE_L)=h@i2Fhbr%(y+ zA*Z?>f{8lULV5;_aN<)?HwCCowR}-L^i<_|e@qMn6ZS9xr_VaBa9@_I= z_9A?o3j?ie*yi{|@L4A^JhH zSX&rlB!w+bCCY9V^pj%4y>+&?`$?RDo$I6veYV16B?3}!^gNV+YKrW2(jCz^bv<|OMrBh>~yC6YlDjC(c`7;)c*c{sBBO1HJES?;dEm%G5&|RH-Dj9FiWHR*;~If$W=3Ka87sB9FHdJE;6-2GRp`ij13QeH*x$m^vCY zBFK0*IGNjD646m^?Ns#~a4d+@`E}OunFbY&RY+ zZq;umuxfprh!XMKTrA%KPDbY|Tv2A=OgPyKY&YyL9*!GVUpb0CZ9zFc`x=BB#(2p5 zx3#}S4<-pAxiBXEBuyQTmg^r zJ&a^}T7EBh{w>R&$D|=J&k1hqdLY*qb0_0>t?15?_p~yAMds^w&9Dl!S|Bs z>G1O2qz4_1u*#Bcx=>uP^Uvd=iN?Mj>8`Tx>dDx4bg^(M2iBY{9 z1S^A5#B)&(YCeg@s6#8~ch=up>4Q8HMijG_n2BAUm^HHsRwq~*adUG+kXEqt>7owG z#;ST^&c*}vk_&G_L(5dh!11G8g~U1>*Q=4Xnw8p{xRe{S-$>B#cEC?Xh814+h~2RU z%x=KvYxTY-Ki>|*ZlY0DM~xskG}C8spY1%;p#|pEOO&rwm~eiv?RUKH-{d-Y?D)9k zzOT^}W#uE!yE_NpRT8a>YrnyZod!gXgI%HTyrx@UVOUI8*&gjh)5%Iv86B`|4|h2a zT{PzNv83}mN_@&m@iM;1iV|3#rVtV9OW~_tp?2PUm!7_L)~)i#aiUV8&P(IZOJ{_= zlt%mV#2-!P-W^7x$8(`<#drw($Kc>nvySd!qxy!iQ_d4*R=?#W%d@R{NU=sM^-=Ssp1L0RM;sK#X;c)~ME+|~RJz3u2T-pXd z#NpWl19Apouae*O2s3?3RdtOdqp7cz@E+EQU1d#h&Zq@<)o z2Hcf2cqV|Q|E{4T!f*`tYp!3xCT4f=J3ekdwqu10)pRP0Le7AFz9#KNt{Rs14ozsZjXLnNXI246)`3zFisWJ4P}9CkY z*;r1oPxux=j@;_z`P+q)Ub}GrZ#0as=)gH9gBboJBxi7Gi>bw|8Zud4X!&krQWIjv+`5E*F-(-FU zblHS(4AAvS5Cd}DKKM*`#9M^s1t{+^wGEo*;W`5p?gI@u(*XQY?cta~#u^lFHDU|8vu9@dY- zfER$I@cwvY*-vpSm^KI*$UU^$Q>gYkT}s^A^afzx706R3*OwXd9dl-L4a*wXUU7rU zR)%^6*uwY3E8dvzXaMJO&q4$gcXeBpLFfU74g)Sq9-oXj)%bTE;d=SQ_0grpuGnJf zk_^Ego-6%n7ROsK2;y+_)fr}%<&fv0-KTgq9nWIUz51s2+$<~wgQ6r5_xJ3EHJ=AP z)6Qpku^MJ`bM3`P>c3RDtPM+#6&b9EE{_yw9KrMeC3G$M4!$qs7FXwrAbw#91UWVn zpC2dl*>6>fVFMe>k!jyHWXv4BxaVflbsH4`O=hXjEtatHzyx6JORXo^io_0$kJa+> zxC{8#aX4e=p0j}%F`rAUwUZdLt7-85I9SWekiWXT2!k2Nz7ULVWUM)lFVM)VTUpwh z*^;;8eKGQ6xj&6fUnMiEy3cRFkI!k+BGF^B%zZUBas{26QsIOPbRt!4*)9#TZ>#lI)wS#j~)L{5LE&edA)X``ds%J#6qm`~WO zz=wTTz}6^Up09|7F}^=1N~?fSSv4#r7&L4>`Yuc6`sH-H3E98hw%_DxMZ2*-AXZIi z<;j%1ZN`#J{auy~fSIg~UdKlwsTP1iqHA$M@m1Be2>1UM7A11@j(aorJ?1-F@rLK! zPllAyB#;!wU^nZHapbt;;ol7ql9e!D~6z16?r z_+MBNz|X5_HLuuQ#!|8VlK0;+kT^9aD6;mI^-PT;4}kTkU7}h4#0Y3#9s?{!h&wmM z%^1;NVK*KcY6pGI*e5-@Fk+5d5+nr#e{M-lfu>yo;lL*0yd6|Li1B-R`A@Gqu+|uB z2K_yuKcDbuK#}qe(!dK-2=)s`W;&q(FkedSE}G)q;K)c~z_0(BEw%f#6W4krtA3UY z#7pvic(T8#0PzrXAYzpAdoLAJ*k-=5O8qZUQ^Lr|$XpsdChtDmXX;FKhCe3n&s?s> zMB$7v-Wg4ot5X7bj}Wl@9sh&S-{DY$Fhs)ZKJA)%dwbtI3@-jP^qerN#)34kK$!@# zr3STJ>0f*FI>}!d*SQ*Xwttg&FxnG`H<-!a ze@hb$Kyv&cQBhC{jpx6)2Ve;Q8~;ak{31JoDFPnfz>re5T(i=}3MbE?zkz{syHIq9 z6itGI%k0~2IRyne$ZcNX70DJZ0M>oI8NOTxm=f4si?exv7kKzoPFb1pDa&^*Cp=Iz z&F81BAibSKzTsvsMCU43<8AVQH-*(bD2OwLR-#9e^0KUQ;tTD-b`yfNW5n{LThM@e z63T=;IwPcp9OS9jxdK6IeDUN)1OTJh2b+;=dwJwY86H$0KE=%bwpr^D#?xvFcJgoscB=;UX3kOIO{}9m7 z(9o}_{--XfTu0ezQOwIrOX&8m^Zz15=mA7@@`ixqWdW$Rw)UUb13?xcb~EUKGGs6KaVez^!=W8$t8&>*vfD?%R zNrA~Z6dBcP^E`NvYuxM`?~D4}N#Rw(Nz(;eBgezc=K*o;*qfzf{0>itgvyAn<<~+C zP`tMYFz>N_wu1=*I!z6E=!V9Y_oFQ=exY9v@H~dUP1v`{`)p?^M0fO4)o0U>k~ESf zgw8P=PKTYdP_h3aplBt3orvk?yRPz4AtY%$1eeo~8&=_ODuMZn^WWma(BY3{_4#Eu z|Ih;02AuXfwm`e{$mnn8^p6+%q6W16-4BpNf5FrBhFhq0ZR?l>!J(l9|K$bfYhFT? zRDbVbhqaqA9ifj9(`{!fN&&|OXJ0}+AYhlqQd|1vb*4uM0j!CfZBcXRoC z^X@FAzjjjm9p;X^&b!;@uiYdU+A%2IsU^Hrs()L_2p~?#0t_V(&tmsIxyAuM7pN;v zwavr(JY>!!8_|5rWpj!M`?s0KEb}k;j=B?U`WAfqJ?PduGH}e?E&d#vUvi^9M?0t-_-9HTgauq82&6c?PaFVEPXOv0#lL-$ zrDg^dY8S--!Fo3UPnN1LiP*HC0E9=5Sx01_ z^=`IS(^W+P7R?*1Dy!(x>RRqgvDlq&cSCv|ZLo0YS2)F zqJdaKd)5=d;aBfb^ga9-V1a>9)!<|`J4))+VHmhc7l%?<_j%&#>T~3|eZy{Gh4py9 zH4?eqY>#oupyXiXR1Dpog}AzNraFEj-JA8=?rI6R0W0hoE?!CZQik|lxK5Uu1=XjM z@xJ&;mefW3#ol7RV-#iY12M7n(lI@w=awPH%350n-ct5=HlYJ4t z=Rv*?MjYcCfM{j)WYQb`i0)_Je7>hZ%3~gkJc;r3I2e*$I&QuKNc=`?_30A1T7c0p z1cCvkDPJ8HrTTQ=Gb@k1E~ce^q3t+;J&@t7nFC$Ng}~-?15g+YL|u{Llp<4FykdWW zzJSkwac38dWPseRS4hX9nY$P8(vL$ZQ?8+E2KALVVd&k#wkXRz%c%;hqsx^HzowCo zT0Mw~1P+6`L_mASvwbqHWes%rjK%hl?>}>0@x&2}sAhUCUaC~pC z2VC#wcs1e|zPb?Q*gewG!*vc7qse^3*17w+Z*kd;nb zwJXGRv3**JjME2_e*l}}-D~RQ;ci`SAEs2yH43DYjf{gddP>KL__)4piD6KfiMrC# zE_Wd~0U&3+ir-udsuv~L$6%y#9Ku!%?_I+&_TGIXk{9nQ=6&*6_Ip zbEpzmi{cWhU#h@C#g8EIXh>jtUk1<wsf7m2*ZMz6b}Y_|4gvIlZi2z5r(nO+ zF!*<4fSL9k*E#6Ej-f>!X;#}A0fsALRVp>P#VJ0+_e>5TnHdueokAMrw}pGUx}w>N zYra&5UF`EUpGLsZ5}P7QWuqu#($YA9Jtp$mJp63(b%T$}v!8tV<0tyf;i?xed*Lb6 zMu$6usA#D)sx!hq{m&VS#9NyE8Ecp^H4{*FX?IO6jJ#4w0#vcz3HrnV@_bLp)gJ_S zctu6U_d1}f{1)ZoEN@D%%3iIiX+SZ#125J zYS%R8LdIrZI?pJn0s|Gz5EPkgK0C_nON$o*62z}p#w*DX@8WPBI;|F7>2iX(R{CA} zf*+;n6w*t-y4#4f{_+}&2Vk-!g5oNSfGkSfA!0ntUdWKB>EdYmh{8~iRP--w0Hh)g zKnxK*!6XXyJ@`Nls0qlE?RUJTlzQ6&92tw&m*^0zVo{&73As`rEb)>%;qLKg zz0}?VEY$6HnR@~`gY%8YvuvJhg`%QaQHq|#X&VqAVCS2;!#6Wm8v>ALc%}g(Bbttl zX98_`%Pv175E?vc$Zb@fJaOvY-}=LiRqVL%EkyK|^kbBO77*NfKO(JnSoW?_u5=jj zUWeGF+BKbcZAV1{8v4l}Sq;;VbGBdFjDCz4!AMdjz8aLoC8}?3z7OP`%)8@Q+(34N znUAw#C#`w5b~~sNw_x=>=^|bm9!&>^zddj_Z6;WQ2Eraj(IFKBh@C#-kq>}3RQ?e! z@7a(8(5^*x4zRt6fEH*T9JvhajcBNZh@MsZAl+&KkFz*YYIf~~ChGwg+TlFz54Uf= zaPy+%@%frHI|fPEn}%PG`4TPyzGFiwiLox-@6w|{EjzF`RX}5_W+AdnvxobqP%tMz zg*qiwaIV`JK-ftPNXX_ZfK)K;13vzMiFgKy?}E-`zGQ{Yj^530IGVD~%9!tV9XGNR zYa)5x+}qnLp3*RepnHAxO=7@lz71=#!J~YY?qxdExGzm;e75M?R;#4VG}cs`07*M9 zZ4L(8#FfWp-@@@PHemHS01?XM*`W=bNfGD|p}(;9(>Opdtq%BLV%?oPfa);J`H(+h zs8)hg$o||X%QV-7G_JiF$*Y-Fay;JFC01bgw1-V5UjwtEaI|HtW*cfRp3`;}bq}tj z0*e|vt$z24?K7@HDn}-i3pv}qu|fP@5KRIs-f+*;xj$IE<|iUr$0?rQ;>4;B3%sOO z#Rr|8z%#DSErs6aMiH;4=_@31#sgW_)A=xP4nQRmIg+gdTr?LxV6gF&!ebY2#iB-h zwp;c4X)XhXvGBkN{b7yx!7c!UU-|CQP($|CTAV0` zErrc>Ut9{ec{gp`om*6N_QZD(6^&H^V9d%E1;C(Zf1y6W+?LCvj{T};c$)c(f-t4# zQ!0Lns^p;d7mZQ_`k+kJP8ZtR$eI5a+Wg)2bBffRK(hYjZN3|1HIIVbcErs@Evg=gdV;MJ2JVt)TH<9`AvHUC)7d(kP-$*+fROKb zjXPtxle6>JvH_QCN`^V)I_{H_mIl0~yqudQ6)-Zj4G_40b$h;c4}=+~0R*lb1qB65 zpI9n_V&`rZ@mXSL`S-_WMp3QX&s}47*AL=yMV~&gPu4?A#+%srb*LXa_|6?6mMM&-Q|U^89SS12(r4aPAe2_}iSFGvV9d`o^LO^!@X-dIm5cg{EL?ZYF1dCgemmK6Z*(4`nrc< z^Wy?XsRV}>14o-%bq3W|2-jW~M0A77heoE7ON*D`83!*W7;0p~@6nt&!Wvz7<{l$= zQyLnC!5Dtnz!9m6I;@Ux?I#=*+P^$3@e?R9smRw2fMG;-#%^=QSal(Cpn2NuTLU<) zgoDlGN_X^M=1KH@zy#{I`r|$%)TRMJ{Rg(amc4Ib<3>=U=)>|M0gyqJ4GqMl-y!ZJ z!*vSi)_Om~X|;`>}JZW=MfgXdq~KgM>D_f#FM$K9~iNfyv4kH}-=myR1zU zh+XnuMOFcM-A(}4pSu+U2JPCZoWj1zuVAt3er_p6RoD-7%~9GvYtO2*G0dCr`QjKf z4LLDv(yJ7qsF%qA)IZ?!wG*WPuY;w4SQw9ZfdVTHpbDUIAtZhGh4Ly=(R46n=i>_= z++xt*y4>W_tMpESg~D&wE2q03S(FfK6=|2z)GZZbn6KuOZ<>)me7>mFkx&r<>G{#b z<+S)@XyEb!3F|6;gH1988>=T)?2`#X)+Eg~BRDtM2|I3cHIhHJ7W3PBDWlL1IQeJS zsT5Mx`SwA6XJUJRJ+jAzGt5gH<0mMvsg`ZXSy@~7HG?W&jB!>p$;ZE>$=9!}8!6H# z85H$`ZpUSJ_?C`WKOfc4DwOKkYlq8gAIYt`UZNCe)E8?O=xco(&o|lD7(FYNUYR-b zfK##+B$$B_SDUa(jp0Z3Zv!Ke<-Z0-zz{~p%1g_gfv$xX3sbzqbxJ~OgSmxmC-l9B z$gPR0_7`8Vymnoz;pxH~6hwr~@@T94*W@UwFB zrPP=)C4>Upt|;gG+N_+Dh@uI{N{?&&7t zg+oz(R%*->mFlwXy*eRx3g0U_aw_T1lWUwp_rtAooBAvwNnAWdGkttTdj~Q0c55@a zb8<69p**nCu-^S*q?WAWN?$O2R#l(3n9VrXE}Dq^SU#hlW9j9@n0lcMQHdF?D-G~c zuUBry=K|_PxGTO&HQdv;@7ERt_IJL0*od2`x*Ev9{_xY%OqDBmF}x@VP}d@Q?gcpE z{%IM$^Qd|!&EU;+QRKSNUcWWC^8?lRWKZRb@dS^8ltrqNCTgk%POS9lLn<)cjlXQbw#X{s7-#+wmrN;9}F(YN{c((b(d`7zi1qS3JH) z%~pLHPiYbfdMR#!_1x)s1+U!25aFa7;S0?|T6p7`(AtckSZxJNUexG(2gB(^_tLyO zVtqV@Va#n{@1hRoG>x6nr5z=eWQ{b^IeeREPesjJNTkd~CFUW|c#bge*o@WeI!r25 zo_1aGxWCkgaZ^DNwqCq$?zwjpd2c*4{K`xGh2~)~@@(aq!t#&voa61FR5mc&Cz}1l zBxUAuT%h>tMh1D+{kMhbsTO|5XJ>uGMM{s$MD`P7owBtC#>X6+ay>^`_fGEkEWO9y zRUef<@Jc{lit1?RZCC4vI8URTx^3blm7AA)CBh-YN{nLewO#@-&EcdE7-Q;H;CDy{ zPal0B9u6kC2&dc4qKZ>5-Xj4kr#vc(tNbhkBKlS5;;5ss_JW5Oa&So^v36$o-pjg$ z$7(p|R?!0rHm{}H(yhgWqY@oB_NGux_xJHv7`RWq+>u;Dcw~^s+gUZ1Tl!Fc1#i)L zxou&(Cp+T8>9(78%XFL+pvrm;s*LoWOp_pOf~-4&I^*Q_4KzVG_sgg(ZVy=rD80ir zrx4&2+v;TA%Z4h}%6DkrDdNMzapLIi+R$*?W^>+r9lM;mbv~^F%wX$a$|q62kkJ)} z9Jur?H$0kn*xbbdTZ@=vTZ{=3i%Zowg=*H~j4UZor7@PAGv%r}DI5&>NiG7(E#LFsQJ)9SjWMthd}NHQv8AhPCHv#`ApAiEET_hi(- zj&sCvI7_z=>UJaRH)I(yuJ-B54(dxbVZ;o63>df;Xx<}3)y1luQIsNrad``dChR+{ z=dM{9S>5O`<-`)k2$yZAeT&jPE%`d{J=;sO^04vEIF^i}vTCrdo&(1BqE`b6g-ACS zDrj%r4a87amlNxFw~St{QP@3?qe#@@FUhNyFtAQf(3OED(<^P8DYmQhYGu1x;W3|$ zlyW~jqit5nGncmpC98HYxdgha^A{1l#!cyqz%D7ssI^c$QB0f~?`QJ20w%~MC)OR@aQt?Ca~I{m_hDUFSm+xxi&$LUVfuCF$i>`8iK zfp|w4~lsoCw7wa)^gVV$j!hv(Gz&oMUd zgMQ#FgR!teG4-f%#@$h;;2DFbvi`oA$bKvI$n!!>siyCqSEbi!C}fl2=%^Y%c4ouu zAF4jV_R75s^qNz@Hx;Z}PDa(wU~#4)5CsNE53SRoafB_#c(<&~QS>e!3H@ig+J^eu zce2neG1S7^GtK$MdbVVaveGtOnjK!#;abx+Qb2QK;IK)^ zN~*bhW||kjs=Y2-F6>g+`yj__x=4GtvwYQK`O+dT=L*Rw=Xa&-@uUBxEVIZl_P1lw zvHOtXVfqgV!h&bQKwXW-3PFlP&zmjRv;3tmyh5;^bmNiD_dn46>_?wDd8kkh2@ol! z>9iKou}kHMR~*ihU8sfvQ9t`sP|;6D%Jg(GgrsLhCplXwf~4_kR5O_p!^9YZtuOr1 zem?(%E_BxHw5)yW{#2FN_Y2s*LW-MZ_s6B44Xxs?O_6HEF<>qlw6%)V`+Un63R2Jc z_&t?{w=3zeF<*zY+}BB^127!3hW$-ltdGc&O$P8l_m%VVNAnq_iw!>7x&$LJzU^)o zpLhy{T}f-mgPESQcIa%8kr-9!w#(G&F0mABJAC=@AV_DZwzXS-Pnl@OLwiclw`?|f zl@&JikjPK2Al>p&Il-FRmh6KYFcUYK^0ToN(!$!03wcrEFJ7=VtL|}|3{jv)Fa@)O zQi12&=OZR=1at$Swqn_f0rDwap=4wQNEf!v=c6Tt_`kPvPMboHRn%##CFir6FJ@#C zx+=y$4mECG)ZKTfmG4WgG&FAX)h=>X3NN>z(alBe?eOHO>?j=%c z9AOrCh!}HdE^v%e@pGZ#=qsx2ye_j8rw2-KWcF6SA3I!c+{rQPt|`Cqn#Fd?fS5`* zM9m{AmDgI^qcAvlp->x)6njF%tYaL#tk%CZWU%bL7u~qrap%HG$c z(H>La5iF=f!ADwGCUX&lrn(Rg9*fv5q6dv?n6lUHB}|uQIi>yV-ekJiCqSBLU5IR%96oAjmL?~PPq=e{t` z+VuvU^!m<|F}=4H;tBuCkij~Y8Q{RdEZjNkc1$=>P4Io#iE zoq$cm^U(Ept!Y}OtIZL&=M(M+`Qe4*u2|eiM|<+!`srK)8{A`_Yl(=@<@0hkTs|wR zQCS9~N1d+^ZF`sMFSAc>6t5mj_Q$7y5nGT;lYP;oodJ7T&)HJ9=zUm^XGi?X>y)}5 zm*XpP9Vs6=t+FAB0Oa8mqY^aP_NzNQk{GDQX+pNM>NOWV-C|2CM8K~M=rD>xR!K^x ztr`n<-=@tBzak$$_O;n5jN@=iE57m?_C%VgweNET21R+0uR79UxH?i5hctRY(%70Q z*lO(Lul<<@=y64L zbSMSQv}&lDXo0n%Y}#d7S|nl*3N&#A4`2Sx_Ib!B}-Xz;i*@~tmouZ zw)7R6cXz#_Non`8Ek^>O7a(qhUS%^$3)Dw)Y??&E{XWLC534xl<{j3|eMz*hfKHn$ z5oer?fsScxQ#v|dr+_P%R4-W>jNB|22PzM{J6L%5o6-)%kS<g*!=@ruQBA(@pAcG2lj<#=Dq>V)i$GV@m4 z>BamGVIwfyIb{64d9mAGx%+h$^*q!R7Y#3%AMmNcgW)kg`rj%{H8 zdr@o0x^VBBTbXICblcR|t9Y(v5pFb*;oN%>CD);h?rgb+s1$+g>J>h-^;m+fIqjz% zE3DY9JaLJC+2NJ^(3TTdiz#YbIu@-{mQf`_9uOLD!EIV!p!6 zD_rl@eZK#=5BC0{JoS8U?GVXXI(w-zsyy?2zC-I^s2SdT@@6$VNdLet-%16CNGaHA zsBZRMqr}$bv<}b0I9hb^czq?L2}uh?&WUZ6@m?jIPaK{PuYPYQUkbCZf^-!H$sR?w zHr@jD6x)(@al9IE8_(`gI^C8>M1PZZb3&*W&W3qbINBdyI9Ux;K?)xR#GieqL~xdV zJ@WJsnm6#H*l>+B!ib!>IPIILw`eAbnr3t4K4Zy#U$FFkNmU%WthPv zuO2$Ig*|?mJH>-_fEZWdY(94avWifE?mxGp=+=z>fq{QremTZqbN%?0P|bBU22dIj z6RoyWqiAvG8S{XU>vCdy`qpT{X>1EgfwxHs)Tm=Wfa5L1w%m?StJwOZ zQCHG`KqyW73Ui}2YYDWckEs%n@OR*CQKvVJD#vb08$zvHNTlKERGl^3g2Uso&)SC{ zofdLoiz?$8;65@>G5vfD6^aKf@-k^OQH>XsHd9GI()Sgr(>(MgJ?77voM0{eb*oyC z#$(eVskm91<>a|?Jw;nmfy^*rXD`3nwbOxyqhrH+jwY6;&ko&ApZc~cu&s}MWYp6` zhlWcV=ruK8{UBJI;E|b4FYopVN*>E5gdcBANI+^3vebP*O7_C{b{46jP{ncY~oy*qK>T(yNxU+S?>WA zy&$3=Z18Xfo_aEV zKLtDl#R717jcFU}0Jr&mvrgj5a6Myy$!?X+udcKN?nCREtK8kx<*8U2a`@;=RUi42 zv}6Y4#hZTTN?!zOSNUaL2d}+my{<%KwvE=*fO?N-m}wr0JOOwRb-$$HV~Pr<6#J(l&n638=}JP5F77WZy1E87Fa!CTSRlnHRGUh&K&TrQj{R5C>0R3UE7OE2POmAqOm#mnvizwH)p{ehqdL6yIw?|`XhJa%I#bUDp z8wcdl%>5ohD1fTrv>jRq`{dzc;Q39aBOV>#nbBG^*&aWlL4!mfZyp*%17cEMY#u7f zU%uS=id$ydehWyRSF9k0r4oUHZfaFu)bVozAdSW1XEEY09*my#X`>owh|?C&lNls| zsrKUWd+eE)-f?XYixRu65ius&H7zyY@CRy8fz$<159b~{IaC0G`y8hUAI~@|+fr`s z;SVNK+|_Y%#$E>M@WQ}SVnDe}oOpE>i4N5?Q0gWAbZ@cC?IlnG6@Ufr#UD)KrQoR# z6nI`S^8ea<>!_&0w_R8e6jYE9MnM4uq#2cxMx|T2QMv@B8^LDi6r=DbknNg-*sL^7!txV=-IY%Io>1Kwni~MkYrt?CHQYKY z{J_A#U|^YbO2CpY<*VCb`Q&r!@23u{C^sZUCbvCim}Xx{U!cXw1Z4^XeW72M;4=c5 zTVJlB6N$gc&=TWCLy!Q{>#~e%)a7@S6AY4Y4Gay%^yy`9-b((j;myl{0}w1hDf=FhYO%9(eFY!_=>g=yJ7kZ`*LXiMyth4=*T$>a zfT(GEx#c}r)OrvocxuV9EHR*)=N+fTk57rINmPdmKqR3V`@G;v&@Q)n?6a?{E&w2c zjdB9z$C7osd0~f{A5XW>Qwe|Mwuju{wude*1Lw$PD5=2ULLr&M762JV1<5^xYq``ZFK3cz}9J z45#4G;yj>Ldw)Lz)DQ|xS|}hN&djZ?_fHYnk65GJwE)mfzQ0}TP&`~Gc(ITTct3mV z5ElPu5}}{)7_cPhzc@c~xvi_Uf|=>6H`q)}Oc4C``WaCP2?S(bF?UOZ0nJOu-HMNl zgq0B|njC&X5b!iGyp&Oc8~15=c+`*aN!}y)W@Vp{`3=7|$uxdv;#>TSmyPEQVJIoU z82~t}EI7KqE*yNHXy%DdEYXFYSNGbu^!xxa><>HPG7E<6j??|=e$r)tkH##(6W))B zzfYf#CD8h6d~Qmj7niddph%#ldBS_-3_$Z-2xL69UXpNx!H3SbqoMPr0lGS8Zr$iQ zMhgF#0SNnl6MVGj-47_5fyRRQ1~^M-jy?>HBIDr=txR{)1K*p3l=S4ayd#=SBSRI= z;KcISR3q6j*kF9AV7ey&Rqu{!v}-l zdj;PBM$ia=j%@3uuD<7^pFEgD8aiwj7Z(G-%^;zXA2ss^Fd^QL9>6MJI&*OAiq(LW z9jyVX-?k(}8|X2p*eY+KRNTK*OKJ1ELTekUVuP3{vQ&scmB`v~f=;{d=$ZF$zw+ws zEPCy;0JDi6esZsy1Wv*C7$a@P_yUaC35}m?)OQF}@}U@DraOx|p*#K005g6T`GQ1j zAr#pt;M`{46F!`qJD9vo7{CIwW!)qg-aMV>) zRNiKH5FBis7^Va}=C^!VJ{)4^s*JdaNl9XW0Ccc1@ZvCV>+4b7?;Dz$&JN+-2e|L& z{#9SQ_>F$1&Ohv==JOF@#8pyOexc9!LMg^h^@E7Db>;yE`~s*wX~Yoq6XUT$+%4pN ze0EYSbLawk4}Q+f-`aQrOj_im-_e~vy#&r$6F<@n(S&Od0s?@h)x_2iCE#2-hkJ_f zG9!q|9`H>qPu&^{IE0qML`POrK2 z$^fw)0-jY1-hYmgFVm>-c9Qi;a~~Wr1F`l6ti6u&t&;HqE=uIA8Z=P+a6d^LcarW0 zIMoks>$wQX5Z$f^%igi0cfZJpka)F*2K>a z`mJN{gGSLC8-P--q786el~j?yK&3h>TR>zhKhp0t5fxFg3}LcTQa3YqIlqiUo<~QX zX-XMw65W{OX;PJh* zcyw%Ru*n@C?#-n9@wxh|DLGO6bnPN}AjI(+Mr-k7niU)G_MIvoy%-^nl9h$Qetfwk zf;$*sZu1o#;4*~+6zi#nW}h%5vjj#g0!0DDkf%JflVcdcl2ldKov8WP4p|ABbX<2p zp?|@Mi84z>x-i+zw5jHvI^Sw}l{FGvo@K_b#p@EGX~+&bkd{j!Z-(|ED#r^+4E z0bH?;kbnSkx_saX%a5iRczC*JjVdaWm;37fNYIJYfUKo>vm)j2eG;A8^H;B6b^|*R zV{md&EVZy_*&LbO2SZ$Yd0gd5)yJ&N1ZSf}R8+zaISE9d*p2+I$v+_r$ZT|9Vr+3S zc%}pr9PE~&@bGYYZ=Wb>v+F78A`rDJbR<iK5r4Ufum#L4Vd~=?bR^T zj{in*V3x$-3jtVVD}#-XFB&IO|B~9V#7!{3*k9x|yh14GAnOC6JC^{N^W`%p59lfU zh!|%G*?u0ns zJXbgLg9!`lml;&XJ7C-3=ime-kXprUg%MykNTwX^+ECBfs%3x&SS7glq3Q}Aqzr$J;D2RYo4k~d)a0i4M7q>Sce^>9(z#lFWDfoXgN53lE20;_+^>^lu<^o+G3*iZw*u{+88FLo#1 zWmP`0B{iqS7C#$C_ar%xiCw=!C*ptt%3V^)Ss?;g7nu*dZ2$tr9*cAg`S$JGgi@6n zd_cmv8_Y+9$Ew)j{A8jrJlCj!1YlSdMjCqz!XVoj<}iCHE0Jj|nH$JU0T@>Zhg&|I zA;D#G$p>S;Y-RLJM1-@%j42RX^i0=dpsDf3-#^@Kp;!W(@Mj+hgQ)Fnh9O4a2jDVm z5J<{!hw}i<)cE8wHuCI-8*y+gg0J8^fX2@9=n2f{clu_%6(qt-#CSY^go%hsDPvOi ze-~-{C_j0Q+S=qlD|(U~1Y$T6RrjJztdU$3V3^FnzB{B&DQ?0TTYkeUagXRE%N^`mXrR> zY=^|qW*mFI6UxsbZcv^nQ9kwmC&=>c8uw*!0GT}+%6tqN><@4EdjQ2@vEK&7|Vu-u_Zmlm5Ck)`U{i?vCF#h@VMQwiM%R@r}abW?_ zP&e{uaE=ZG%*RA-;{%!=(z5B>bVni_^OpRDMxqtf4u{|zqz_srvUgHdT0*Ekh(mgaY%H7xVk;n4tKu=?AhK`+p)>8+JBN_rCX_dYn>|Ot;uTg<|VoD zuRqI^$9Nx=M{jm6*>BFo4?AZdGD+A}Q`LgC94w{#tuvDMv-L%*qu?iN{y`C-jdgo|SZv7meiZgKMLl>Ld1r`TG%lV4Pmf5(B* zL)r!pC^1?TcwssaI~WW+Gq1z=eDRYB^3Cq&ffFNbugdwncIxqaV+$43*fkg?^~o5L zsEZrRcOTcn6Qwdm=#3YcrDBVut*Q8RC<G6D??|g zcKnC!^8Rqdt2TjgYo(^udJU~3B6sku)jaIomxOkjQWnNhBig4L$xwBx8b2p^^cfL* za8;8XheD448x!_mtnRfODnYc6ap0XC+n)uv%kQrAI4H*L&8OKiPll!I?vUcM+ptGK zMkSjZ(tgue4{W^(SPnIa;M+4R5IOYFlO*{||Lxom<0ZA_iup^qo3An)(-egBlGSaS zy23rx>!eWVP^GLVS9r_sv2@Q|^WwH9qXqTQ>JBxVP(Q)b6C-=yNILU9gX^-1F)L31 z=WPPNKx0^>)+?6jUUc&@$K;ljy3v>05z-e9;5)&g^HTs0twe@|M;RBxaSZ%7( z9(_24N5x|5wMee_1$N7NZt>9$7Lw0u>56=D3~3KN!P|F_7`d)~r_hO~Pw8WLTi3Lp z-Tj*C#fezqFZ)4mGGm=f3_ea78|FxI-tnBlg1z*OwT^`Qf&fQgH}53uE6*4IE|f)r z@Jim~2%pQ`hw}Mms(I||t!2y)kAh9$%sl~P7eGlufsvh3l!7$e@GIz2x}K}tX<*r~ z@Yo)HHQZRT$m5#P`NB=FtXR>{K$e zkRF#0bkLfKUn3BCdg3JXASv;SGdcpuM2P@b>?!8~G@UUCM5|gLh-lKdH*r}lsQpvS zvK{IgP9L(+(MGq(C`o6l-*aK2KaOI0Z*O|s<<{b^*n;seJaV9C80c43^y@ZtS^Q#X zw&&e8Dzq%vHeGwey1s6~l~!E2{3r1R#0`KeQ5TzsF|)T7X!@As0LVJrbN39%ZFzHb z_(uf(muEnWS-aQ}BakEuiWoC?Ob1X4DXXUNyrw&cz% zu-j#gW8%Q4W4J2|Y-g}Pp0QsX(%o8Lw6R!z)?q!0OvqMZg0>r^lNNoEgYOq16eGW` zc#7??x8CL#5xM9g5pY?6d6C|Obvn&Sx~RfbYW{6>?s7``(r(N|NQH@8!Z(?M3bFo= ziOaTEdcv9bxfddw>OANK*#p8)$;~t>Fiqduf3{Ov0|2)&R>K@Jsyqkkcen4+z~OE6 zPmz~g^AmG%f!>UIuiWGD%{QD=Jd zqbT_8nU=ID^oOjC_goyrPn@kW2WhEZu#RH{g&b#FZ|A)j*lx_YwLg5;xGRxh0e`}w zW8>GOXvx{ujPa1!Z7D>dT;t$7yVLd&+xwpHWcQn_dJ9~#N`=Y=Z+_G~Sv+3r+2N&8 z<|cTs#4u+wvpy(NFV#II$E<6WOc3-puENp1TxQ@A_(1d+es7zw32n$A=B2G(Ub27H zuX5a~T|ra2G&4nJ^hfD<_NvBs@JR{!?h?Tq^I37FZ?1N<2RClz4H9H|{31XugeC!; z8X^?jm@f&t3uXSi96wyRlJ!Y9MIoMjdU`tK?OToesxOTNRnx?^myKThLRW3wbbw?v zAPf1M7;x%kH|$@=iN;g`7$+SYo4<3;+h7Jg)>E%rFej}T#5d;q#deTR`YdV0eH=sJ ziDrV$0*V-ZIwxF@a#>hVxO}GP?!i4_ZtSlY#phrku08HKRS<`C|438H!AW@*a#5WdS3R9wZ^$(JaOsqkMNJPKa0MoFi z0TN+ZgxF&e&P%C9l&3rATms5Nw@{e7#Uc9s2R~%C9E}ui$c1|a<-zAz?|FI!hvQ%9 z%{B^c-d^YFFnxS}`oeHr0{w-P)x4N>0lmCBlb$zgPSgZ4-U&Nf+?PaNxhiB)#8SYGm*+{D} z@_B$%H(4F*-#AJ^;XO}+QV`#Av?mri;aFoxyJWqj61B1~QqkIIgCrAzMh|x817E4? zGq$Y3$dBO70$s{To8VIbGG3aA3(&3~0_h7EhqUA;#v9+Ce062$Z1kT*A`z>%oLbq% zF}R*zK!{_eGkQ|r$H8^xfLUKf4AEt7p7W**E?x}cv~E{S-0@S%;NzMXRY#%cs>rhc zm_iV+bLv^C=#)A8Ip0i&WAkCYg9b<>1n2yQe_zS34dJs1=X!4uw~C@!l=2Rw5@Kcu zh|L$$v>lHmRVvU5-wCYHt#?x|U9tk8GE#|vriFZu5Qke5(Xl8=3eGGzmxao*m+b)` z>Uk`n*y!}Ez9GCyG>uK~3k!_D5WAIVv1);=MbvTW4dg52=GlIj&O#fsL<5h(Gq&9e zBxf!kx{Rnl!jSdF7Si84^V0FhVSA=;cD}I6@oCw}r>k-8tFg5%t?%oy z3%i%gQ*hhR+(ZqW@qbsRv252BUP53YHTafKsYTk|9NTKMtZQw2#lg@*hKuEkrt9z5 zoYq~6Te^cX6h(eacPtMoCRO%O`sD2pZXb0u_ADwR)X(o6&}*{3LQ{!-~_4 z+6s3&I$T@wD#V^_2bLN=Bg;C)GYGJJEQhaws5M|sKaf|NH_iNQ`0psbwIspF= z)Zg3UZT6KoH%vJu&jGbe_iwnBwFqIEG;XyrGhDy^bZ2y37-yFAll?68=r2|6L9_uG zi{6hK^_R7X-V~#r!TsSOkEc?qdIXn+ORZiQPnYwEMXQL3+^NAnLmiQD{rY`FQ8_;| zSt{9k9J&eyVpRsusbsF^23ThX zjfexUKPFkucpU`u7bS_*JMC+0P_fZ6d24bLkE^eC*W|BoW+bW7>~GguxlAGjil_RY z=~i!E$S4mFkIfWzSVey;F~HBF*wQdeMNOpke{yV|H#Yhp)pk|asO8nH9Q}72y-qa2c@L2u^L1? zDg5Qwms@@xdN%PtJ0*amL2j*z`+0r^lnf3vuhKSh-K;)t53+Kl$wE za%vKJ48dKg+3z_cPI0qhR@oc)11_?!TIyHPZm;+S;zWE;aj$(-@)e=e%O4Nl)96Sw zX_38@iXQ!_P#NN+r>kgD_BOGuRzD!3^0n7FH;ng9a)%}zJ&N9=mw8?xHRV>{kiit` zxrohnt5O!>6Q!C>$SMm@<64$uoyO%KB|2w^HJIi4xO{u- zFx3?A8q&9%<|O!;HM6J>#k2~qD0d-6wy zYRgns3-oPl@#VFU75N8s@2?fgZ^y95ZWV^EvT&EU>TYhIr7Q9b;I!^l{54bLI2{>x zRj!}YzPqRRQDC#z?#bR#%gR$+l*az3QH#OiWY2!}jTW;C`SayoZCDK*Hv6}tsQ5(V z$)k^Q*bR&i(4#{BFgAHsD!AU>Nd#;-#GXysCXcPe`Tnc}p#NN_psm=SxhvHjtu4nE zmE8Th2C?mES@uc)Z7l-E!KIfUYJSv>;SC$Fb zijE<-3X=Eh^(f$lkJ;t=ID@?RyCw9to3yP>YPRHNB1qt^kI@Dt*V9<_tiY_sQov_m ziJ+&7zE0WgvPdHffz1jeVp+Y|866}f^*vLD<YI2{9_&q zcaWM^y=QHn-FTF~2Xim4rtO2pJG}eRd0d(iET`9hHN`~!YT`IwMyI}K_Tim(j>SSm z8pexMNq#CgM21R8X4d`UcpC>OYmfxIRJ+HbTecBai-q!cizY%tgx=ov{lPijA zCDps51OeEpmk}^!2a-ez+mD`_eW~|4&B`{abDVREYZ3aMAJL=WywC=f;yfo)qaU{^ z;Bg&yO$0b3;`CM$5wP=t)?R_~u;TjR}oWn^SL{W>FRt7jBT&Xy}OYnV$b zy2vK|%_o7V)vp4@bEu!TkN5AmtWOBUUMopD#h;;@(;l;sk<|UBOtPzvgtEw*#&D&- z80NVgZKYMJX&tvA7rB#bnC|$kK4-GYYhS2|K$m*W{|@_O-%1&gvz^bA@~*6-1y$uH2O%Zp9PMS35AC|0R6Iv>bTY+D0XX*8$h zKxzdTSwJ?@W#HRVvX0uwFe@`lDiqf&+c@tJ)7NDDmU&gr&+!x-srnp& zP*kP3g(X2oVgp-gVwb8}clitV+ki;ssS2Cs9=TA|vbI|JA#$!aM_sS0j@S~_>d5uQ zHMYBpzR1d}3~8b>@E$29F(klBkuDlg)Nrvh)+4oSy%D=N$2jtbjSfRDVi;qGuM$sz ze7dBTGzUOLrDIdK>CcargYV?;9z_-oYDqpbp@0*zQ^D69(W5_vTZ5vD8HnK4Lrh85 zLc-Ve@>5QioS$J`oC?L|=LSZ@m_q~e7+>j$P!M3IO4wApqUYBPRoIW_xtQq84L7w6 zKX8}^g*~xX0j7-(PJzMcbABQ0#Ca_`F`am~G@^WshOYUR+^(oTCKIjDO*VfwrGCUF{ECw^Aq#UQL)(HpYomC z4UpKe4gAWXI){cfhSj@@?u$H%;MKf`Jbzx2RlCgu$}@kc+dhMsbyld4ky3b&HI&*# zHLjZvPn+8c)?YDFNnP(6Tf4c`FHz5{9>dgM*>@E&taF{av=vRxUwI#K>Xt(X+2f0pEKr}B9GFb+hrR5^tSeR4Dq3IFb2{x! z{eE09Pq*xy$UG}pkhw>?1*Oz7JmCK*IZ>v)jnKQoYeiV9d^VJ8>&JraFQ^{sj9gJn$G}dHFh>TKIrI{<)GH;Pa+;6KD3# zzGFk{@G}l}_=rK@Qi!ly?CQ;L_OXpFKed}yo_wR_lWyw7QBFN~sZ+nt$#U$YBb95{ z3G<{*VTJAGcUp}3AIanO-@g!al5PrUNtSvi-tp)=(^85272Q%#HlAcVz}lS~ELRBC zbNH~hDL9VcdN)3XYnd#T^3fLE^DyM2L)nmvc}4U08Vy&I=+@e|ixr=@`>w!b245D) zSTky4oGNU1P-Ns}Ur&+Ea8*(GscVWlG^(a$In)sv-pZ|0G+Z6Zy5_!O6dZkwX)0`j zKM}Aj0hzujSW@SnKqpzm{$o@fpRHwNEutm=*KE5pTG%Vk`KCnZ1I>_#xa)3C^PRyO z#T}D7oyN-G&D!OFS_Is(Y$LrELBUV;0~qPTPYgAPoLxvUMTABc+OQ|XM{;WjWYR^V z10whe^xe$+lM3c~3+u?j+8(`SF*kpKHRKqZ{Hy%)*@@<#-i ztW~-z2Dg*DKDQW;e5#Bz;4@)zwS9NlMejo zZHuy=@`Aapi)|4D;uLTy#H`ieDAGy4%lV0@0&f9WUY4ACIwkT#+J)8hWoz?%QyzPmX(QRv~wsF)znJqZ_Lcr+)JSR}H>LcktrWR5u$;chCy8tRcAPz!`ap zr7mK?mG_q9$+rd6p1}X%j#$&x7IQ2f<}R?$h+$nH>#si2Nd=K84XzF_a=5&U zQR?=!{K;Z5V5-8*(ojzd_|nwAIn)qLUfI31(t``!V>9$931EGFOve)ks^hGR$&wlolWsGy$) z3}p2)GW;T7)thr+M;n|wM+r?A4%vZE)(t?gQ3=%kjm2zku6YR~LF3c{=y z_Cq#mGTotw(5Hh2Hcf&{D802S>O9d{L@3$5lg{$2xL3lj0(Q8f!Lh0`5;I zn?c&r!ugO`ZO5i91@4BA3OuFP5_z?rR!rF{lA0ifblA&ZIk1#O9oXE11D)hGHGJVV zh&VIyDEwhu@&g5rispV?$`7Q5vxf?OD|sdP{3FI0qo9x_`>dJuksN=J^_|c#t2XLU=pg^v3@d_(FI1i&qb~ev%gRl$ zZPbDX8*}vg7b3I{UmWLqXzfziBUISby!xVI@6y~M#h4Y0(Um6Ah$<2vLmC4={D*T+ z6u;3fvp>v~8UdRnuSEdU9xm9wK7+TX63bTl61aO)V+i1hC%s;*OhhNjbE*jFrN8HJ zb-J#{`X=rX=3%jj0az|5fHC6|_tzPJuX0+L`|M@&tUN|>ldtCe83jMggJ|va^;u<1 zyvD`txvcIF%e4$-O2{;P4yEmN$hG&Kj1I}fPxo*mZ2Za-*?6`xK6Be%Tdjn0qD#e+ z-EVIQHzXr(C>xU&p{o$-G5IF7L-mu8_|q4#=i@yj*mu?*n8Qdxq6b7lA?Jtn*m<>X zRNQac+EP~Fiy7K7D|`EnsTUmfi+414YHzvxbjp%q^M`S8>J20-;PzPcCsld@fDp3O zmj(Kc_lJQ*_j7zwqxM9xrAC&l=ZW7G)M(Y4wb_bS?LH{C6>4!L|JG$|@A|C3AB#3A zSV1Mm&&%5+B^4?qzw{(c*{9C?(cBP*1zV30nskaJ;JYVZ?S0SsWekWObg@}~;rvMy zW08IV%2xwn6E%pv0L`0D;o~#s`mV_M!rY1+I4ssXqDcEoo$pI*rhJ&0V%z7k$}q{0 zo%WKlnfCBTBJ|D5oHZzltVc4hY=&^Hu@)ZYj?O{83xw?eq(l6d(oeM>aure(SdGY7 zi$$w*FlgUeeaGci^Zny0ZxqDN9)A93ZGRoSJb2}?#dTORi|f# z)_qs~wZf?2edD6s#iHWk{$t!-6^tz%8@czKMv-F%PY!D12{_Ul3)dCrWwvTL&D0bM zt$I@P`f8Ns`t$SyF0au&t2=A>q~|zYOZP_Po#fK{L`f zahTl+a13ARJ<@64=z21WXCb~rN`9hyR{{4Z&_1AHCCv^ zFUjoPy2~dy|B`nJHST1*r3_r2OjCytnF{{qmKY8D+{6z)KTR-`7$W}HArNw^LV(92 zZyHw`2Soe>B#HM)HIR5R z0J#v*?Ir?rC~5?7)qC4l6Mi0C9!K{n_TZhxt9>wq2SCFv7eKv7k4VgfYSk$6G`0JU`!<2whibmwXYg=L0Qq3yeV>AE@kXbG&An?} z54uh}Usv+7JwSm#a#v*<#qrpq$i~%i3uq-@!vlcB=XyK4=i^eJr#;3$oJMdoijvXj z$^M9&sK7j<&oz*2#1?!48ZN1iSjit06h_9+0wlBq5QL|L!xzUZc^6-ckenfcUw@cl z*92X?9vuDokhI|Y5>gT!vN?92h%?x>e1`)}6Ep%!*!V-$15}t@MT3KV08zVe;Q|i! z3-N=Y4Elu4?TWzl&c@VDmzI~m^||)}O>5}-j5uE@;HR!vJAke$&l8i(=?<>^ko>nm zn_%L_Uv3HW4J3#A{t{zVBW4P#IXX2%5Xs0if{HDD?i>L zKmz6F=Y`03a;#A8pdM6{Uc*S!w2LV*F62{uXdMF`H}Y%&(98&HRxSFdZ65IUE`Z$$ zRKcnfIA1v)-PKcQ;LZf$%N-aT4AkcM1uqynza7Zs4(9Cm_n7Vanha0HhlQS=zVLM9~4WER~IhN@|zOQfIGb9yLYF1+v*c&(aIw zJtRADc#n5k>{BG-q0Yt`R=;xbcYlWIaO;5kNyPTPRK5E3EL@Zld8Ek<-JS1Qh}47R z$vG-w;@!p)e~i9^x%2<_f>!{ZQqk$(t%LEvVA^Bgdtlk9l#9d*d8F>iIUhAa0E_+m zq3y8j%{9&iT921F%@oaqaX1gxOu|_KbPeepn``s;5>&h=nj)KkMk!FzBr^c&jnFKR z!7zTL#04)jLcbpH6_|%eb&hlBA~aVaADGwH=uPCt4 zDF!KCIre9-O~=KyRf;=Q*GyNNUHrt}c0ZpG@obCTo7S`pzcH%BPu1eF|=?si8UV(^Rkb z1Z#)mB`kM)D(T;>EPnIBJqxPoH->G6(+sgXx-)8z=%w^w=Gk9+&`~v)X$1d3}KkpKs=3?6@ z5@=A@>`;=g5P17PhL=oT16_Cg(TJV*%q0L6P6gV0Jx_d>n3b28my|Ux27s^9f`xML zxGM}FO=OlXg!uUQyq(|OUJvYb9CYYz0BWO@d7$^n!p;6Km3K+Ncl&_veo;Q!rvF|- zb$>V4W?0B&LoN&co7;Kch1>T$E^=FG{A@v=@g_Q}q9$Oh|LKA*^X%|g zUMpYPl=0v!==g@B2Yrq{H97}xv;1Ih8=hRze@_|Tb{DarEl>@Ir)Y|F|N~S5VXYUu-}Z?ct~ddzx&F5kw{VmM zbW<1ST|iH^|G77Dn6HD(GTHK&)j!_#wjigeR&BXb)&DOCP(>E(CFIU+%A>FR>*@*7 zKqE}m?)2}qe{F0su?M453e=qXqGx=3^?{bVsLK&{bE}SZVC%m>U8(ynFhM=&gw_r zdqoInEWZ3jpqa4kYL|M`*6M7N0~_J-%1hOH>KyExlPYZZutEsXNY|+zffDp{ShkT2 zXbtuOG(%&s&w_a6GH(GTZ2ozxuqK1UbVu@CsV}6DCc)8;6#uk4B_ZsqH`-^J6%M*q z06N_O$zbiVgHH1a1aGSS9N=YEJ4#(4tr_uz4cctpjbvXyr($N#kot;=v)`1i%V`(|L@c5l^25X~kfNvbA&z_RkJ3d~pK#z;`62hx`>3sdEUvANU zNxd6x2>`(fxap`4z-prlQOO||0FC1^0bMP##t|mG2O0&2bfu}+UAC<2O>zX?Sxurs zl_2qCR{aVAt^Lm%V)O01mT3h!ACNOQK%YT{^p90Q1l{9R&zMv2^4j9?Vy0o8x_ypi zOB9EMTDr!4i&EOfhqW>hti&=GqBPO#a7M<*^SNX5Pozt`foOr4eBfSyQ+)d~-?D7a z>^lB=*JQRzAUQn9?k1YEf$4xEFCPH4SwK1+hc8kziUNDIdaU{>KuNL76Z_tQDgWh%#^gdjNdI`X3X}@Lk7g7njIxByaMZ37K^_p7mXA}+1=-Rb#wHsEh z6=aD8ZiDtB^ISk4zoaeGVdis+tJV!6SJ`$L)NP_1bas&l?5-mcl*2A|u)Jp!CE$|V z?+8?F#JP)yx9Z>XxUQJNd3bnS{hOO*7s1k8=Vba70C88h_xI_iLsSliC)`->AUBoNaam&%_G(IJtmBeja^QjI`{h#yu`Lk zRlRuD%qMfoMDNfti;#e=6_FMC1~isvT7UC7I%&S&2~+m@jB^c326)jr*uQ}LyksvL zV%h+8%yf%j>;Q!1P)VOA67h%|yxVhm<(&$`gY z>pT$sa9ZpKO+}035Y?sYZlIsF;Fa8kLUUavg|l655zfquB={PvIms`2%BgooP7i^0 z0fJYsCKlQY$>h#x-C_!v%UkDC-!7iZZPHO{Ai2+H_uld);kM&^?jtaswpl>aMzts_ z9gLj>i4MvMC0w~XZ&MD0@hA1@8q_&2nb+-)QSU!yKQFYAkP-1kqf(>P{w0!RJrBcn z0H-C1IzT$Rd&odNj4KpmyDrQ5q=BG~8IZR$Xh^a(mJ9%8xFUjGyRbP8G$=?Bs}=Mx zYMlq9$2n)22V}Yxpp#AHcuq|>=+WuBumt|;_|%YSZoQ7}F9IP`NsFpOu;8c3bZ{9Q zEuSWFIWh5o>2&`E!8*T7q|0hN1CO8euY(8cDQe3ce1GrsaN*K`DP6Vk%V5VII`+Rl z1T754FMPi*zEr;G(dM+w{Vx8x)HrDM0F%?$T$?syEuQ}P&b>d*ZGji+D0GfmNQd*z zSbsO@#N9p~!Kzo*v)j`O_?3vRu8#YP1{_18PBt#?_s7(0AlUnIO=C=())sYE>u2qVd;B(wAI_@SWZI!)hBEYJaz$frC# z23qJN5-C@ZpcBhrD!r98=iTH-Rmj0^^nA&N$K)xw6) zbj(iOZkHg{wY!E~l(Fo8Gn5iQhbaEA`ktPJtsyxup=9?RkBws`tyKEkQ2S>nn4Od) zf7_W6ZIY0qFfasC+I8QRJ6z8bRA4>Z3KmlU@|tKW2 zl;uYFIU(fl{1vE;?Mtm3g|%5-l-zUfYi=>>qFX8)EJItQlU7sR`xR=X>3CIk%CD`1 zc3TlsnGo-D1>H8q`qI%R0jNgZ{EATEB+R80%u31gE_6VVTVA_6H*GR` zePB%M5fVs}1jN5f4xCVnnN6Xa;clGMhq+dwYu@#Yaiemm=(EmGw8hd}IT9p~awL9P^@2v}+qx@WR$)Pc3m0v2+V68WrcI4GiUk%~s&px7^ zD14;7j)IhT20YdWm9q2%+cjZgmchk1!Sp8z*wv$=)w|Zs->~K7j;or>+_~f3V_Ge~ z*#*wd{BJ!+*))7sK)$bNrPpJPhe@a(P$8hCFz6TQ_M?sJ1xtm--CA#!dg_UmUh!v4}#R7J8gi2>GQS#L5&-G6r%`6$# zY1sg40=EGzFEDwtle?b>LBxf!n+327uDY{?1v~e zcYj< z_V<=^<{b2WRzYeuEV+;@9qUg>uBZ{T2%_O`Z1HW`pSSTs4C$jKSTkbP;^%zMvOf-h zeCx1DA=*V<)G*ZgX=k&|%Mq9geufYLzB>Zr(C|W?ce@vXVP77dk|zIt|0I-{7E!{V zl4NkxYwNtUFCcfyg)gx&!;++t5-_ieoY4mHugF83w%dfZt%lbUS%6(y=Chv~3COeT z4{Wuc+68?g*jzPM3WQ3grfGxP#~iqYZ59TK<4)nmwW3G`Sk`8!uA=CBy?evFGE+AX8d= zfmQDJ+(HbG zh+-x>t@MnAXi$U}3Q|I8A5?3hu@})pn$!}0sAXR#or~*TpOq!jQQc5$K=EoZ--iCV zFtzyE0R9D{5bcL^nteyb4l=;bW&HtM`Fy-@EaHz}ypJ>k98i^1ky~E0DXZl=H+OS= zJVAqV|5kSml-*bRfnZvR51?yt;_7$N*~H?HmKisATxWrsOf2&|=P>ROSpvO_*qQ7= zZ~%7x<~Jqe>03T&zTU+G7{0xUQxvdPWcvB@?{OmW z;AV&nk(({K&A#IsyJkiOGv*-hJTn{{Y&K B{V@Ol