Task_Runner_E2E #104
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| # Task Runner E2E tests for bare metal approach | |
| name: Task_Runner_E2E # Please do not modify the name as it is used in the composite action | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" # Run every day at midnight | |
| workflow_call: | |
| workflow_dispatch: | |
| inputs: | |
| num_rounds: | |
| description: "Number of rounds to train" | |
| required: false | |
| default: "5" | |
| type: string | |
| num_collaborators: | |
| description: "Number of collaborators" | |
| required: false | |
| default: "2" | |
| type: string | |
| model_name: | |
| description: "Model name" | |
| required: false | |
| default: "all" | |
| type: choice | |
| options: | |
| - all | |
| - torch/mnist | |
| - keras/mnist | |
| - keras/torch/mnist | |
| - keras/jax/mnist | |
| python_version: | |
| description: "Python version" | |
| required: false | |
| default: "all" | |
| type: choice | |
| options: | |
| - all | |
| - "3.10" | |
| - "3.11" | |
| - "3.12" | |
| jobs_to_run: | |
| description: "Jobs to run" | |
| type: choice | |
| default: "all" | |
| options: | |
| - all | |
| - test_with_tls | |
| - test_without_tls | |
| - test_without_client_auth | |
| - test_memory_logs | |
| required: false | |
| permissions: | |
| contents: read | |
| # Environment variables common for all the jobs | |
| # DO NOT use double quotes for the values of the environment variables | |
| env: | |
| NUM_ROUNDS: ${{ inputs.num_rounds || 5 }} | |
| NUM_COLLABORATORS: ${{ inputs.num_collaborators || 2 }} | |
| MODEL_NAME: ${{ inputs.model_name || 'all' }} | |
| PYTHON_VERSION: ${{ inputs.python_version || 'all' }} | |
| JOBS_TO_RUN: ${{ inputs.jobs_to_run || 'all' }} | |
| jobs: | |
| input_selection: | |
| if: | | |
| (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') || | |
| (github.event_name == 'workflow_dispatch') || | |
| (github.event.pull_request.draft == false) | |
| name: Input value selection | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| # Output all the variables related to models and python versions to be used in the matrix strategy | |
| # for different jobs, however their usage depends on the selected job. | |
| selected_jobs: ${{ steps.input_selection.outputs.jobs_to_run }} | |
| selected_models_for_tls: ${{ steps.input_selection.outputs.models_for_tls }} | |
| selected_python_for_tls: ${{ steps.input_selection.outputs.python_for_tls }} | |
| selected_models_for_non_tls: ${{ steps.input_selection.outputs.models_for_non_tls }} | |
| selected_models_for_no_client_auth: ${{ steps.input_selection.outputs.models_for_no_client_auth }} | |
| selected_models_for_memory_logs: ${{ steps.input_selection.outputs.models_for_memory_logs }} | |
| selected_python_for_non_tls: ${{ steps.input_selection.outputs.python_for_non_tls }} | |
| selected_python_for_no_client_auth: ${{ steps.input_selection.outputs.python_for_no_client_auth }} | |
| selected_python_for_memory_logs: ${{ steps.input_selection.outputs.python_for_memory_logs }} | |
| steps: | |
| - name: Job to select input values | |
| id: input_selection | |
| run: | | |
| # --------------------------------------------------------------- | |
| # Models like XGBoost (xgb_higgs) and torch/histology require runners with higher memory and CPU to run. | |
| # Thus these models are excluded from the matrix for now. | |
| # Default combination if no input is provided (i.e. 'all' is selected). | |
| # * TLS - models [keras/torch/mnist, keras/jax/mnist] and python versions [3.10, 3.11] | |
| # * Non-TLS - models [torch/mnist] and python version [3.12] | |
| # * No client auth - models [keras/mnist] and python version [3.10] | |
| # * Memory logs - models [torch/mnist] and python version [3.11] | |
| # --------------------------------------------------------------- | |
| echo "jobs_to_run=${{ env.JOBS_TO_RUN }}" >> "$GITHUB_OUTPUT" | |
| if [ "${{ env.MODEL_NAME }}" == "all" ]; then | |
| echo "models_for_tls=[\"keras/torch/mnist\", \"keras/jax/mnist\"]" >> "$GITHUB_OUTPUT" | |
| echo "models_for_non_tls=[\"torch/mnist\"]" >> "$GITHUB_OUTPUT" | |
| echo "models_for_no_client_auth=[\"keras/mnist\"]" >> "$GITHUB_OUTPUT" | |
| echo "models_for_memory_logs=[\"torch/mnist\"]" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "models_for_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" | |
| echo "models_for_non_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" | |
| echo "models_for_no_client_auth=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" | |
| echo "models_for_memory_logs=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT" | |
| fi | |
| if [ "${{ env.PYTHON_VERSION }}" == "all" ]; then | |
| echo "python_for_tls=[\"3.10\", \"3.11\", \"3.12\"]" >> "$GITHUB_OUTPUT" | |
| echo "python_for_non_tls=[\"3.12\"]" >> "$GITHUB_OUTPUT" | |
| echo "python_for_no_client_auth=[\"3.10\"]" >> "$GITHUB_OUTPUT" | |
| echo "python_for_memory_logs=[\"3.11\"]" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "python_for_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" | |
| echo "python_for_non_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" | |
| echo "python_for_no_client_auth=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" | |
| echo "python_for_memory_logs=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT" | |
| fi | |
| test_with_tls: | |
| name: With TLS | |
| needs: input_selection | |
| if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_with_tls' | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_tls) }} | |
| python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_tls) }} | |
| exclude: # Keras does not support Python 3.12 | |
| - model_name: "keras/mnist" | |
| python_version: "3.12" | |
| - model_name: "keras/jax/mnist" | |
| python_version: "3.12" | |
| - model_name: "keras/torch/mnist" | |
| python_version: "3.12" | |
| fail-fast: false # do not immediately fail if one of the combinations fail | |
| env: | |
| MODEL_NAME: ${{ matrix.model_name }} | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| steps: | |
| - name: Checkout OpenFL repository | |
| id: checkout_openfl | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 # needed for detecting changes | |
| submodules: "true" | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Pre test run | |
| uses: ./.github/actions/tr_pre_test_run | |
| if: ${{ always() }} | |
| - name: Run Task Runner E2E tests with TLS | |
| id: run_tests | |
| run: | | |
| python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ | |
| -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ | |
| --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} | |
| echo "Task runner end to end test run completed" | |
| - name: Post test run | |
| uses: ./.github/actions/tr_post_test_run | |
| if: ${{ always() }} | |
| with: | |
| test_type: "With_TLS" | |
| test_without_tls: | |
| name: Without TLS | |
| needs: input_selection | |
| if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_without_tls' | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_non_tls) }} | |
| python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_non_tls) }} | |
| exclude: # Keras does not support Python 3.12 | |
| - model_name: "keras/mnist" | |
| python_version: "3.12" | |
| - model_name: "keras/jax/mnist" | |
| python_version: "3.12" | |
| - model_name: "keras/torch/mnist" | |
| python_version: "3.12" | |
| fail-fast: false # do not immediately fail if one of the combinations fail | |
| env: | |
| MODEL_NAME: ${{ matrix.model_name }} | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| steps: | |
| - name: Checkout OpenFL repository | |
| id: checkout_openfl | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 # needed for detecting changes | |
| submodules: "true" | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Pre test run | |
| uses: ./.github/actions/tr_pre_test_run | |
| if: ${{ always() }} | |
| - name: Run Task Runner E2E tests without TLS | |
| id: run_tests | |
| run: | | |
| python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ | |
| -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ | |
| --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls | |
| echo "Task runner end to end test run completed" | |
| - name: Post test run | |
| uses: ./.github/actions/tr_post_test_run | |
| if: ${{ always() }} | |
| with: | |
| test_type: "Without_TLS" | |
| test_without_client_auth: | |
| name: Without Client Auth | |
| needs: input_selection | |
| if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_without_client_auth' | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_no_client_auth) }} | |
| python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_no_client_auth) }} | |
| exclude: # Keras does not support Python 3.12 | |
| - model_name: "keras/mnist" | |
| python_version: "3.12" | |
| - model_name: "keras/jax/mnist" | |
| python_version: "3.12" | |
| - model_name: "keras/torch/mnist" | |
| python_version: "3.12" | |
| fail-fast: false # do not immediately fail if one of the combinations fail | |
| env: | |
| MODEL_NAME: ${{ matrix.model_name }} | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| steps: | |
| - name: Checkout OpenFL repository | |
| id: checkout_openfl | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 # needed for detecting changes | |
| submodules: "true" | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Pre test run | |
| uses: ./.github/actions/tr_pre_test_run | |
| if: ${{ always() }} | |
| - name: Run Task Runner E2E tests without TLS | |
| id: run_tests | |
| run: | | |
| python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ | |
| -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ | |
| --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth | |
| echo "Task runner end to end test run completed" | |
| - name: Post test run | |
| uses: ./.github/actions/tr_post_test_run | |
| if: ${{ always() }} | |
| with: | |
| test_type: 'Without_Client_Auth' | |
| test_memory_logs: | |
| name: With Memory Logs | |
| needs: input_selection | |
| if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_memory_logs' | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_memory_logs) }} | |
| python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_memory_logs) }} | |
| exclude: # Keras does not support Python 3.12 | |
| - model_name: "keras/mnist" | |
| python_version: "3.12" | |
| - model_name: "keras/jax/mnist" | |
| python_version: "3.12" | |
| - model_name: "keras/torch/mnist" | |
| python_version: "3.12" | |
| fail-fast: false # do not immediately fail if one of the combinations fail | |
| env: | |
| MODEL_NAME: ${{ matrix.model_name }} | |
| PYTHON_VERSION: ${{ matrix.python_version }} | |
| steps: | |
| - name: Checkout OpenFL repository | |
| id: checkout_openfl | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 # needed for detecting changes | |
| submodules: "true" | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Pre test run | |
| uses: ./.github/actions/tr_pre_test_run | |
| if: ${{ always() }} | |
| - name: Run Task Runner E2E tests with TLS and memory logs | |
| id: run_tests | |
| run: | | |
| python -m pytest -s tests/end_to_end/test_suites/memory_logs_tests.py \ | |
| -k test_log_memory_usage_basic --model_name ${{ env.MODEL_NAME }} \ | |
| --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} \ | |
| --log_memory_usage | |
| echo "Task runner memory logs test run completed" | |
| - name: Post test run | |
| uses: ./.github/actions/tr_post_test_run | |
| if: ${{ always() }} | |
| with: | |
| test_type: "With_Memory_Logs" | |
| test_straggler_check: | |
| name: With TLS (torch/mnist_straggler_check, 3.10) | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| if: | | |
| (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') || | |
| (github.event_name == 'workflow_dispatch') || | |
| (github.event.pull_request.draft == false) | |
| env: | |
| MODEL_NAME: "torch/mnist_straggler_check" | |
| PYTHON_VERSION: "3.10" | |
| steps: | |
| - name: Checkout OpenFL repository | |
| id: checkout_openfl | |
| uses: actions/checkout@v4 | |
| - name: Pre test run | |
| uses: ./.github/actions/tr_pre_test_run | |
| if: ${{ always() }} | |
| - name: Run Straggler Handling Interface Test | |
| id: run_tests | |
| run: | | |
| python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ | |
| -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ | |
| --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} | |
| echo "Straggler handling test run completed" | |
| - name: Post test run | |
| uses: ./.github/actions/tr_post_test_run | |
| if: ${{ always() }} | |
| with: | |
| test_type: "With_TLS" | |
| test_eden_compression: | |
| name: With TLS (torch/mnist_eden_compression, 3.10) | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 30 | |
| if: | | |
| (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') || | |
| (github.event_name == 'workflow_dispatch') || | |
| (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'eden_compression')) | |
| env: | |
| MODEL_NAME: "torch/mnist_eden_compression" | |
| PYTHON_VERSION: "3.10" | |
| steps: | |
| - name: Checkout OpenFL repository | |
| id: checkout_openfl | |
| uses: actions/checkout@v4 | |
| - name: Pre test run | |
| uses: ./.github/actions/tr_pre_test_run | |
| if: ${{ always() }} | |
| - name: Run Eden Compression Test | |
| id: run_tests | |
| run: | | |
| python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \ | |
| -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \ | |
| --num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} | |
| echo "Eden compression test run completed" | |
| - name: Post test run | |
| uses: ./.github/actions/tr_post_test_run | |
| if: ${{ always() }} | |
| with: | |
| test_type: "With_TLS" |