[FLINK-36192][autocaler] Optimize the logic when partitions or key groups cannot be evenly distributed to subtasks #4012
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ################################################################################ | |
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| ################################################################################ | |
| # We need to specify repo related information here since Apache INFRA doesn't differentiate | |
| # between several workflows with the same names while preparing a report for GHA usage | |
| # https://infra-reports.apache.org/#ghactions | |
| name: Flink Kubernetes Operator CI | |
| on: | |
| push: | |
| pull_request: | |
| jobs: | |
| test_ci: | |
| runs-on: ubuntu-latest | |
| name: test_ci | |
| strategy: | |
| matrix: | |
| java-version: [ 11, 17, 21 ] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java-version }} | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| distribution: 'adopt' | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven- | |
| - name: Build with Maven | |
| run: | | |
| mvn -B clean install javadoc:javadoc -Pgenerate-docs | |
| if [[ $(git diff HEAD | wc -l) -gt 0 ]]; then | |
| echo "Please generate the java doc via 'mvn clean install -DskipTests -Pgenerate-docs' again" | |
| exit 1 | |
| fi | |
| - name: Validate helm chart linting | |
| run: | | |
| helm lint helm/flink-kubernetes-operator | |
| - name: Tests in flink-kubernetes-operator | |
| run: | | |
| cd flink-kubernetes-operator | |
| mvn -B verify -Dit.skip=false | |
| cd .. | |
| - name: Tests in flink-kubernetes-webhook | |
| run: | | |
| cd flink-kubernetes-webhook | |
| mvn -B verify -Dit.skip=false | |
| cd .. | |
| - name: Tests in flink-autoscaler-plugin-jdbc | |
| run: | | |
| cd flink-autoscaler-plugin-jdbc | |
| mvn -B verify -Dit.skip=false | |
| cd .. | |
| e2e_ci: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| version: ["v1_20","v1_19","v1_18","v1_17","v1_16"] | |
| namespace: ["default","flink"] | |
| mode: ["native", "standalone"] | |
| java-version: [ 11, 17, 21 ] | |
| test: | |
| - test_application_kubernetes_ha.sh | |
| - test_application_operations.sh | |
| - test_dynamic_config.sh | |
| - test_sessionjob_kubernetes_ha.sh | |
| - test_sessionjob_operations.sh | |
| - test_multi_sessionjob.sh | |
| - test_autoscaler.sh | |
| - test_flink_operator_ha.sh | |
| - test_snapshot.sh | |
| include: | |
| - namespace: flink | |
| extraArgs: '--create-namespace --set "watchNamespaces={default,flink}"' | |
| - version: v1_20 | |
| image: flink:1.20 | |
| - version: v1_19 | |
| image: flink:1.19 | |
| - version: v1_18 | |
| image: flink:1.18 | |
| - version: v1_17 | |
| image: flink:1.17 | |
| - version: v1_16 | |
| image: flink:1.16 | |
| exclude: | |
| - namespace: default | |
| test: test_multi_sessionjob.sh | |
| - namespace: default | |
| test: test_autoscaler.sh | |
| - namespace: default | |
| test: test_dynamic_config.sh | |
| - mode: standalone | |
| test: test_autoscaler.sh | |
| - mode: standalone | |
| test: test_dynamic_config.sh | |
| - mode: standalone | |
| test: test_snapshot.sh | |
| - version: v1_16 | |
| test: test_autoscaler.sh | |
| - version: v1_16 | |
| test: test_dynamic_config.sh | |
| - version: v1_16 | |
| test: test_flink_operator_ha.sh | |
| - version: v1_16 | |
| test: test_snapshot.sh | |
| - version: v1_17 | |
| test: test_dynamic_config.sh | |
| - version: v1_17 | |
| test: test_flink_operator_ha.sh | |
| - version: v1_17 | |
| test: test_snapshot.sh | |
| - version: v1_18 | |
| test: test_dynamic_config.sh | |
| - version: v1_18 | |
| test: test_flink_operator_ha.sh | |
| - version: v1_18 | |
| test: test_snapshot.sh | |
| - version: v1_19 | |
| test: test_snapshot.sh | |
| - version: v1_16 | |
| java-version: 17 | |
| - version: v1_17 | |
| java-version: 17 | |
| - version: v1_16 | |
| java-version: 21 | |
| - version: v1_17 | |
| java-version: 21 | |
| - version: v1_18 | |
| java-version: 21 | |
| name: e2e_ci | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK ${{ matrix.java-version }} | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| distribution: 'adopt' | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.m2/repository | |
| key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-maven- | |
| - name: Start minikube | |
| run: | | |
| source e2e-tests/utils.sh | |
| start_minikube | |
| - name: Install cert-manager | |
| run: | | |
| kubectl get pods -A | |
| kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.8.2/cert-manager.yaml | |
| kubectl -n cert-manager wait --all=true --for=condition=Available --timeout=300s deploy | |
| - name: Build image | |
| run: | | |
| export SHELL=/bin/bash | |
| export DOCKER_BUILDKIT=1 | |
| eval $(minikube -p minikube docker-env) | |
| docker build --progress=plain --no-cache -f ./Dockerfile -t flink-kubernetes-operator:ci-latest --progress plain . | |
| docker images | |
| - name: Start the operator | |
| run: | | |
| if [[ "${{ matrix.test }}" == "test_flink_operator_ha.sh" ]]; then | |
| sed -i "s/# kubernetes.operator.leader-election.enabled: false/kubernetes.operator.leader-election.enabled: true/" helm/flink-kubernetes-operator/conf/flink-conf.yaml | |
| sed -i "s/# kubernetes.operator.leader-election.lease-name: flink-operator-lease/kubernetes.operator.leader-election.lease-name: flink-operator-lease/" helm/flink-kubernetes-operator/conf/flink-conf.yaml | |
| sed -i "s/replicas: 1/replicas: 2/" helm/flink-kubernetes-operator/values.yaml | |
| fi | |
| helm --debug install flink-kubernetes-operator -n ${{ matrix.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest ${{ matrix.extraArgs }} | |
| kubectl wait --for=condition=Available --timeout=120s -n ${{ matrix.namespace }} deploy/flink-kubernetes-operator | |
| kubectl get pods -n ${{ matrix.namespace }} | |
| - name: Run Flink e2e tests | |
| run: | | |
| sed -i "s/image: flink:.*/image: ${{ matrix.image }}/" e2e-tests/data/*.yaml | |
| sed -i "s/flinkVersion: .*/flinkVersion: ${{ matrix.version }}/" e2e-tests/data/*.yaml | |
| sed -i "s/mode: .*/mode: ${{ matrix.mode }}/" e2e-tests/data/*.yaml | |
| git diff HEAD | |
| echo "Running e2e-tests/$test" | |
| bash e2e-tests/${{ matrix.test }} || exit 1 | |
| git reset --hard | |
| - name: Stop the operator | |
| run: | | |
| helm uninstall -n ${{ matrix.namespace }} flink-kubernetes-operator | |
| - name: Stop minikube | |
| run: | | |
| source e2e-tests/utils.sh | |
| stop_minikube |