Create release branch for version 0.13.1 #14
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Java CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - 'release-*' | |
| pull_request: | |
| paths-ignore: | |
| - '**.bmp' | |
| - '**.gif' | |
| - '**.jpg' | |
| - '**.jpeg' | |
| - '**.md' | |
| - '**.pdf' | |
| - '**.png' | |
| - '**.svg' | |
| - '**.yaml' | |
| - '**.yml' | |
| - '.gitignore' | |
| branches: | |
| - master | |
| - 'release-*' | |
| env: | |
| MVN_ARGS: -e -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn | |
| SPARK_COMMON_MODULES: hudi-spark-datasource/hudi-spark,hudi-spark-datasource/hudi-spark-common | |
| jobs: | |
| validate-source: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| architecture: x64 | |
| - name: Check Binary Files | |
| run: ./scripts/release/validate_source_binary_files.sh | |
| - name: Check Copyright | |
| run: | | |
| ./scripts/release/create_source_directory.sh hudi-tmp-repo | |
| cd hudi-tmp-repo | |
| ./scripts/release/validate_source_copyright.sh | |
| - name: RAT check | |
| run: ./scripts/release/validate_source_rat.sh | |
| test-spark: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - scalaProfile: "scala-2.11" | |
| sparkProfile: "spark2.4" | |
| sparkModules: "hudi-spark-datasource/hudi-spark2" | |
| - scalaProfile: "scala-2.12" | |
| sparkProfile: "spark2.4" | |
| sparkModules: "hudi-spark-datasource/hudi-spark2" | |
| - scalaProfile: "scala-2.12" | |
| sparkProfile: "spark3.1" | |
| sparkModules: "hudi-spark-datasource/hudi-spark3.1.x" | |
| - scalaProfile: "scala-2.12" | |
| sparkProfile: "spark3.2" | |
| sparkModules: "hudi-spark-datasource/hudi-spark3.2.x" | |
| - scalaProfile: "scala-2.12" | |
| sparkProfile: "spark3.3" | |
| sparkModules: "hudi-spark-datasource/hudi-spark3.3.x" | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| architecture: x64 | |
| - name: Build Project | |
| env: | |
| SCALA_PROFILE: ${{ matrix.scalaProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| run: | |
| mvn clean install -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl hudi-examples/hudi-examples-spark,hudi-spark-datasource/hudi-spark -am -DskipTests=true $MVN_ARGS | |
| - name: Quickstart Test | |
| env: | |
| SCALA_PROFILE: ${{ matrix.scalaProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| run: | |
| mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl hudi-examples/hudi-examples-spark $MVN_ARGS | |
| - name: UT - Common & Spark | |
| env: | |
| SCALA_PROFILE: ${{ matrix.scalaProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SPARK_MODULES: ${{ matrix.sparkModules }} | |
| if: ${{ !endsWith(env.SPARK_PROFILE, '2.4') }} # skip test spark 2.4 as it's covered by Azure CI | |
| run: | |
| mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS | |
| - name: FT - Spark | |
| env: | |
| SCALA_PROFILE: ${{ matrix.scalaProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SPARK_MODULES: ${{ matrix.sparkModules }} | |
| if: ${{ !endsWith(env.SPARK_PROFILE, '2.4') }} # skip test spark 2.4 as it's covered by Azure CI | |
| run: | |
| mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS | |
| test-flink: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - flinkProfile: "flink1.13" | |
| - flinkProfile: "flink1.14" | |
| - flinkProfile: "flink1.15" | |
| - flinkProfile: "flink1.16" | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| architecture: x64 | |
| - name: Build Project | |
| env: | |
| SCALA_PROFILE: 'scala-2.12' | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| run: | |
| mvn clean install -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink -am -Davro.version=1.10.0 -DskipTests=true $MVN_ARGS | |
| - name: Quickstart Test | |
| env: | |
| SCALA_PROFILE: 'scala-2.12' | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| run: | |
| mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink $MVN_ARGS | |
| validate-bundles: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - flinkProfile: 'flink1.16' | |
| sparkProfile: 'spark3.3' | |
| sparkRuntime: 'spark3.3.2' | |
| - flinkProfile: 'flink1.15' | |
| sparkProfile: 'spark3.3' | |
| sparkRuntime: 'spark3.3.1' | |
| - flinkProfile: 'flink1.14' | |
| sparkProfile: 'spark3.2' | |
| sparkRuntime: 'spark3.2.3' | |
| - flinkProfile: 'flink1.13' | |
| sparkProfile: 'spark3.1' | |
| sparkRuntime: 'spark3.1.3' | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| architecture: x64 | |
| - name: Build Project | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SCALA_PROFILE: 'scala-2.12' | |
| run: | | |
| mvn clean package -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS | |
| # TODO remove the sudo below. It's a needed workaround as detailed in HUDI-5708. | |
| sudo chown -R "$USER:$(id -g -n)" hudi-platform-service/hudi-metaserver/target/generated-sources | |
| mvn clean package -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version=1.10.0 | |
| - name: IT - Bundle Validation - OpenJDK 8 | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_RUNTIME: ${{ matrix.sparkRuntime }} | |
| SCALA_PROFILE: 'scala-2.12' | |
| run: | | |
| HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) | |
| ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk8 | |
| - name: IT - Bundle Validation - OpenJDK 11 | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_RUNTIME: ${{ matrix.sparkRuntime }} | |
| SCALA_PROFILE: 'scala-2.12' | |
| run: | | |
| HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) | |
| ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk11 | |
| - name: IT - Bundle Validation - OpenJDK 17 | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SPARK_RUNTIME: ${{ matrix.sparkRuntime }} | |
| SCALA_PROFILE: 'scala-2.12' | |
| if: ${{ endsWith(env.SPARK_PROFILE, '3.3') }} # Only Spark 3.3 supports Java 17 as of now | |
| run: | | |
| HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) | |
| ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk17 |