Add some data validation examples #49
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| push: | |
| jobs: | |
| test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - java: 17 | |
| - java: 11 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Sync the current branch with the latest | |
| if: github.repository != 'high-performance-spark/high-performance-spark-examples' | |
| id: sync-branch | |
| run: | | |
| git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF##*/} | |
| git -c user.name='Spark Test Account' -c user.email='[email protected]' merge --no-commit --progress --squash FETCH_HEAD || echo "no merge needed." | |
| git -c user.name='Spark Test Account' -c user.email='[email protected]' commit -m "Merged commit" || echo "no merge needed." | |
| - name: Setup JDK | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ matrix.java }} | |
| cache: sbt | |
| - name: Scala Build and Test | |
| run: sbt clean package +test | |
| python-test: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install tox tox-gh-actions | |
| - name: Run tox | |
| run: | | |
| cd python; tox | |
| run-sql-examples: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Cache Spark and friends | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| spark*.tgz | |
| iceberg*.jar | |
| key: spark-artifacts | |
| - name: Cache Data | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| data/fetched/* | |
| key: data-fetched | |
| - name: Run sql examples | |
| run: | |
| ./run_sql_examples.sh | |
| run-pyspark-examples: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Cache Spark and friends | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| spark*.tgz | |
| iceberg*.jar | |
| key: spark-artifacts | |
| - name: Cache Data | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| data/fetched/* | |
| key: data-fetched | |
| - name: Run PySpark examples | |
| run: | |
| ./run_pyspark_examples.sh | |
| style: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Shellcheck | |
| run: | | |
| sudo apt-get install -y shellcheck | |
| shellcheck $(find -name "*.sh") | |
| - name: Setup JDK | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: temurin | |
| java-version: 17 | |
| cache: sbt | |
| - name: scala | |
| run: | |
| sbt scalastyle |