docker-images.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


# This workflow will build docker images when commit merged or pushed to master.
# or tags pushed.

name: Publish Docker Image

on:
  push:
    branches:
      - "master"
    tags:
      - "v*"


concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  docker-amoro:
    name: Push Amoro Docker Image to Docker Hub
    runs-on: ubuntu-latest
    if: ${{ startsWith(github.repository, 'NetEase/') }}
    strategy:
      matrix:
        hadoop: [ "v3" ]
    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK 8
        uses: actions/setup-java@v3
        with:
          java-version: '8'
          distribution: 'temurin'
          cache: maven
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Set up Docker tags
        uses: docker/metadata-action@v5
        id: meta
        with:
          flavor: |
            latest=auto
          images: |
            name=arctic163/amoro
          tags: |
            type=ref,event=branch,enable=${{ matrix.hadoop == 'v3' }},suffix=-snapshot
            type=ref,event=branch,enable=${{ matrix.hadoop == 'v3' }},suffix=-snapshot-hadoop3
            type=ref,event=branch,enable=${{ matrix.hadoop == 'v2' }},suffix=-snapshot-hadoop2
            type=semver,event=tag,enable=${{ matrix.hadoop == 'v3' }},pattern={{version}}
            type=semver,event=tag,enable=${{ matrix.hadoop == 'v2' }},pattern={{version}}-hadoop2

      - name: Print tags
        run: echo '${{ steps.meta.outputs.tags }}'

      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Set Maven Build Properties
        if: ${{ matrix.hadoop != 'v3' }}
        run: |
          echo "MVN_HADOOP=-Dhadoop=${{ matrix.hadoop }}" >> $GITHUB_ENV

      - name: Build dist module with Maven
        run: mvn clean install -pl 'ams/dist' -am -e ${MVN_HADOOP} -DskipTests -B -ntp

      - name: Build and Push Amoro Docker Image
        uses: docker/build-push-action@v4
        with:
          context: .
          push: true
          file: docker/amoro/Dockerfile
          platforms: linux/amd64,linux/arm64
          tags: ${{ steps.meta.outputs.tags }}

  docker-optimizer-flink:
    name: Push Amoro Optimizer-Flink Docker Image to Docker Hub
    runs-on: ubuntu-latest
    if: ${{ startsWith(github.repository, 'NetEase/') }}
    strategy:
      matrix:
        flink: [ "1.14.6", "1.18.1" ]
    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK 8
        uses: actions/setup-java@v3
        with:
          java-version: '8'
          distribution: 'temurin'
          cache: maven
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Set up Docker tags
        uses: docker/metadata-action@v5
        id: meta
        with:
          flavor: |
            latest=auto
          images: |
            name=arctic163/optimizer-flink
          tags: |
            type=ref,event=branch,enable=${{ matrix.flink == '1.14.6' }},suffix=-snapshot
            type=ref,event=branch,enable=${{ matrix.flink == '1.14.6' }},suffix=-snapshot-flink1.14
            type=ref,event=branch,enable=${{ matrix.flink == '1.18.1' }},suffix=-snapshot-flink1.18
            type=semver,enable=${{ matrix.flink == '1.14.6' }},pattern={{version}}
            type=semver,enable=${{ matrix.flink == '1.14.6' }},pattern={{version}}-flink1.14
            type=semver,enable=${{ matrix.flink == '1.18.1' }},pattern={{version}}-flink1.18

      - name: Print tags
        run: echo '${{ steps.meta.outputs.tags }}'

      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Set optimizer flink version
        run: |
          OPTIMIZER_FLINK=${{ matrix.flink }} && \
          echo "OPTIMIZER_FLINK=-Dflink-optimizer.flink-version${OPTIMIZER_FLINK}" >> $GITHUB_ENV
          if [[ "$OPTIMIZER_FLINK" < "1.15" ]]; then
            echo "Adding -Pflink-pre-1.15 for Flink version less than 1.15"
            echo "OPTIMIZER_FLINK=-Pflink-pre-1.15 -Dflink-optimizer.flink-version=${OPTIMIZER_FLINK}" >> $GITHUB_ENV
          fi

      - name: Set ENV Amoro version
        id: version
        run: |
          AMORO_VERSION=`cat pom.xml | grep 'amoro-parent' -C 3 | grep -Eo '<version>.*</version>' | awk -F'[><]' '{print $3}'` \
          && echo "$AMORO_VERSION" \
          && echo "AMORO_VERSION=${AMORO_VERSION}" >> $GITHUB_ENV \
          && echo "AMORO_VERSION=${AMORO_VERSION}" >> $GITHUB_OUTPUT

      - name: Build optimizer module with Maven
        run: mvn clean package -pl 'ams/optimizer/flink-optimizer' -am -e ${OPTIMIZER_FLINK} -DskipTests -B -ntp

      - name: Build and Push Flink Optimizer Docker Image
        uses: docker/build-push-action@v4
        env:
          AMORO_VERSION: ${{ steps.version.outputs.AMORO_VERSION }}
        with:
          context: .
          push: true
          file: docker/optimizer-flink/Dockerfile
          platforms: linux/amd64,linux/arm64
          tags: ${{ steps.meta.outputs.tags }}
          build-args: |
            FLINK_VERSION=${{ matrix.flink }}
            OPTIMIZER_JOB=ams/optimizer/flink-optimizer/target/flink-optimizer-${{ env.AMORO_VERSION }}-jar-with-dependencies.jar

  docker-optimizer-spark:
    name: Push Amoro Optimizer-Spark Docker Image to Docker Hub
    runs-on: ubuntu-latest
    if: ${{ startsWith(github.repository, 'NetEase/') }}
    strategy:
      matrix:
        spark: [ "3.3.3" ]
    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK 8
        uses: actions/setup-java@v3
        with:
          java-version: '8'
          distribution: 'temurin'
          cache: maven
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Set up Docker tags
        uses: docker/metadata-action@v5
        id: meta
        with:
          flavor: |
            latest=auto
          images: |
            name=arctic163/optimizer-spark
          tags: |
            type=ref,event=branch,enable=${{ matrix.spark == '3.3.3' }},suffix=-snapshot-spark3.3
            type=semver,enable=${{ matrix.spark == '3.3.3' }},pattern={{version}}-spark3.3

      - name: Print tags
        run: echo '${{ steps.meta.outputs.tags }}'

      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Set optimizer spark version
        run: |
          OPTIMIZER_SPARK=${{ matrix.spark }} && \
          echo "OPTIMIZER_SPARK=-Dspark-optimizer.spark-version${OPTIMIZER_SPARK}" >> $GITHUB_ENV

      - name: Set ENV Amoro version
        id: version
        run: |
          AMORO_VERSION=`cat pom.xml | grep 'amoro-parent' -C 3 | grep -Eo '<version>.*</version>' | awk -F'[><]' '{print $3}'` \
          && echo "$AMORO_VERSION" \
          && echo "AMORO_VERSION=${AMORO_VERSION}" >> $GITHUB_ENV \
          && echo "AMORO_VERSION=${AMORO_VERSION}" >> $GITHUB_OUTPUT

      - name: Build optimizer module with Maven
        run: mvn clean package -pl 'ams/optimizer/spark-optimizer' -am -e ${OPTIMIZER_SPARK} -DskipTests -B -ntp

      - name: Build and Push Spark Optimizer Docker Image
        uses: docker/build-push-action@v4
        env:
          AMORO_VERSION: ${{ steps.version.outputs.AMORO_VERSION }}
        with:
          context: .
          push: true
          file: docker/optimizer-spark/Dockerfile
          platforms: linux/amd64,linux/arm64
          tags: ${{ steps.meta.outputs.tags }}
          build-args: |
            SPARK_VERSION=${{ matrix.spark }}
            OPTIMIZER_JOB=ams/optimizer/spark-optimizer/target/spark-optimizer-${{ env.AMORO_VERSION }}-jar-with-dependencies.jar


  docker-quickdemo:
    name: Push Amoro Quick-Demo Docker Image to Docker Hub
    runs-on: ubuntu-latest
    needs: docker-amoro
    if: ${{ startsWith(github.repository, 'NetEase/') && startsWith(github.ref, 'refs/tags/' )}}
    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK 8
        uses: actions/setup-java@v3
        with:
          java-version: '8'
          distribution: 'temurin'
          cache: maven
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Set up Docker tags
        uses: docker/metadata-action@v5
        id: meta
        with:
          flavor: |
            latest=auto
          images: |
            name=arctic163/quickdemo
          tags: |
            type=ref,event=branch,suffix=-snapshot
            type=semver,pattern={{version}}

      - name: Print tags
        run: echo '${{ steps.meta.outputs.tags }}'

      - name: Set Amoro Tag
        id: tag
        run: |
          AMORO_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n 1) && echo "AMORO_TAG=${AMORO_TAG#*:}" >> $GITHUB_OUTPUT

      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Build optimizer module with Maven
        run: mvn clean package -pl 'mixed/flink/v1.17/flink-runtime' -am -e -DskipTests -B -ntp

      - name: Build and Push Flink Optimizer Docker Image
        uses: docker/build-push-action@v4
        with:
          context: .
          push: true
          file: docker/quickdemo/Dockerfile
          platforms: linux/amd64,linux/arm64
          tags: ${{ steps.meta.outputs.tags }}
          build-args: |
            AMORO_TAG=${{ steps.tag.outputs.AMORO_TAG }}
            FLINK_VERSION=1.18.1
            ICEBERG_VERSION=1.3.1