CSPL-3723: Initial implementation for GenAI deployment with AIPlatform & AIService controllers

.dockerignore

@@ -1,4 +1,8 @@
-# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
-# Ignore build and test binaries.
+.git
+.idea
+test
+vendor
 bin/
-testbin/
+build/_output/bin/splunk-operator-local
+splunk-operator-*.yaml
+splunk-operator-*.tar.gz

.env

@@ -0,0 +1,19 @@
+OPERATOR_SDK_VERSION=v1.31.0
+REVIEWERS=vivekr-splunk,rlieberman-splunk,patrykw-splunk,Igor-splunk,kasiakoziol
+GO_VERSION=1.23.0
+AWSCLI_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.8.6.zip
+KUBECTL_VERSION=v1.29.1
+AZ_CLI_VERSION=2.30.0
+EKSCTL_VERSION=v0.191.0
+EKS_CLUSTER_K8_VERSION=1.31
+EKS_INSTANCE_TYPE=m5.2xlarge
+EKS_INSTANCE_TYPE_ARM64=c6g.4xlarge
+SPLUNK_ENTERPRISE_RELEASE_IMAGE=splunk/splunk:9.4.1
+RELATED_IMAGE_SPLUNK_ENTERPRISE=splunk/splunk:9.2.3
+RELATED_IMAGE_RAY_HEAD=667741767953.dkr.ecr.us-west-2.amazonaws.com/ml-platform/ray/ray-head:build-5
+RELATED_IMAGE_RAY_WORKER=667741767953.dkr.ecr.us-west-2.amazonaws.com/ml-platform/ray/ray-worker-gpu:build-6
+RELATED_IMAGE_WEAVIATE=semitechnologies/weaviate:stable-v1.28-007846a
+RELATED_IMAGE_POST_INSTALL_HOOK=667741767953.dkr.ecr.us-west-2.amazonaws.com/vivek/ml-platform/saia/ai-helm-post-hook:0.0.5
+CLUSTER_NAME=sok-ml-platform
+MODEL_VERSION=v0.3.14-36-g1549f5a
+RAY_VERSION=2.44.0

.github/workflows/build-image.yml

@@ -0,0 +1,46 @@
+name: Build and Push Image
+on:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      SPLUNK_ENTERPRISE_IMAGE: splunk/splunk-enterprise:latest
+      SPLUNK_OPERATOR_IMAGE_NAME: splunk/splunk-ai-operator:latest
+      ECR_REPOSITORY: ${{ secrets.ECR_REPOSITORY }}
+      S3_REGION: us-west-2
+    steps:
+      - uses: sigstore/cosign-installer@main
+      - uses: actions/checkout@v2
+      - uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359
+        id: dotenv
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ steps.dotenv.outputs.GO_VERSION }}
+      - name: Install Ginkgo
+        run: make setup/ginkgo
+      - uses: docker/setup-buildx-action@v2.5.0
+      - name: Install Operator SDK
+        run: |
+          export ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac)
+          export OS=$(uname | awk '{print tolower($0)}')
+          export OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/${{ steps.dotenv.outputs.OPERATOR_SDK_VERSION }}
+          sudo curl -LO ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH}
+          sudo chmod +x operator-sdk_${OS}_${ARCH}
+          sudo mv operator-sdk_${OS}_${ARCH} /usr/local/bin/operator-sdk
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - uses: aws-actions/amazon-ecr-login@v1
+      - name: Build and push Splunk Operator Image
+        run: |
+          make docker-buildx IMG=${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:$GITHUB_SHA
+      - name: Sign Splunk Operator image with a key
+        run: |
+          cosign sign --yes --key env://COSIGN_PRIVATE_KEY  ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_OPERATOR_IMAGE_NAME }}:${{ github.sha }}
+        env:
+          COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
+          COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}

.github/workflows/check-formatting.yml

@@ -0,0 +1,16 @@
+name: Check Formatting
+on:
+  workflow_call:
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359
+        id: dotenv
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ steps.dotenv.outputs.GO_VERSION }}
+      - run: make fmt && if [[ $? -ne 0 ]]; then false; fi
+      - run: make vet && if [[ $? -ne 0 ]]; then false; fi

.github/workflows/main.yml

@@ -0,0 +1,33 @@
+name: Build and Test
+
+on:
+  pull_request: {}
+  push:
+    branches:
+      - main
+      - develop
+      - CSPL-3725
+permissions:
+  actions: read
+  contents: read
+  security-events: write
+
+jobs:
+  check-formatting:
+    uses: ./.github/workflows/check-formatting.yml
+
+  unit-tests:
+    uses: ./.github/workflows/unit-tests.yml
+    needs: check-formatting
+
+  build-image:
+    uses: ./.github/workflows/build-image.yml
+    needs: unit-tests
+
+  vulnerability-scan:
+    uses: ./.github/workflows/vulnerability-scan.yml
+    needs: build-image
+
+  smoke-tests:
+    uses: ./.github/workflows/smoke-tests.yml
+    needs: vulnerability-scan

.github/workflows/smoke-tests.yml

@@ -0,0 +1,141 @@
+name: Smoke Tests
+on:
+  workflow_call:
+
+jobs:
+  run-smoke-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [ basic ]
+    env:
+      CLUSTER_NODES: 1
+      CLUSTER_WORKERS: 3
+      TEST_FOCUS: "${{ matrix.test }}"
+      TEST_TO_SKIP: "^(?:[^s]+|s(?:$|[^m]|m(?:$|[^o]|o(?:$|[^k]|k(?:$|[^e])))))*$"
+      TEST_CLUSTER_PLATFORM: eks
+      CLUSTER_WIDE: "true"
+      DEPLOYMENT_TYPE: ""
+    steps:
+      - name: Set Test Cluster Name
+        run: echo "TEST_CLUSTER_NAME=eks-integration-test-cluster-${{ matrix.test }}-$GITHUB_RUN_ID" >> $GITHUB_ENV
+
+      - uses: actions/checkout@v2
+
+      - uses: falti/dotenv-action@d4d12eaa0e1dd06d5bdc3d7af3bf4c8c93cb5359
+        id: dotenv
+
+      - name: Set SPLUNK_ENTERPRISE_IMAGE for main branch
+        if: github.ref == 'refs/heads/main'
+        run: echo "SPLUNK_ENTERPRISE_IMAGE=${{ steps.dotenv.outputs.SPLUNK_ENTERPRISE_RELEASE_IMAGE }}" >> $GITHUB_ENV
+
+      - uses: Azure/setup-kubectl@v3
+        with:
+          version: ${{ steps.dotenv.outputs.KUBECTL_VERSION }}
+
+      - uses: actions/setup-python@v2
+
+      - name: Install AWS CLI
+        run: |
+          curl "${{ steps.dotenv.outputs.AWSCLI_URL }}" -o "awscliv2.zip"
+          unzip awscliv2.zip
+          sudo ./aws/install --update
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ steps.dotenv.outputs.GO_VERSION }}
+
+      - run: make setup/ginkgo
+
+      - name: Install Helm
+        run: |
+          curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+          chmod 700 get_helm.sh
+          DESIRED_VERSION=v3.8.2 bash get_helm.sh
+
+      - name: Install eksctl
+        run: |
+          curl -sL "https://github.com/weaveworks/eksctl/releases/download/${{ steps.dotenv.outputs.EKSCTL_VERSION }}/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp
+          sudo mv /tmp/eksctl /usr/local/bin
+
+      - uses: docker/setup-buildx-action@v2.5.0
+
+      - name: Install Operator SDK
+        run: |
+          sudo curl -L -o /usr/local/bin/operator-sdk https://github.com/operator-framework/operator-sdk/releases/download/${{ steps.dotenv.outputs.OPERATOR_SDK_VERSION }}/operator-sdk-${{ steps.dotenv.outputs.OPERATOR_SDK_VERSION }}-x86_64-linux-gnu
+          sudo chmod +x /usr/local/bin/operator-sdk
+
+      - uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set SPLUNK_OPERATOR_IMAGE
+        run: echo "SPLUNK_OPERATOR_IMAGE=splunk/splunk-ai-operator:$GITHUB_SHA" >> $GITHUB_ENV
+
+      - run: docker pull ${{ env.SPLUNK_ENTERPRISE_IMAGE }}
+
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+
+      - uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Tag and Push Splunk Enterprise Image
+        run: |
+          docker tag ${{ env.SPLUNK_ENTERPRISE_IMAGE }} ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }}
+          docker push ${{ secrets.ECR_REPOSITORY }}/${{ env.SPLUNK_ENTERPRISE_IMAGE }}
+
+      - name: Install kubetest2 and eks plugin
+        run: |
+          go install sigs.k8s.io/kubetest2@latest
+          go install sigs.k8s.io/kubetest2/kubetest2-eks@latest
+          echo "$HOME/go/bin" >> $GITHUB_PATH
+
+      - uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/github-role
+          aws-region: us-west-2
+
+      - name: Create EKS Cluster
+        run: |
+          kubetest2 eks --cluster-name ai-splunk-operator-cluster --region us-west-2 --up --nodes 3 --nodes-min 1 --nodes-max 5
+
+      - run: kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
+
+      - run: kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.5/aio/deploy/recommended.yaml
+
+      - name: Setup Kustomize
+        run: |
+          sudo snap install kustomize
+          mkdir -p ./bin
+          cp /snap/bin/kustomize ./bin/kustomize
+
+      - name: Run smoke test
+        run: make int-test
+
+      - name: Collect Test Logs
+        if: ${{ always() }}
+        run: |
+          mkdir -p /tmp/pod_logs
+          find ./test -name "*.log" -exec cp {} /tmp/pod_logs \;
+
+      - uses: actions/upload-artifact@v4.4.0
+        if: ${{ always() }}
+        with:
+          name: "splunk-pods-logs--artifacts-${{ matrix.test }}"
+          path: "/tmp/pod_logs/**"
+
+      - name: Cleanup
+        if: ${{ always() }}
+        run: |
+          make cleanup
+          make clean
+
+      - name: Delete EKS Cluster
+        if: always()
+        run: |
+          kubetest2 eks --cluster-name ai-splunk-operator-cluster --region us-west-2 --down