fix: update cicd workflow to follow best practices

.github/workflows/dev.yml

@@ -0,0 +1,58 @@
+name: Deploy to Dev Environment
+
+concurrency: 1
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize]
+    branches: [main]
+    paths:
+      - "**/*.yml"
+      - "**/*.py"
+      - "src/**"
+      - "resources/**"
+
+env:
+  DATABRICKS_BUNDLE_ENV: dev
+  DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
+  BUNDLE_VAR_user_email: ${{ secrets.DATABRICKS_USER_EMAIL }}
+
+jobs:
+  deploy:
+    name: Deploy bundle to dev environment
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Databricks CLI
+        uses: databricks/setup-cli@main
+
+      - name: Install dependencies
+        run: pip install uv
+
+      - name: Deploy Databricks bundle
+        run: databricks bundle deploy
+
+  pipeline-update:
+    name: Run pipeline update
+    runs-on: ubuntu-latest
+    needs: deploy
+    if: success()
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Databricks CLI
+        uses: databricks/setup-cli@main
+
+      - name: Run pipeline update
+        shell: bash
+        run: |
+          set -euo pipefail
+          echo "Starting pipeline update..."
+          databricks bundle run flightradar_databricks_job --refresh-all 2>&1 | tee pipeline_update.log
+          echo "Pipeline update completed successfully"

.github/workflows/staging.yml

@@ -0,0 +1,57 @@
+name: Deploy to Staging Environment
+
+concurrency: 1
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+    paths:
+      - "**/*.yml"
+      - "**/*.py"
+      - "src/**"
+      - "resources/**"
+
+env:
+  DATABRICKS_BUNDLE_ENV: staging
+  DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
+  BUNDLE_VAR_user_email: ${{ secrets.DATABRICKS_USER_EMAIL }}
+
+jobs:
+  deploy:
+    name: Deploy bundle to staging environment
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Databricks CLI
+        uses: databricks/setup-cli@main
+
+      - name: Install dependencies
+        run: pip install uv
+
+      - name: Deploy Databricks bundle
+        run: databricks bundle deploy -t staging
+
+  pipeline-update:
+    name: Run pipeline update
+    runs-on: ubuntu-latest
+    needs: deploy
+    if: success()
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Databricks CLI
+        uses: databricks/setup-cli@main
+
+      - name: Run pipeline update
+        shell: bash
+        run: |
+          set -euo pipefail
+          echo "Starting pipeline update..."
+          databricks bundle run flightradar_databricks_job --refresh-all 2>&1 | tee pipeline_update.log
+          echo "Pipeline update completed successfully"