Merge branch 'main' into add-envoy-proxy-example-jake

Author: JakeSCahill

.github/workflows/test-docs.yml

@@ -11,6 +11,7 @@ jobs:
     outputs:
       data-transforms: ${{ steps.filter.outputs.data-transforms }}
       envoy-shadowing: ${{ steps.filter.outputs.envoy-shadowing }}
+      redpanda-migrator: ${{ steps.filter.outputs.redpanda-migrator }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -24,6 +25,8 @@ jobs:
               - 'data-transforms/**'
             envoy-shadowing:
               - 'docker-compose/envoy-shadowing/**'
+            redpanda-migrator:
+              - 'docker-compose/redpanda-migrator-demo/**'
   run-tests:
     needs: setup
     strategy:
@@ -49,6 +52,9 @@ jobs:
         run: |
           # Redpanda uses Linux AIO. With 6 brokers each needing ~10000 events,
           # we exceed the default limit of 65536. Increase it.
+        if: needs.setup.outputs.redpanda-migrator == 'true'
+        run: |
+          # Redpanda uses Linux AIO. Increase limit to support multiple brokers.
           sudo sysctl -w fs.aio-max-nr=1048576
 
       - name: Test data transforms
@@ -60,6 +66,9 @@ jobs:
       - name: Test envoy shadowing
         if: needs.setup.outputs.envoy-shadowing == 'true'
         run: npm run test-envoy-shadowing
+      - name: Test redpanda migrator
+        if: needs.setup.outputs.redpanda-migrator == 'true'
+        run: npm run test-migrator
 
       - name: Upload debug artifacts
         if: failure()

docker-compose/README.adoc

@@ -7,4 +7,5 @@
 - link:./three-brokers/README.adoc[Three brokers]
 - link:./owl-shop/README.adoc[Owl shop streaming application]
 - link:./console-plain-login/README.adoc[Redpanda Console with Plain Login Authentication]
-- link:./cdc/README.adoc[Change data capture (CDC) with Debezium]
+- link:./cdc/README.adoc[Change data capture (CDC) with Debezium]
+- link:./redpanda-migrator-demo/README.md[Data migration from Kafka to Redpanda]

docker-compose/iceberg/spark/Dockerfile

@@ -31,7 +31,7 @@ RUN apt-get update && \
 
 # Set environment variables
 ENV SPARK_HOME="/opt/spark"
-ENV SPARK_VERSION=3.5.6
+ENV SPARK_VERSION=3.5.7
 ENV SPARK_MAJOR_VERSION=3.5
 ENV ICEBERG_VERSION=1.6.0
 ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip

docker-compose/jira-metrics-pipeline/.env.example

@@ -0,0 +1,11 @@
+# Redpanda, Console, and Connect versions
+REDPANDA_VERSION=v25.3.1
+REDPANDA_CONSOLE_VERSION=v3.3.1
+REDPANDA_CONNECT_VERSION=4.70.0
+
+# JIRA configuration
+# Get your JIRA API token from: https://id.atlassian.com/manage-profile/security/api-tokens
+JIRA_BASE_URL=https://your-domain.atlassian.net
+JIRA_USERNAME=your-email@example.com
+JIRA_API_TOKEN=your-api-token-here
+JIRA_PROJECT=YOUR_PROJECT_KEY