[WIP][improve][ci] Enable Netty leak detection in CI and make leaks fail the build

.github/workflows/pulsar-ci-flaky.yaml

@@ -154,6 +154,7 @@ jobs:
       TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup
       THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis || 10000 }}
       THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps
+      NETTY_LEAK_DUMP_DIR: ${{ github.workspace }}/target/netty-leak-dumps
     runs-on: ubuntu-22.04
     timeout-minutes: 100
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -224,6 +225,14 @@ jobs:
             cat threadleak*.txt | awk '/^Summary:/ {print "::warning::" $0 "\n"; next} {print}'  
           fi
 
+      - name: Report detected Netty leaks
+        if: ${{ always() }}
+        run: |
+          if [ -d "$NETTY_LEAK_DUMP_DIR" ]; then
+            cd "$NETTY_LEAK_DUMP_DIR"
+            cat netty_leak_*.txt
+          fi
+
       - name: Create Jacoco reports
         if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
         continue-on-error: true
@@ -266,6 +275,7 @@ jobs:
             /tmp/*.hprof
             **/hs_err_*.log
             **/core.*
+            ${{ env.NETTY_LEAK_DUMP_DIR }}/*
             ${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/*
             ${{ env.THREAD_LEAK_DETECTOR_DIR }}/*
           retention-days: 7

.github/workflows/pulsar-ci.yaml

@@ -230,6 +230,7 @@ jobs:
       TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup
       THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis || 10000 }}
       THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps
+      NETTY_LEAK_DUMP_DIR: ${{ github.workspace }}/target/netty-leak-dumps
     runs-on: ubuntu-22.04
     timeout-minutes: ${{ matrix.timeout || 60 }}
     needs: ['preconditions', 'build-and-license-check']
@@ -347,6 +348,10 @@ jobs:
             cat threadleak*.txt | awk '/^Summary:/ {print "::warning::" $0 "\n"; next} {print}'
           fi
 
+      - name: Report detected Netty leaks
+        if: ${{ always() }}
+        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh report_netty_leaks
+
       - name: Upload Surefire reports
         uses: actions/upload-artifact@v4
         if: ${{ !success() || env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
@@ -364,6 +369,7 @@ jobs:
             /tmp/*.hprof
             **/hs_err_*.log
             **/core.*
+            ${{ env.NETTY_LEAK_DUMP_DIR }}/*
             ${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/*
             ${{ env.THREAD_LEAK_DETECTOR_DIR }}/*
           retention-days: 7
@@ -552,6 +558,7 @@ jobs:
       PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
       DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
       CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
+      NETTY_LEAK_DUMP_DIR: ${{ github.workspace }}/target/netty-leak-dumps
     strategy:
       fail-fast: false
       matrix:
@@ -689,20 +696,37 @@ jobs:
           report_paths: 'test-reports/TEST-*.xml'
           annotate_only: 'true'
 
+      - name: Report detected Netty leaks
+        if: ${{ always() }}
+        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh report_netty_leaks
+
       - name: Upload Surefire reports
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}
         with:
-          name: Integration-${{ matrix.group }}-surefire-reports
+          name: Integration-${{ matrix.name }}-surefire-reports
           path: surefire-reports
           retention-days: 7
 
+      - name: Upload possible heap dump, core dump or crash files
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: Integration-${{ matrix.name }}-dumps
+          path: |
+            /tmp/*.hprof
+            **/hs_err_*.log
+            **/core.*
+            ${{ env.NETTY_LEAK_DUMP_DIR }}/*
+          retention-days: 7
+          if-no-files-found: ignore
+
       - name: Upload container logs
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}
         continue-on-error: true
         with:
-          name: Integration-${{ matrix.group }}-container-logs
+          name: Integration-${{ matrix.name }}-container-logs
           path: tests/integration/target/container-logs
           retention-days: 7
 
@@ -959,6 +983,7 @@ jobs:
       PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
       DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
       CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
+      NETTY_LEAK_DUMP_DIR: ${{ github.workspace }}/target/netty-leak-dumps
     strategy:
       fail-fast: false
       matrix:
@@ -1066,6 +1091,10 @@ jobs:
           report_paths: 'test-reports/TEST-*.xml'
           annotate_only: 'true'
 
+      - name: Report detected Netty leaks
+        if: ${{ always() }}
+        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh report_netty_leaks
+
       - name: Upload container logs
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}
@@ -1083,6 +1112,19 @@ jobs:
           path: surefire-reports
           retention-days: 7
 
+      - name: Upload possible heap dump, core dump or crash files
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: System-${{ matrix.group }}-dumps
+          path: |
+            /tmp/*.hprof
+            **/hs_err_*.log
+            **/core.*
+            ${{ env.NETTY_LEAK_DUMP_DIR }}/*
+          retention-days: 7
+          if-no-files-found: ignore
+
       - name: Wait for ssh connection when build fails
         # ssh access is enabled for builds in own forks
         uses: ./.github/actions/ssh-access
@@ -1189,6 +1231,7 @@ jobs:
       PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
       DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
       CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
+      NETTY_LEAK_DUMP_DIR: ${{ github.workspace }}/target/netty-leak-dumps
     strategy:
       fail-fast: false
       matrix:
@@ -1273,23 +1316,40 @@ jobs:
           report_paths: 'test-reports/TEST-*.xml'
           annotate_only: 'true'
 
+      - name: Report detected Netty leaks
+        if: ${{ always() }}
+        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh report_netty_leaks
+
       - name: Upload container logs
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}
         continue-on-error: true
         with:
-          name: System-${{ matrix.group }}-container-logs
+          name: Flaky-System-${{ matrix.group }}-container-logs
           path: tests/integration/target/container-logs
           retention-days: 7
 
       - name: Upload Surefire reports
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}
         with:
-          name: System-${{ matrix.name }}-surefire-reports
+          name: Flaky-System-${{ matrix.name }}-surefire-reports
           path: surefire-reports
           retention-days: 7
 
+      - name: Upload possible heap dump, core dump or crash files
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: Flaky-System-${{ matrix.group }}-dumps
+          path: |
+            /tmp/*.hprof
+            **/hs_err_*.log
+            **/core.*
+            ${{ env.NETTY_LEAK_DUMP_DIR }}/*
+          retention-days: 7
+          if-no-files-found: ignore
+
       - name: Wait for ssh connection when build fails
         # ssh access is enabled for builds in own forks
         uses: ./.github/actions/ssh-access

build/pulsar_ci_tool.sh

@@ -579,6 +579,37 @@ ci_create_inttest_coverage_report() {
   echo "::endgroup::"
 }
 
+ci_report_netty_leaks() {
+  if [ -z "$NETTY_LEAK_DUMP_DIR" ]; then
+    echo "NETTY_LEAK_DUMP_DIR isn't set"
+    return 0
+  fi
+  local temp_file=$(mktemp -t netty_leak.XXXX)
+  {
+    if [ -d "$NETTY_LEAK_DUMP_DIR" ]; then
+      find "$NETTY_LEAK_DUMP_DIR" -maxdepth 1 -type f -name "netty_leak_*.txt" -exec cat {} \;
+    fi
+    if [ -d tests/integration/target/container-logs ]; then
+      find tests/integration/target/container-logs -type f -name "*.tar.gz" -exec tar -Ozxvf {} --wildcards --wildcards-match-slash '*/netty_leak_*.txt' \;
+    fi
+  } > $temp_file
+  if [ -s $temp_file ]; then
+    {
+      echo "::warning::Netty leaks found"
+      echo "Test file locations in stack traces:"
+      grep -h -i test $temp_file | grep org.apache | sed 's/^[[:space:]]*//;s/[[:space:]]*$//;s/^Hint: //' | sort -u
+      echo
+      echo "Details:"
+      cat $temp_file
+    } | tee $NETTY_LEAK_DUMP_DIR/leak_report.txt
+    touch target/netty_leaks_found
+  else
+    echo "No netty leaks found."
+    touch target/netty_leaks_not_found
+  fi
+  rm $temp_file
+}
+
 if [ -z "$1" ]; then
   echo "usage: $0 [ci_tool_function_name]"
   echo "Available ci tool functions:"

buildtools/pom.xml

@@ -141,12 +141,17 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <!-- for testing FastThreadLocalStateCleaner -->
     <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty-common</artifactId>
       <version>${netty.version}</version>
-      <scope>test</scope>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-buffer</artifactId>
+      <version>${netty.version}</version>
+      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.mockito</groupId>