From 59bc7566489c4467ab064f7ce9667724d2eeb6fa Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 29 Aug 2025 15:00:55 +0800 Subject: [PATCH 1/6] fix: enhance query formatting to support ? placeholders and improve compatibility for DBAPI --- chdb/dbapi/cursors.py | 65 +++++++++++++++++++++++++++------ tests/test_dbapi_persistence.py | 53 +++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 12 deletions(-) diff --git a/chdb/dbapi/cursors.py b/chdb/dbapi/cursors.py index c2b464ac247..e7b25edcbe8 100644 --- a/chdb/dbapi/cursors.py +++ b/chdb/dbapi/cursors.py @@ -6,7 +6,7 @@ # You can use it to load large dataset. RE_INSERT_VALUES = re.compile( r"\s*((?:INSERT|REPLACE)\b.+\bVALUES?\s*)" - + r"(\(\s*(?:%s|%\(.+\)s)\s*(?:,\s*(?:%s|%\(.+\)s)\s*)*\))" + + r"(\(\s*(?:%s|%\(.+\)s|\?)\s*(?:,\s*(?:%s|%\(.+\)s|\?)\s*)*\))" + r"(\s*(?:ON DUPLICATE.*)?);?\s*\Z", re.IGNORECASE | re.DOTALL, ) @@ -99,6 +99,42 @@ def _escape_args(self, args, conn): # Worst case it will throw a Value error return conn.escape(args) + def _format_query(self, query, args, conn): + """Format query with arguments supporting ? and % placeholders.""" + if args is None: + return query + + if isinstance(args, (tuple, list)) and '?' in query: + escaped_args = self._escape_args(args, conn) + result = [] + arg_index = 0 + in_string = False + quote_char = None + + for i, char in enumerate(query): + # Track string literals to avoid replacing ? inside strings + if not in_string and char in ("'", '"'): + in_string = True + quote_char = char + elif in_string and char == quote_char: + # Check if it's an escaped quote + if i == 0 or query[i-1] != '\\': + in_string = False + quote_char = None + + # Only replace ? outside of string literals + if char == '?' and not in_string and arg_index < len(escaped_args): + result.append(str(escaped_args[arg_index])) + arg_index += 1 + else: + result.append(char) + + return ''.join(result) + elif '%' in query: + return query % self._escape_args(args, conn) + + return query + def mogrify(self, query, args=None): """ Returns the exact string that is sent to the database by calling the @@ -107,11 +143,7 @@ def mogrify(self, query, args=None): This method follows the extension to the DB API 2.0 followed by Psycopg. """ conn = self._get_db() - - if args is not None: - query = query % self._escape_args(args, conn) - - return query + return self._format_query(query, args, conn) def execute(self, query, args=None): """Execute a query @@ -124,12 +156,11 @@ def execute(self, query, args=None): :return: Number of affected rows :rtype: int - If args is a list or tuple, %s can be used as a placeholder in the query. + If args is a list or tuple, ? can be used as a placeholder in the query. If args is a dict, %(name)s can be used as a placeholder in the query. + Also supports %s placeholder for backward compatibility. """ - if args is not None: - query = query % self._escape_args(args, self.connection) - + query = self._format_query(query, args, self.connection) self._cursor.execute(query) # Get description from column names and types @@ -194,13 +225,23 @@ def _do_execute_many( postfix = postfix.encode(encoding) sql = prefix args = iter(args) - v = values % escape(next(args), conn) + + first_arg = next(args) + if '?' in values: + v = self._format_query(values, first_arg, conn) + else: + v = values % escape(first_arg, conn) + if isinstance(v, str): v = v.encode(encoding, "surrogateescape") sql += v rows = 0 for arg in args: - v = values % escape(arg, conn) + if '?' in values: + v = self._format_query(values, arg, conn) + else: + v = values % escape(arg, conn) + if isinstance(v, str): v = v.encode(encoding, "surrogateescape") if len(sql) + len(v) + len(postfix) + 1 > max_stmt_length: diff --git a/tests/test_dbapi_persistence.py b/tests/test_dbapi_persistence.py index 5cb3d270cbb..90101f332ba 100644 --- a/tests/test_dbapi_persistence.py +++ b/tests/test_dbapi_persistence.py @@ -35,6 +35,59 @@ def test_persistence(self): row = cur2.fetchone() self.assertEqual(("he", 32), row) + def test_placeholder1(self): + conn = dbapi.connect(path=test_state_dir) + cur = conn.cursor() + + cur.execute("CREATE DATABASE test ENGINE = Atomic;") + cur.execute( + "CREATE TABLE test.users (id UInt64, name String, age UInt32) " + "ENGINE = MergeTree ORDER BY id;" + ) + + cur.execute("INSERT INTO test.users (id, name, age) VALUES (?, ?, ?)", + (1, 'Alice', 25)) + + cur.execute("SELECT name, age FROM test.users WHERE id = ? AND age > ?", + (1, 20)) + row = cur.fetchone() + self.assertEqual(("Alice", 25), row) + + data = [(2, 'Bob', 30), (3, 'Charlie', 35), (4, 'David', 28)] + cur.executemany("INSERT INTO test.users (id, name, age) VALUES (?, ?, ?)", + data) + + cur.execute("SELECT COUNT(*) FROM test.users WHERE id > 1") + count = cur.fetchone()[0] + self.assertEqual(3, count) + cur.execute("SELECT name FROM test.users WHERE age = ? ORDER BY id", (30,)) + result = cur.fetchone() + self.assertEqual(("Bob",), result) + cur.close() + conn.close() + + def test_placeholder2(self): + conn = dbapi.connect(path=test_state_dir) + cur = conn.cursor() + + # Create table + cur.execute("CREATE DATABASE compat ENGINE = Atomic;") + cur.execute( + "CREATE TABLE compat.test (id UInt64, value String) " + "ENGINE = MergeTree ORDER BY id;" + ) + + # Test %s placeholders still work + cur.execute("INSERT INTO compat.test (id, value) VALUES (%s, %s)", + (1, 'test_value')) + + cur.execute("SELECT value FROM compat.test") + result = cur.fetchone() + self.assertEqual(("test_value",), result) + + cur.close() + conn.close() + if __name__ == "__main__": unittest.main() From f0db7a6e13f1efdbc3465adc8f6969d26e15a964 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 2 Sep 2025 11:03:01 +0800 Subject: [PATCH 2/6] improve query formatting to handle both ? and % placeholders --- chdb/dbapi/cursors.py | 79 ++++++++++++++++----------------- tests/test_dbapi_persistence.py | 2 - 2 files changed, 38 insertions(+), 43 deletions(-) diff --git a/chdb/dbapi/cursors.py b/chdb/dbapi/cursors.py index e7b25edcbe8..032ce0118b7 100644 --- a/chdb/dbapi/cursors.py +++ b/chdb/dbapi/cursors.py @@ -101,39 +101,46 @@ def _escape_args(self, args, conn): def _format_query(self, query, args, conn): """Format query with arguments supporting ? and % placeholders.""" - if args is None: + if args is None or ('?' not in query and '%' not in query): return query - if isinstance(args, (tuple, list)) and '?' in query: - escaped_args = self._escape_args(args, conn) - result = [] - arg_index = 0 - in_string = False - quote_char = None - - for i, char in enumerate(query): - # Track string literals to avoid replacing ? inside strings - if not in_string and char in ("'", '"'): + escaped_args = self._escape_args(args, conn) + if not isinstance(escaped_args, (tuple, list)): + escaped_args = (escaped_args,) + + result = [] + arg_index = 0 + max_args = len(escaped_args) + i = 0 + query_len = len(query) + in_string = False + quote_char = None + + while i < query_len: + char = query[i] + if not in_string: + if char in ("'", '"'): in_string = True quote_char = char - elif in_string and char == quote_char: - # Check if it's an escaped quote - if i == 0 or query[i-1] != '\\': - in_string = False - quote_char = None - - # Only replace ? outside of string literals - if char == '?' and not in_string and arg_index < len(escaped_args): - result.append(str(escaped_args[arg_index])) - arg_index += 1 - else: - result.append(char) - - return ''.join(result) - elif '%' in query: - return query % self._escape_args(args, conn) - - return query + elif arg_index < max_args: + if char == '?': + result.append(str(escaped_args[arg_index])) + arg_index += 1 + i += 1 + continue + elif char == '%' and i + 1 < query_len and query[i + 1] == 's': + result.append(str(escaped_args[arg_index])) + arg_index += 1 + i += 2 + continue + elif char == quote_char and (i == 0 or query[i - 1] != '\\'): + in_string = False + quote_char = None + + result.append(char) + i += 1 + + return ''.join(result) def mogrify(self, query, args=None): """ @@ -218,7 +225,6 @@ def _do_execute_many( self, prefix, values, postfix, args, max_stmt_length, encoding ): conn = self._get_db() - escape = self._escape_args if isinstance(prefix, str): prefix = prefix.encode(encoding) if isinstance(postfix, str): @@ -226,22 +232,13 @@ def _do_execute_many( sql = prefix args = iter(args) - first_arg = next(args) - if '?' in values: - v = self._format_query(values, first_arg, conn) - else: - v = values % escape(first_arg, conn) - + v = self._format_query(values, next(args), conn) if isinstance(v, str): v = v.encode(encoding, "surrogateescape") sql += v rows = 0 for arg in args: - if '?' in values: - v = self._format_query(values, arg, conn) - else: - v = values % escape(arg, conn) - + v = self._format_query(values, arg, conn) if isinstance(v, str): v = v.encode(encoding, "surrogateescape") if len(sql) + len(v) + len(postfix) + 1 > max_stmt_length: diff --git a/tests/test_dbapi_persistence.py b/tests/test_dbapi_persistence.py index 90101f332ba..4dc70ae336f 100644 --- a/tests/test_dbapi_persistence.py +++ b/tests/test_dbapi_persistence.py @@ -76,8 +76,6 @@ def test_placeholder2(self): "CREATE TABLE compat.test (id UInt64, value String) " "ENGINE = MergeTree ORDER BY id;" ) - - # Test %s placeholders still work cur.execute("INSERT INTO compat.test (id, value) VALUES (%s, %s)", (1, 'test_value')) From 6eefaa47dbb93794f985288474d89d400a3ab6b9 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 2 Sep 2025 16:27:01 +0800 Subject: [PATCH 3/6] improve query handling for better performance --- chdb/dbapi/cursors.py | 76 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/chdb/dbapi/cursors.py b/chdb/dbapi/cursors.py index 032ce0118b7..cf54e87eaeb 100644 --- a/chdb/dbapi/cursors.py +++ b/chdb/dbapi/cursors.py @@ -221,6 +221,31 @@ def executemany(self, query, args): self.rowcount = sum(self.execute(query, arg) for arg in args) return self.rowcount + def _find_placeholder_positions(self, query): + positions = [] + i = 0 + query_len = len(query) + in_string = False + quote_char = None + + while i < query_len: + char = query[i] + if not in_string: + if char in ("'", '"'): + in_string = True + quote_char = char + elif char == '?': + positions.append((i, 1)) # (position, length) + elif char == '%' and i + 1 < query_len and query[i + 1] == 's': + positions.append((i, 2)) + i += 1 + elif char == quote_char and (i == 0 or query[i - 1] != '\\'): + in_string = False + quote_char = None + i += 1 + + return positions + def _do_execute_many( self, prefix, values, postfix, args, max_stmt_length, encoding ): @@ -229,24 +254,65 @@ def _do_execute_many( prefix = prefix.encode(encoding) if isinstance(postfix, str): postfix = postfix.encode(encoding) + + # Pre-compute placeholder positions + placeholder_positions = self._find_placeholder_positions(values) + sql = prefix args = iter(args) - v = self._format_query(values, next(args), conn) + if not placeholder_positions: + values_bytes = values.encode(encoding, "surrogateescape") if isinstance(values, str) else values + sql += values_bytes + rows = 0 + for _ in args: + if len(sql) + len(values_bytes) + len(postfix) + 2 > max_stmt_length: + rows += self.execute(sql + postfix) + sql = prefix + values_bytes + else: + sql += ",".encode(encoding) + sql += values_bytes + rows += self.execute(sql + postfix) + self.rowcount = rows + return rows + + template_parts = [] + last_pos = 0 + for pos, length in placeholder_positions: + template_parts.append(values[last_pos:pos]) + last_pos = pos + length + template_parts.append(values[last_pos:]) + + def format_values_fast(escaped_arg): + if len(escaped_arg) != len(placeholder_positions): + return values + result = template_parts[0] + for i, val in enumerate(escaped_arg): + result += str(val) + template_parts[i + 1] + return result + + def format_values_with_positions(arg): + escaped_arg = self._escape_args(arg, conn) + if not isinstance(escaped_arg, (tuple, list)): + escaped_arg = (escaped_arg,) + return format_values_fast(escaped_arg) + + v = format_values_with_positions(next(args)) if isinstance(v, str): v = v.encode(encoding, "surrogateescape") sql += v rows = 0 + for arg in args: - v = self._format_query(values, arg, conn) + v = format_values_with_positions(arg) if isinstance(v, str): v = v.encode(encoding, "surrogateescape") - if len(sql) + len(v) + len(postfix) + 1 > max_stmt_length: + if len(sql) + len(v) + len(postfix) + 2 > max_stmt_length: # +2 for comma rows += self.execute(sql + postfix) - sql = prefix + sql = prefix + v else: sql += ",".encode(encoding) - sql += v + sql += v rows += self.execute(sql + postfix) self.rowcount = rows return rows From 3fa7a25a92b18ca862271b5d4a05d090f2ab236d Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 3 Sep 2025 11:39:56 +0800 Subject: [PATCH 4/6] upload core file when segment fault. --- .../workflows/build_linux_arm64_wheels-gh.yml | 39 ++++++++++++++++++- .github/workflows/build_linux_x86_wheels.yml | 38 +++++++++++++++++- .../workflows/build_macos_arm64_wheels.yml | 39 ++++++++++++++++++- .github/workflows/build_macos_x86_wheels.yml | 39 ++++++++++++++++++- 4 files changed, 151 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml index a28985e858d..8466aa56bf1 100644 --- a/.github/workflows/build_linux_arm64_wheels-gh.yml +++ b/.github/workflows/build_linux_arm64_wheels-gh.yml @@ -119,13 +119,50 @@ jobs: sudo rm -f dist/*linux_aarch64.whl ls -lh dist shell: bash + - name: Enable core dumps + run: | + ulimit -c unlimited + echo 'tmp/core.%p' | sudo tee /proc/sys/kernel/core_pattern + mkdir -p tmp + - name: Run tests run: | python3 -m pip install dist/*.whl python3 -m pip install pandas pyarrow psutil deltalake python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)" + ulimit -c unlimited make test - continue-on-error: false + continue-on-error: true + + - name: Check for core files and upload if found + if: always() # Run if tests fail due to segfaults + run: | + echo "Checking for core files..." + find tmp -name 'core.*' -type f 2>/dev/null | head -10 + find . -name 'core.*' -type f 2>/dev/null | head -10 + + # Create archive if core files exist + if find tmp -name 'core.*' -type f 2>/dev/null | grep -q .; then + mkdir -p core_files + find tmp -name 'core.*' -type f -exec cp {} core_files/ \; + tar -czf core-files-linux-aarch64-${{ matrix.python-version }}-${{ github.run_id }}.tar.gz core_files/ + echo "CORE_FILES_EXIST=true" >> $GITHUB_ENV + elif find . -name 'core.*' -type f 2>/dev/null | grep -q .; then + mkdir -p core_files + find . -name 'core.*' -type f -exec cp {} core_files/ \; + tar -czf core-files-linux-aarch64-${{ matrix.python-version }}-${{ github.run_id }}.tar.gz core_files/ + echo "CORE_FILES_EXIST=true" >> $GITHUB_ENV + else + echo "No core files found" + echo "CORE_FILES_EXIST=false" >> $GITHUB_ENV + fi + + - name: Upload core files as artifact + if: always() && env.CORE_FILES_EXIST == 'true' + uses: actions/upload-artifact@v4 + with: + name: core-files-linux-aarch64-${{ matrix.python-version }}-${{ github.run_id }} + path: core-files-*.tar.gz - name: Upload wheels to release if: startsWith(github.ref, 'refs/tags/v') run: | diff --git a/.github/workflows/build_linux_x86_wheels.yml b/.github/workflows/build_linux_x86_wheels.yml index cb9dfd63c2b..91339b9cc95 100644 --- a/.github/workflows/build_linux_x86_wheels.yml +++ b/.github/workflows/build_linux_x86_wheels.yml @@ -117,13 +117,49 @@ jobs: sudo rm -f dist/*-linux_x86_64.whl ls -lh dist shell: bash + - name: Enable core dumps + run: | + ulimit -c unlimited + echo 'tmp/core.%p' | sudo tee /proc/sys/kernel/core_pattern + mkdir -p tmp + - name: Run tests run: | python3 -m pip install dist/*.whl python3 -m pip install pandas pyarrow psutil deltalake python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)" + ulimit -c unlimited make test - continue-on-error: false + continue-on-error: true + + - name: Check for core files and upload if found + if: always() # Run if tests fail due to segfaults + run: | + echo "Checking for core files..." + find tmp -name 'core.*' -type f 2>/dev/null | head -10 + find . -name 'core.*' -type f 2>/dev/null | head -10 + + if find tmp -name 'core.*' -type f 2>/dev/null | grep -q .; then + mkdir -p core_files + find tmp -name 'core.*' -type f -exec cp {} core_files/ \; + tar -czf core-files-linux-x86_64-${{ matrix.python-version }}-${{ github.run_id }}.tar.gz core_files/ + echo "CORE_FILES_EXIST=true" >> $GITHUB_ENV + elif find . -name 'core.*' -type f 2>/dev/null | grep -q .; then + mkdir -p core_files + find . -name 'core.*' -type f -exec cp {} core_files/ \; + tar -czf core-files-linux-x86_64-${{ matrix.python-version }}-${{ github.run_id }}.tar.gz core_files/ + echo "CORE_FILES_EXIST=true" >> $GITHUB_ENV + else + echo "No core files found" + echo "CORE_FILES_EXIST=false" >> $GITHUB_ENV + fi + + - name: Upload core files as artifact + if: always() && env.CORE_FILES_EXIST == 'true' + uses: actions/upload-artifact@v4 + with: + name: core-files-linux-x86_64-${{ matrix.python-version }}-${{ github.run_id }} + path: core-files-*.tar.gz - name: Upload wheels to release if: startsWith(github.ref, 'refs/tags/v') run: | diff --git a/.github/workflows/build_macos_arm64_wheels.yml b/.github/workflows/build_macos_arm64_wheels.yml index c6c025486da..d0231ced20e 100644 --- a/.github/workflows/build_macos_arm64_wheels.yml +++ b/.github/workflows/build_macos_arm64_wheels.yml @@ -121,13 +121,50 @@ jobs: - name: Fix wheel platform tag run: | python3 -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/*.whl + - name: Enable core dumps + run: | + ulimit -c unlimited + sudo sysctl -w kern.corefile=$(pwd)/tmp/core.%P + mkdir -p tmp + - name: Run tests run: | python3 -m pip install dist/*.whl python3 -m pip install pandas pyarrow psutil deltalake python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)" + ulimit -c unlimited make test - continue-on-error: false + continue-on-error: true + + - name: Check for core files and upload if found + if: always() # Run if tests fail due to segfaults + run: | + echo "Checking for core files..." + find tmp -name 'core.*' -type f 2>/dev/null | head -10 + find . -name 'core.*' -type f 2>/dev/null | head -10 + + # Create archive if core files exist + if find tmp -name 'core.*' -type f 2>/dev/null | grep -q .; then + mkdir -p core_files + find tmp -name 'core.*' -type f -exec cp {} core_files/ \; + tar -czf core-files-macos-arm64-${{ matrix.python-version }}-${{ github.run_id }}.tar.gz core_files/ + echo "CORE_FILES_EXIST=true" >> $GITHUB_ENV + elif find . -name 'core.*' -type f 2>/dev/null | grep -q .; then + mkdir -p core_files + find . -name 'core.*' -type f -exec cp {} core_files/ \; + tar -czf core-files-macos-arm64-${{ matrix.python-version }}-${{ github.run_id }}.tar.gz core_files/ + echo "CORE_FILES_EXIST=true" >> $GITHUB_ENV + else + echo "No core files found" + echo "CORE_FILES_EXIST=false" >> $GITHUB_ENV + fi + + - name: Upload core files as artifact + if: always() && env.CORE_FILES_EXIST == 'true' + uses: actions/upload-artifact@v4 + with: + name: core-files-macos-arm64-${{ matrix.python-version }}-${{ github.run_id }} + path: core-files-*.tar.gz - name: Show files run: ls -lh dist shell: bash diff --git a/.github/workflows/build_macos_x86_wheels.yml b/.github/workflows/build_macos_x86_wheels.yml index 369c78158ec..00affa16f26 100644 --- a/.github/workflows/build_macos_x86_wheels.yml +++ b/.github/workflows/build_macos_x86_wheels.yml @@ -121,13 +121,50 @@ jobs: - name: Fix wheel platform tag run: | python3 -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl + - name: Enable core dumps + run: | + ulimit -c unlimited + sudo sysctl -w kern.corefile=$(pwd)/tmp/core.%P + mkdir -p tmp + - name: Run tests run: | python3 -m pip install dist/*.whl python3 -m pip install pandas pyarrow psutil deltalake python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)" + ulimit -c unlimited make test - continue-on-error: false + continue-on-error: true + + - name: Check for core files and upload if found + if: always() # Run if tests fail due to segfaults + run: | + echo "Checking for core files..." + find tmp -name 'core.*' -type f 2>/dev/null | head -10 + find . -name 'core.*' -type f 2>/dev/null | head -10 + + # Create archive if core files exist + if find tmp -name 'core.*' -type f 2>/dev/null | grep -q .; then + mkdir -p core_files + find tmp -name 'core.*' -type f -exec cp {} core_files/ \; + tar -czf core-files-macos-x86_64-${{ matrix.python-version }}-${{ github.run_id }}.tar.gz core_files/ + echo "CORE_FILES_EXIST=true" >> $GITHUB_ENV + elif find . -name 'core.*' -type f 2>/dev/null | grep -q .; then + mkdir -p core_files + find . -name 'core.*' -type f -exec cp {} core_files/ \; + tar -czf core-files-macos-x86_64-${{ matrix.python-version }}-${{ github.run_id }}.tar.gz core_files/ + echo "CORE_FILES_EXIST=true" >> $GITHUB_ENV + else + echo "No core files found" + echo "CORE_FILES_EXIST=false" >> $GITHUB_ENV + fi + + - name: Upload core files as artifact + if: always() && env.CORE_FILES_EXIST == 'true' + uses: actions/upload-artifact@v4 + with: + name: core-files-macos-x86_64-${{ matrix.python-version }}-${{ github.run_id }} + path: core-files-*.tar.gz - name: Show files run: ls -lh dist shell: bash From b6045aaba7ecaba087d9438a324ad7667d661ab8 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 3 Sep 2025 12:08:29 +0800 Subject: [PATCH 5/6] Add core dump collection and upload for Linux and macOS workflows --- .../workflows/build_linux_arm64_wheels-gh.yml | 24 ++++++++++++++++++ .github/workflows/build_linux_x86_wheels.yml | 24 ++++++++++++++++++ .../workflows/build_macos_arm64_wheels.yml | 25 +++++++++++++++++++ .github/workflows/build_macos_x86_wheels.yml | 25 +++++++++++++++++++ 4 files changed, 98 insertions(+) diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml index bdc8b7a5ca5..d4e8b7ca4b2 100644 --- a/.github/workflows/build_linux_arm64_wheels-gh.yml +++ b/.github/workflows/build_linux_arm64_wheels-gh.yml @@ -158,6 +158,11 @@ jobs: sudo rm -f dist/*linux_aarch64.whl ls -lh dist shell: bash + - name: Setup core dump collection + run: | + mkdir -p tmp/core + echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern + ulimit -c unlimited - name: Test wheel on all Python versions run: | export PATH="$HOME/.pyenv/bin:$PATH" @@ -171,6 +176,25 @@ jobs: pyenv shell --unset done continue-on-error: false + - name: Check and upload core files if present + if: always() + run: | + if ls tmp/core/core.* >/dev/null 2>&1; then + echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV + tar -czvf core-files-linux-aarch64.tar.gz tmp/core/core.* + echo "Core files tar created: core-files-linux-aarch64.tar.gz" + ls -lh core-files-linux-aarch64.tar.gz + else + echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV + echo "No core files found in tmp/core" + fi + continue-on-error: true + - name: Upload core files if present + if: always() && env.CORE_FILES_FOUND == 'true' + uses: actions/upload-artifact@v4 + with: + name: core-files-linux-aarch64 + path: core-files-linux-aarch64.tar.gz - name: Upload wheels to release if: startsWith(github.ref, 'refs/tags/v') run: | diff --git a/.github/workflows/build_linux_x86_wheels.yml b/.github/workflows/build_linux_x86_wheels.yml index a3d799d3679..8a7ba35f8d2 100644 --- a/.github/workflows/build_linux_x86_wheels.yml +++ b/.github/workflows/build_linux_x86_wheels.yml @@ -157,6 +157,11 @@ jobs: sudo rm -f dist/*-linux_x86_64.whl ls -lh dist shell: bash + - name: Setup core dump collection + run: | + mkdir -p tmp/core + echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern + ulimit -c unlimited - name: Test wheel on all Python versions run: | export PATH="$HOME/.pyenv/bin:$PATH" @@ -170,6 +175,25 @@ jobs: pyenv shell --unset done continue-on-error: false + - name: Check and upload core files if present + if: always() + run: | + if ls tmp/core/core.* >/dev/null 2>&1; then + echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV + tar -czvf core-files-linux-x86_64.tar.gz tmp/core/core.* + echo "Core files tar created: core-files-linux-x86_64.tar.gz" + ls -lh core-files-linux-x86_64.tar.gz + else + echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV + echo "No core files found in tmp/core" + fi + continue-on-error: true + - name: Upload core files artifact + if: always() && env.CORE_FILES_FOUND == 'true' + uses: actions/upload-artifact@v4 + with: + name: core-files-linux-x86_64 + path: core-files-linux-x86_64.tar.gz - name: Upload wheels to release if: startsWith(github.ref, 'refs/tags/v') run: | diff --git a/.github/workflows/build_macos_arm64_wheels.yml b/.github/workflows/build_macos_arm64_wheels.yml index f1cfbcb5222..4579a95d4d8 100644 --- a/.github/workflows/build_macos_arm64_wheels.yml +++ b/.github/workflows/build_macos_arm64_wheels.yml @@ -157,6 +157,12 @@ jobs: eval "$(pyenv init -)" pyenv shell 3.8 python -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/*.whl + - name: Setup core dump + run: | + mkdir -p tmp/core + echo "tmp/core/core.%p" > /tmp/core_pattern + sudo cp /tmp/core_pattern /proc/sys/kernel/core_pattern 2>/dev/null + ulimit -c unlimited - name: Test wheel on all Python versions run: | export PATH="$HOME/.pyenv/bin:$PATH" @@ -170,6 +176,25 @@ jobs: pyenv shell --unset done continue-on-error: false + - name: Check and upload core files if present + if: always() + run: | + if ls tmp/core/core.* >/dev/null 2>&1; then + echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV + tar -czvf core-files-macos-arm64.tar.gz tmp/core/core.* + echo "Core files tar created: core-files-macos-arm64.tar.gz" + ls -lh core-files-macos-arm64.tar.gz + else + echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV + echo "No core files found in tmp/core" + fi + continue-on-error: true + - name: Upload core files artifact + if: always() && env.CORE_FILES_FOUND == 'true' + uses: actions/upload-artifact@v4 + with: + name: core-files-macos-arm64 + path: core-files-macos-arm64.tar.gz - name: Show files run: ls -lh dist shell: bash diff --git a/.github/workflows/build_macos_x86_wheels.yml b/.github/workflows/build_macos_x86_wheels.yml index d1376a44228..4cdda7f44c9 100644 --- a/.github/workflows/build_macos_x86_wheels.yml +++ b/.github/workflows/build_macos_x86_wheels.yml @@ -157,6 +157,12 @@ jobs: eval "$(pyenv init -)" pyenv shell 3.8 python -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl + - name: Setup core dump collection + run: | + mkdir -p tmp/core + echo "tmp/core/core.%p" > /tmp/core_pattern + sudo cp /tmp/core_pattern /proc/sys/kernel/core_pattern 2>/dev/null + ulimit -c unlimited - name: Test wheel on all Python versions run: | export PATH="$HOME/.pyenv/bin:$PATH" @@ -170,6 +176,25 @@ jobs: pyenv shell --unset done continue-on-error: false + - name: Check and upload core files if present + if: always() + run: | + if ls tmp/core/core.* >/dev/null 2>&1; then + echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV + tar -czvf core-files-macos-x86_64.tar.gz tmp/core/core.* + echo "Core files tar created: core-files-macos-x86_64.tar.gz" + ls -lh core-files-macos-x86_64.tar.gz + else + echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV + echo "No core files found in tmp/core" + fi + continue-on-error: true + - name: Upload core files artifact + if: always() && env.CORE_FILES_FOUND == 'true' + uses: actions/upload-artifact@v4 + with: + name: core-files-macos-x86_64 + path: core-files-macos-x86_64.tar.gz - name: Show files run: ls -lh dist shell: bash From 6eaffb9bd6c8ca58434748d8cc6a0a75a344a956 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 3 Sep 2025 13:10:06 +0800 Subject: [PATCH 6/6] debug core upload on macos --- .github/workflows/build_macos_arm64_wheels.yml | 4 ++-- .github/workflows/build_macos_x86_wheels.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_macos_arm64_wheels.yml b/.github/workflows/build_macos_arm64_wheels.yml index 4579a95d4d8..13e70935898 100644 --- a/.github/workflows/build_macos_arm64_wheels.yml +++ b/.github/workflows/build_macos_arm64_wheels.yml @@ -160,8 +160,8 @@ jobs: - name: Setup core dump run: | mkdir -p tmp/core - echo "tmp/core/core.%p" > /tmp/core_pattern - sudo cp /tmp/core_pattern /proc/sys/kernel/core_pattern 2>/dev/null + sudo sysctl kern.corefile=$PWD/tmp/core/core.%P + sudo sysctl kern.coredump=1 ulimit -c unlimited - name: Test wheel on all Python versions run: | diff --git a/.github/workflows/build_macos_x86_wheels.yml b/.github/workflows/build_macos_x86_wheels.yml index 4cdda7f44c9..bda715ea6a3 100644 --- a/.github/workflows/build_macos_x86_wheels.yml +++ b/.github/workflows/build_macos_x86_wheels.yml @@ -160,8 +160,8 @@ jobs: - name: Setup core dump collection run: | mkdir -p tmp/core - echo "tmp/core/core.%p" > /tmp/core_pattern - sudo cp /tmp/core_pattern /proc/sys/kernel/core_pattern 2>/dev/null + sudo sysctl kern.corefile=$PWD/tmp/core/core.%P + sudo sysctl kern.coredump=1 ulimit -c unlimited - name: Test wheel on all Python versions run: |