From d2ac198bb3d79a0f3bd769faaf3e6bc6c44adcdf Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 13:40:10 -0400 Subject: [PATCH 01/91] Update yscope-dev-utils --- tools/yscope-dev-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index e300d1ba..5c6bfbd0 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit e300d1bab4c2f33cbf9ddf9f0c08185faf035070 +Subproject commit 5c6bfbd00ad4e9cbfdfab9708298fb11bda4ae8c From cc8a0977836682e8ed9aa3e39aa3268ffbb2447c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 13:44:03 -0400 Subject: [PATCH 02/91] Use boost install task --- dep-tasks.yaml | 160 +------------------------------------------------ 1 file changed, 1 insertion(+), 159 deletions(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index a451ddc1..324a7124 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -147,7 +147,7 @@ tasks: internal: true run: "once" cmds: - - task: "boost-download-and-install" + - task: ":utils:boost:download-and-install" vars: WORK_DIR: "{{.G_DEPS_DIR}}/boost" FILE_SHA256: "2128a4c96862b5c0970c1e34d76b1d57e4a1016b80df85ad39667f30b1deba26" @@ -161,161 +161,3 @@ tasks: - "program_options" - "regex" - "system" - - # Runs the bootstrap.sh generate step in the given source directory. Boost only supports - # in-source generation and building. - # - # @param {string} SOURCE_DIR Project source directory. - # @param {string} INSTALL_PREFIX Path prefix of where the project should be installed. - # @param {string[]} TARGETS Target libraries to build. - # @param {string[]} [EXTRA_ARGS] Any additional arguments to pass to the generate command. - boost-generate: - internal: true - dir: "{{.SOURCE_DIR}}" - cmds: - - >- - ./bootstrap.sh - --prefix="{{.INSTALL_PREFIX}}" - --exec-prefix="{{.INSTALL_PREFIX}}" - --with-libraries={{(join "," .TARGETS)}} - {{- range .EXTRA_ARGS}} - "{{.}}" - {{- end}} - - # Runs the b2 build step for boost. The caller must have previously called `generate` on - # `SOURCE_DIR` for this task to succeed. - # - # @param {string} SOURCE_DIR Directory containing the boost source. - # @param {string[]} [EXTRA_ARGS] Any additional arguments to pass to the build command. - # @param {int} [JOBS] The maximum number of concurrent processes to use when building. If - # omitted, the b2 default number is used. Before 1.76.0, the number was 1. Since 1.76.0, the - # default is the number of cores. - boost-build: - internal: true - dir: "{{.SOURCE_DIR}}" - cmds: - - >- - ./b2 - {{- range .EXTRA_ARGS}} - "{{.}}" - {{- end}} - {{- if .JOBS}} - "-j{{.JOBS}}" - {{- end}} - - # Runs the b2 install step for boost. The caller must have previously called `build` on - # `SOURCE_DIR` for this task to succeed. If `CMAKE_SETTINGS_DIR` is set, a settings file will be - # created in that directory, containing a `boost_ROOT` CMake variable that points to - # `INSTALL_PREFIX`. - # - # @param {string} SOURCE_DIR Directory containing the boost source. - # @param {string} INSTALL_PREFIX Path prefix of where the project should be installed. - # @param {string} [CMAKE_SETTINGS_DIR] If set, the directory where the project's CMake settings - # file should be stored. - # @param {string[]} [EXTRA_ARGS] Any additional arguments to pass to the install command. - boost-install: - internal: true - dir: "{{.SOURCE_DIR}}" - cmds: - - >- - ./b2 - install - {{- range .EXTRA_ARGS}} - "{{.}}" - {{- end}} - - >- - {{- if .CMAKE_SETTINGS_DIR}} - echo "set(BOOST_ROOT - \"{{.INSTALL_PREFIX}}\" - CACHE PATH - \"Package root for boost.\" - )" >> "{{.CMAKE_SETTINGS_DIR}}/boost.cmake" - {{- end}} - - # Downloads boost from `URL` and installs boost. - # - # General parameters - # @param {string} [WORK_DIR={{.TASK_DIR}}] Base directory to store the install and src - # directories inside. - # @param {string} [SOURCE_DIR={{.WORK_DIR}}/boost-src] Directory in which to extract the tar - # file. - # - # Download parameters - # @param {string} FILE_SHA256 Content hash to verify the downloaded tar file against. - # @param {string} URL - # - # Boost generate parameters - # @param {string} [INSTALL_PREFIX={{.WORK_DIR}}/boost-install] Path prefix of where the project - # should be installed. - # @param {string[]} TARGETS Target libraries to build. - # @param {string[]} [GEN_ARGS] Any additional arguments to pass to the generate command. - # - # Boost build parameters - # @param {int} [JOBS] The maximum number of concurrent processes to use when building. If - # omitted, the b2 default number is used. Before 1.76.0, the number was 1. Since 1.76.0, the - # default is the number of cores. - # @param {string[]} [BUILD_ARGS] Any additional arguments to pass to the build command. - # - # Boost install parameters - # @param {string[]} [INSTALL_ARGS] Any additional arguments to pass to the install command. - # @param {string} [CMAKE_SETTINGS_DIR] If set, the directory where the project's CMake settings - # file should be stored. - boost-download-and-install: - internal: true - label: "{{.TASK}}:{{.URL}}-{{.INSTALL_PREFIX}}" - vars: - # General parameters - WORK_DIR: >- - {{default .ROOT_DIR .WORK_DIR}} - SOURCE_DIR: >- - {{default (printf "%s/boost-src" .WORK_DIR) .SOURCE_DIR}} - - # Boost generate parameters - INSTALL_PREFIX: >- - {{default (printf "%s/boost-install" .WORK_DIR) .INSTALL_PREFIX}} - TARGETS: - ref: "default (list) .TARGETS" - GEN_ARGS: - ref: "default (list) .GEN_ARGS" - - # Boost build parameters - BUILD_ARGS: - ref: "default (list) .BUILD_ARGS" - JOBS: >- - {{default "" .JOBS}} - - # Boost install parameters - INSTALL_ARGS: - ref: "default (list) .INSTALL_ARGS" - CMAKE_SETTINGS_DIR: >- - {{default "" .CMAKE_SETTINGS_DIR}} - requires: - vars: ["FILE_SHA256", "URL"] - deps: - - task: ":utils:remote:download-and-extract-tar" - vars: - FILE_SHA256: "{{.FILE_SHA256}}" - OUTPUT_DIR: "{{.SOURCE_DIR}}" - URL: "{{.URL}}" - cmds: - - task: "boost-generate" - vars: - SOURCE_DIR: "{{.SOURCE_DIR}}" - INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" - TARGETS: - ref: ".TARGETS" - EXTRA_ARGS: - ref: ".GEN_ARGS" - - task: "boost-build" - vars: - SOURCE_DIR: "{{.SOURCE_DIR}}" - JOBS: "{{.JOBS}}" - EXTRA_ARGS: - ref: ".BUILD_ARGS" - - task: "boost-install" - vars: - SOURCE_DIR: "{{.SOURCE_DIR}}" - INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" - CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" - EXTRA_ARGS: - ref: ".INSTALL_ARGS" From 7f68bdc0d8ec884eaa14096282b680a531d482a2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 14:00:23 -0400 Subject: [PATCH 03/91] Update install task variable names --- dep-tasks.yaml | 52 +++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 324a7124..3c7c05f8 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -30,12 +30,12 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "absl" + CMAKE_PACKAGE_NAME: "absl" WORK_DIR: "{{.G_DEPS_DIR}}/absl" - FILE_SHA256: "b396401fd29e2e679cace77867481d388c807671dc2acc602a0259eeb79b7811" - URL: "https://github.com/abseil/abseil-cpp/archive/refs/tags/20250127.1.tar.gz" + TAR_SHA256: "b396401fd29e2e679cace77867481d388c807671dc2acc602a0259eeb79b7811" + TAR_URL: "https://github.com/abseil/abseil-cpp/archive/refs/tags/20250127.1.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" @@ -45,12 +45,12 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "Catch2" + CMAKE_PACKAGE_NAME: "Catch2" WORK_DIR: "{{.G_DEPS_DIR}}/Catch2" - FILE_SHA256: "1ab2de20460d4641553addfdfe6acd4109d871d5531f8f519a52ea4926303087" - URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v3.8.0.tar.gz" + TAR_SHA256: "1ab2de20460d4641553addfdfe6acd4109d871d5531f8f519a52ea4926303087" + TAR_URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v3.8.0.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" @@ -76,12 +76,12 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "fmt" + CMAKE_PACKAGE_NAME: "fmt" WORK_DIR: "{{.G_DEPS_DIR}}/fmtlib" - FILE_SHA256: "6cb1e6d37bdcb756dbbe59be438790db409cdb4868c66e888d5df9f13f7c027f" - URL: "https://github.com/fmtlib/fmt/archive/refs/tags/11.0.2.tar.gz" + TAR_SHA256: "6cb1e6d37bdcb756dbbe59be438790db409cdb4868c66e888d5df9f13f7c027f" + TAR_URL: "https://github.com/fmtlib/fmt/archive/refs/tags/11.0.2.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" @@ -95,16 +95,16 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "spdlog" + CMAKE_PACKAGE_NAME: "spdlog" WORK_DIR: "{{.G_DEPS_DIR}}/spdlog" - FILE_SHA256: "9962648c9b4f1a7bbc76fd8d9172555bad1871fdb14ff4f842ef87949682caa5" - URL: "https://github.com/gabime/spdlog/archive/refs/tags/v1.15.0.tar.gz" + TAR_SHA256: "9962648c9b4f1a7bbc76fd8d9172555bad1871fdb14ff4f842ef87949682caa5" + TAR_URL: "https://github.com/gabime/spdlog/archive/refs/tags/v1.15.0.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DSPDLOG_BUILD_TESTS=OFF" - "-DUSE_EXTERNAL_FMT=ON" - - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" + - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" install-mariadb-connector-cpp: @@ -113,13 +113,13 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "mariadb-connector-cpp" + CMAKE_PACKAGE_NAME: "mariadb-connector-cpp" WORK_DIR: "{{.G_DEPS_DIR}}/mariadb-connector-cpp" - FILE_SHA256: "0e3dfe9f2bc3f7bb6f7c159009556290064a7c23402ea08019fa8aebfc3ff2c9" - URL: "https://github.com/mariadb-corporation/mariadb-connector-cpp/archive/refs/tags/\ + TAR_SHA256: "0e3dfe9f2bc3f7bb6f7c159009556290064a7c23402ea08019fa8aebfc3ff2c9" + TAR_URL: "https://github.com/mariadb-corporation/mariadb-connector-cpp/archive/refs/tags/\ 1.1.5.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" @@ -132,16 +132,16 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "msgpack-cxx" + CMAKE_PACKAGE_NAME: "msgpack-cxx" WORK_DIR: "{{.G_DEPS_DIR}}/msgpack" - FILE_SHA256: "7504b7af7e7b9002ce529d4f941e1b7fb1fb435768780ce7da4abaac79bb156f" - URL: "https://github.com/msgpack/msgpack-c/releases/download/\ + TAR_SHA256: "7504b7af7e7b9002ce529d4f941e1b7fb1fb435768780ce7da4abaac79bb156f" + TAR_URL: "https://github.com/msgpack/msgpack-c/releases/download/\ cpp-7.0.0/msgpack-cxx-7.0.0.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" + - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" install-boost: internal: true From 2394649ab8c6454733ff34d8736fb1b8960d2022 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 14:11:02 -0400 Subject: [PATCH 04/91] Set CMP0074 to NEW to find boost --- dep-tasks.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 3c7c05f8..d58a61f0 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -105,6 +105,7 @@ tasks: - "-DSPDLOG_BUILD_TESTS=OFF" - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" + - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" install-mariadb-connector-cpp: @@ -142,6 +143,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" + - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" install-boost: internal: true From 6ff3e33f9d5b36e3b54be01fbf6b263b4c9ca6a9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 14:40:52 -0400 Subject: [PATCH 05/91] Add uv to install script --- tools/scripts/lib_install/linux/install-dev.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/scripts/lib_install/linux/install-dev.sh b/tools/scripts/lib_install/linux/install-dev.sh index 73fd319b..f88c9cc9 100755 --- a/tools/scripts/lib_install/linux/install-dev.sh +++ b/tools/scripts/lib_install/linux/install-dev.sh @@ -36,3 +36,6 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" lib_install_scripts_dir="$script_dir/.." # TODO https://github.com/y-scope/spider/issues/86 "$lib_install_scripts_dir"/check-cmake-version.sh + +# Install uv +curl -LsSf https://astral.sh/uv/install.sh | sh From 7ae8a782aaf63043f4f1c4b31a47f3dc57bfb346 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 15:04:40 -0400 Subject: [PATCH 06/91] Fix cpp-lint root paths --- lint-tasks.yaml | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index d81b7247..85417f30 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -45,7 +45,7 @@ tasks: - task: "cpp-static-fix" cpp-format-check: - sources: &cpp_source_files + sources: &cpp_lint_source_files - "{{.ROOT_DIR}}/.clang-format" - "{{.ROOT_DIR}}/.clang-tidy" - "{{.TASKFILE}}" @@ -59,11 +59,14 @@ tasks: vars: FLAGS: ["--dry-run"] INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] - ROOT_PATHS: *cpp_source_files + ROOT_PATHS: &cpp_source_files + - "{{.G_SRC_SPIDER_DIR}}" + - "{{.G_EXAMPLES_DIR}}" + - "{{.G_TEST_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" cpp-format-fix: - sources: *cpp_source_files + sources: *cpp_lint_source_files deps: ["cpp-configs", "venv"] cmds: - task: ":utils:cpp-lint:clang-format" @@ -152,18 +155,6 @@ tasks: taskfile.yaml \ test-tasks.yaml - clang-tidy: - internal: true - requires: - vars: ["FLAGS", "SRC_DIR"] - cmd: |- - . "{{.G_LINT_VENV_DIR}}/bin/activate" - find "{{.SRC_DIR}}" \ - -type f \ - \( -iname "*.cpp" -o -iname "*.h" -o -iname "*.hpp" \) \ - -print0 | \ - xargs -0 --no-run-if-empty clang-tidy {{.FLAGS}} - cmake: internal: true requires: From 56502f90d6df56d5e0d97ed0f872b7682743455b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 15:16:33 -0400 Subject: [PATCH 07/91] Fix clang-tidy file pattern --- lint-tasks.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 85417f30..659480e5 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -82,7 +82,7 @@ tasks: # When we eventually determine which errors can be safely fixed, we'll allow clang-tidy to # fix them. aliases: ["cpp-static-fix"] - sources: *cpp_source_files + sources: *cpp_lint_source_files deps: [":config-cmake-project", "cpp-configs", "venv"] cmds: - task: ":utils:cpp-lint:clang-tidy-find" @@ -90,21 +90,21 @@ tasks: FLAGS: - "--config-file '{{.ROOT_DIR}}/.clang-tidy'" - "-p '{{.G_SPIDER_COMPILE_COMMANDS_DB}}'" - INCLUDE_PATTERNS: - - "{{.G_SRC_SPIDER_DIR}}/**" - - "{{.G_TEST_DIR}}/**" + INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" - ROOT_PATHS: *cpp_source_files + ROOT_PATHS: + - "{{.G_SRC_SPIDER_DIR}}" + - "{{.G_TEST_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" - task: ":utils:cpp-lint:clang-tidy-find" vars: FLAGS: - "--config-file '{{.ROOT_DIR}}/.clang-tidy'" - "-p '{{.G_EXAMPLES_COMPILE_COMMANDS_DB}}'" - INCLUDE_PATTERNS: - - "{{.G_EXAMPLES_DIR}}/**" + INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" - ROOT_PATHS: *cpp_source_files + ROOT_PATHS: + - "{{.G_EXAMPLES_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" py-check: From 6f6becfff0e90960d46fe07ce442012de70150b1 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:19:38 -0400 Subject: [PATCH 08/91] Limit build parallelism --- .github/workflows/code-linting-checks.yaml | 2 ++ .github/workflows/unit-tests.yaml | 2 ++ dep-tasks.yaml | 6 ++++++ taskfile.yaml | 2 ++ 4 files changed, 12 insertions(+) diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index c6b1e911..4ee4e58e 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -47,6 +47,8 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 + env: + SPIDER_DEP_BUILD_PARALLELISM: "1" run: "task deps:lib_install" - run: "task lint:check -C $(nproc)" diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 4b42a93c..0e6089d3 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -50,6 +50,8 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 + env: + SPIDER_DEP_BUILD_PARALLELISM: "1" run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" diff --git a/dep-tasks.yaml b/dep-tasks.yaml index a451ddc1..9652125e 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,6 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-Catch2: internal: true @@ -53,6 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} download-ystdlib: internal: true @@ -85,6 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-spdlog: internal: true @@ -106,6 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-mariadb-connector-cpp: internal: true @@ -123,6 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-msgpack: internal: true @@ -142,6 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-boost: internal: true diff --git a/taskfile.yaml b/taskfile.yaml index 556ef348..ecece7a0 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,6 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + G_DEP_BUILD_PARALLELISM: >- + {{default "" (env SPIDER_DEP_BUILD_PARALLELISM)}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From a74523daa93aef57b0d279863583f7b3b4a7240b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:34:32 -0400 Subject: [PATCH 09/91] Bug fix --- dep-tasks.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 9652125e..9262da10 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,7 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-Catch2: internal: true @@ -54,7 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" download-ystdlib: internal: true @@ -87,7 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-spdlog: internal: true @@ -109,7 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-mariadb-connector-cpp: internal: true @@ -127,7 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-msgpack: internal: true @@ -147,7 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-boost: internal: true From 77ee49460e2dfb7752f4416b0b595a16ced5fc16 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:37:22 -0400 Subject: [PATCH 10/91] Bug fix --- taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfile.yaml b/taskfile.yaml index ecece7a0..4a110c06 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -21,7 +21,7 @@ vars: G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" G_DEP_BUILD_PARALLELISM: >- - {{default "" (env SPIDER_DEP_BUILD_PARALLELISM)}} + {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From 3698339529129c883a0d1d927a873e2b879b4dce Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:47:19 -0400 Subject: [PATCH 11/91] Bug fix --- dep-tasks.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 4f45bd44..fe90b354 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -169,4 +169,3 @@ tasks: - "program_options" - "regex" - "system" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" From 369e9f1bbf876d7ef1b1ed209f953681d5f83211 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 22:57:18 -0400 Subject: [PATCH 12/91] Rename variables to mirror CLP core Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- taskfile.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/taskfile.yaml b/taskfile.yaml index 4a110c06..d44c1a46 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,8 +20,11 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + # Build parameters + # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. G_DEP_BUILD_PARALLELISM: >- {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} + G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From 90aa5a285b7068d245c52663dc82cbac3063ee67 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 22:57:45 -0400 Subject: [PATCH 13/91] Rename variables to mirror clp core --- .github/workflows/code-linting-checks.yaml | 2 +- .github/workflows/unit-tests.yaml | 2 +- dep-tasks.yaml | 12 ++++++------ taskfile.yaml | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index 4ee4e58e..48f8eacd 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -48,7 +48,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_BUILD_PARALLELISM: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task lint:check -C $(nproc)" diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 0e6089d3..1ced6a88 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -51,7 +51,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_BUILD_PARALLELISM: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 9262da10..cab98d77 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,7 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-Catch2: internal: true @@ -54,7 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" download-ystdlib: internal: true @@ -87,7 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-spdlog: internal: true @@ -109,7 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-mariadb-connector-cpp: internal: true @@ -127,7 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-msgpack: internal: true @@ -147,7 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-boost: internal: true diff --git a/taskfile.yaml b/taskfile.yaml index 4a110c06..20da0f3b 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,8 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" - G_DEP_BUILD_PARALLELISM: >- - {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} + G_DEPS_MAX_PARALLELISM_PER_TASK: >- + {{default "" (env "SPIDER_DEPS_MAX_PARALLELISM_PER_TASK")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From 7faac8fc6a1e2b196abb0853b23c8ea3153cdeb9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 23:07:53 -0400 Subject: [PATCH 14/91] Revert "Merge branch 'dep-concurrency' of github.com:sitaowang1998/spider into dep-concurrency" This reverts commit 1769c95feb3868e4a462ccaa694b97f880947f12, reversing changes made to 90aa5a285b7068d245c52663dc82cbac3063ee67. --- .github/workflows/code-linting-checks.yaml | 2 +- .github/workflows/unit-tests.yaml | 2 +- dep-tasks.yaml | 12 ++++++------ taskfile.yaml | 7 ++----- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index 1178f059..48f8eacd 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -48,7 +48,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_MAX_PARALLELISM_PER_TASK: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task lint:check -C $(nproc)" diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 675e7c6e..1ced6a88 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -51,7 +51,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_MAX_PARALLELISM_PER_TASK: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" diff --git a/dep-tasks.yaml b/dep-tasks.yaml index ea282d43..cab98d77 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,7 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-Catch2: internal: true @@ -54,7 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" download-ystdlib: internal: true @@ -87,7 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-spdlog: internal: true @@ -109,7 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-mariadb-connector-cpp: internal: true @@ -127,7 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-msgpack: internal: true @@ -147,7 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-boost: internal: true diff --git a/taskfile.yaml b/taskfile.yaml index d44c1a46..20da0f3b 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,11 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" - # Build parameters - # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. - G_DEP_BUILD_PARALLELISM: >- - {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} - + G_DEPS_MAX_PARALLELISM_PER_TASK: >- + {{default "" (env "SPIDER_DEPS_MAX_PARALLELISM_PER_TASK")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From f494a90b85e24d4e9ee07875e01e7cdb65e5387c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 23:10:25 -0400 Subject: [PATCH 15/91] Add comment for deps parallelism default value --- taskfile.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskfile.yaml b/taskfile.yaml index 20da0f3b..470edd58 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,6 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + # Build parameters + # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. G_DEPS_MAX_PARALLELISM_PER_TASK: >- {{default "" (env "SPIDER_DEPS_MAX_PARALLELISM_PER_TASK")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" From ff2fe1c69a7ff4d952d546babba27413dcb0370f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 23:28:39 -0400 Subject: [PATCH 16/91] Update yscope-dev-utils --- tools/yscope-dev-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index 5c6bfbd0..b965f19f 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit 5c6bfbd00ad4e9cbfdfab9708298fb11bda4ae8c +Subproject commit b965f19f3e11c548f32bf7ab8e28ff38906be018 From 65841a019afaaf7aaaf56ef5b516743a482f9861 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 10:31:24 -0400 Subject: [PATCH 17/91] Add latest python lint config files --- mypy.ini | 8 ++++++++ ruff.toml | 29 +++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 mypy.ini diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..4c66b111 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +strict = true + +# Additional output +pretty = true +show_error_code_links = true +show_error_context = true +show_error_end = true diff --git a/ruff.toml b/ruff.toml index 969421ff..cb6ee887 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,6 +1,27 @@ line-length = 100 -lint.select = ["I"] -[lint.isort] -case-sensitive = false -order-by-type = false \ No newline at end of file +[lint] +select = ["ALL"] +ignore = [ + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful +] +isort.order-by-type = false + +[format] +docstring-code-format = true +docstring-code-line-length = 100 From 8564fc259a54fdbe08ec27b39f33dc0ca3e4f593 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 11:03:48 -0400 Subject: [PATCH 18/91] Update ruff lint tasks --- lint-requirements.txt | 1 - lint-tasks.yaml | 16 +++++++--------- mypy.ini | 8 -------- 3 files changed, 7 insertions(+), 18 deletions(-) delete mode 100644 mypy.ini diff --git a/lint-requirements.txt b/lint-requirements.txt index 717adb50..690b9797 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,4 +1,3 @@ -black>=24.4.2 clang-format>=20.1.0 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 659480e5..04f8cef3 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -111,29 +111,27 @@ tasks: cmds: - task: "py" vars: - BLACK_FLAGS: "--check" - RUFF_FLAGS: "" + RUFF_FORMAT_FLAGS: "--diff" py-fix: cmds: - task: "py" vars: - BLACK_FLAGS: "" - RUFF_FLAGS: "--fix" + RUFF_CHECK_FLAGS: "--fix" py: internal: true - requires: - vars: ["BLACK_FLAGS", "RUFF_FLAGS"] + vars: + RUFF_CHECK_FLAGS: "{{.RUFF_CHECK_FLAGS | default \"\"}}" + RUFF_FORMAT_FLAGS: "{{.RUFF_FORMAT_FLAGS | default \"\"}}" deps: ["venv"] cmds: - for: - "tests/integration" cmd: |- . "{{.G_LINT_VENV_DIR}}/bin/activate" - cd "{{.ITEM}}" - black --color --line-length 100 {{.BLACK_FLAGS}} . - ruff check {{.RUFF_FLAGS}} . + ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}" + ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}" yml: aliases: diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 4c66b111..00000000 --- a/mypy.ini +++ /dev/null @@ -1,8 +0,0 @@ -[mypy] -strict = true - -# Additional output -pretty = true -show_error_code_links = true -show_error_context = true -show_error_end = true From a6e7d29a435f66eeac173a1a28415c7d1ab3c820 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:13:55 -0400 Subject: [PATCH 19/91] Fix ruff lint --- ruff.toml | 8 ++ tests/integration/__init__.py | 1 + tests/integration/client.py | 118 +++++++++++++++++---- tests/integration/test_client.py | 27 ++++- tests/integration/test_scheduler_worker.py | 85 +++++++++++++-- tests/integration/test_signal.py | 52 ++++++--- 6 files changed, 244 insertions(+), 47 deletions(-) diff --git a/ruff.toml b/ruff.toml index cb6ee887..a7a590d3 100644 --- a/ruff.toml +++ b/ruff.toml @@ -2,6 +2,7 @@ line-length = 100 [lint] select = ["ALL"] +extend-select = ["PT"] ignore = [ "COM812", # Redundant and conflicts with ruff format "D203", # No blank line before docstrings (D211) @@ -22,6 +23,13 @@ ignore = [ ] isort.order-by-type = false +[lint.per-file-ignores] +"tests/integration/test_*.py" = [ + "S101", # Allow use of `assert` (security warning) + "ANN001", # Ignore missing type annotations for arguments + "ANN201", # Ignore missing type annotations for parameters +] + [format] docstring-code-format = true docstring-code-line-length = 100 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index e69de29b..4d2ed726 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -0,0 +1 @@ +"""Integration tests for Spider C++ projects.""" diff --git a/tests/integration/client.py b/tests/integration/client.py index 1b658c5a..834ae2b5 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -1,7 +1,7 @@ import re import uuid from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple +from collections.abc import Generator import mysql.connector import pytest @@ -9,48 +9,73 @@ @dataclass class TaskInput: + """ + TaskInput represents an input to a task. + It can either be a direct value, a reference to another task's output, or a reference to data. + """ type: str - task_output: Optional[Tuple[uuid.UUID, int]] = None - value: Optional[str] = None - data_id: Optional[uuid.UUID] = None + task_output: tuple[uuid.UUID, int] | None = None + value: str | None = None + data_id: uuid.UUID | None = None @dataclass class TaskOutput: + """ + TaskOutput represents an output of a task. + It can either be a direct value or a reference to data. + """ type: str - value: Optional[str] = None - data_id: Optional[uuid.UUID] = None + value: str | None = None + data_id: uuid.UUID | None = None @dataclass class Task: + """ + Task represents a unit of work in the task graph. + """ id: uuid.UUID function_name: str - inputs: List[TaskInput] - outputs: List[TaskOutput] + inputs: list[TaskInput] + outputs: list[TaskOutput] timeout: float = 0.0 max_retries: int = 0 @dataclass class TaskGraph: + """ + TaskGraph represents a directed acyclic graph of tasks. + """ id: uuid.UUID - tasks: Dict[uuid.UUID, Task] - dependencies: List[Tuple[uuid.UUID, uuid.UUID]] + tasks: dict[uuid.UUID, Task] + dependencies: list[tuple[uuid.UUID, uuid.UUID]] @dataclass class Driver: + """ + Driver represents a client that can submit jobs to the task graph. + """ id: uuid.UUID @dataclass class Data: + """ + Data represents a Spider Data object. + """ id: uuid.UUID value: str -def create_connection(storage_url: str): +def create_connection(storage_url: str) -> mysql.connector.MySQLConnection: + """ + Creation a MariaDB connection from a JDBC URL. + :param storage_url: JDBC URL for the MariaDB database. + :return: The created MySQL connection. + """ pattern = re.compile( r"jdbc:mariadb://(?P[^:/]+):(?P\d+)/(?P[^?]+)\?user=(?P[^&]+)&password=(?P[^&]+)" ) @@ -68,7 +93,13 @@ def create_connection(storage_url: str): ) -def is_head_task(task_id: uuid.UUID, dependencies: List[Tuple[uuid.UUID, uuid.UUID]]): +def is_head_task(task_id: uuid.UUID, dependencies: list[tuple[uuid.UUID, uuid.UUID]]) -> bool: + """ + Check if the task is a head task, meaning it has no parent. + :param task_id: the ID of the task to check. + :param dependencies: list of dependencies where each dependency is a tuple (parent_id, child_id). + :return: True if the task has no parent, False otherwise. + """ return not any(dependency[1] == task_id for dependency in dependencies) @@ -76,13 +107,25 @@ def is_head_task(task_id: uuid.UUID, dependencies: List[Tuple[uuid.UUID, uuid.UU @pytest.fixture(scope="session") -def storage(): +def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: + """ + Fixture to create a database connection for the test session. Yields a connection object + and ensures it is closed after the tests are done. + :return: + """ conn = create_connection(g_storage_url) yield conn conn.close() -def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph): +def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: + """ + Submit a job to the database. + :param conn: database connection object. + :param client_id: client ID of the driver submitting the job. + :param graph: task graph to be submitted. + :return: None + """ cursor = conn.cursor() cursor.execute( @@ -136,7 +179,13 @@ def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph): cursor.close() -def get_task_outputs(conn, task_id: uuid.UUID) -> List[TaskOutput]: +def get_task_outputs(conn, task_id: uuid.UUID) -> list[TaskOutput]: + """ + Get the outputs of a task by its ID. + :param conn: database connection object. + :param task_id: the ID of the task whose outputs are to be retrieved. + :return: list of TaskOutput objects representing the outputs of the task. + """ cursor = conn.cursor() cursor.execute( @@ -158,6 +207,12 @@ def get_task_outputs(conn, task_id: uuid.UUID) -> List[TaskOutput]: def get_task_state(conn, task_id: uuid.UUID) -> str: + """ + Get the state of a task by its ID. + :param conn: database connection object. + :param task_id: the ID of the task whose state is to be retrieved. + :return: the state of the task as a string. + """ cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) @@ -168,7 +223,13 @@ def get_task_state(conn, task_id: uuid.UUID) -> str: return state -def remove_job(conn, job_id: uuid.UUID): +def remove_job(conn, job_id: uuid.UUID) -> None: + """ + Remove a job from the database by its ID. + :param conn: database connection object. + :param job_id: the ID of the job to be removed. + :return: None + """ cursor = conn.cursor() cursor.execute("DELETE FROM jobs WHERE id = %s", (job_id.bytes,)) @@ -176,7 +237,13 @@ def remove_job(conn, job_id: uuid.UUID): cursor.close() -def add_driver(conn, driver: Driver): +def add_driver(conn, driver: Driver) -> None: + """ + Register a new driver in the database. + :param conn: database connection object. + :param driver: driver object to be registered. + :return: None + """ cursor = conn.cursor() cursor.execute("INSERT INTO drivers (id) VALUES (%s)", (driver.id.bytes,)) @@ -185,7 +252,14 @@ def add_driver(conn, driver: Driver): cursor.close() -def add_driver_data(conn, driver: Driver, data: Data): +def add_driver_data(conn, driver: Driver, data: Data) -> None: + """ + Add a new data associated with a driver in the database. + :param conn: database connection object. + :param driver: driver object to which the data is associated. + :param data: data object to be added. + :return: None + """ cursor = conn.cursor() cursor.execute("INSERT INTO data (id, value) VALUES (%s, %s)", (data.id.bytes, data.value)) @@ -198,7 +272,13 @@ def add_driver_data(conn, driver: Driver, data: Data): cursor.close() -def remove_data(conn, data: Data): +def remove_data(conn, data: Data) -> None: + """ + Remove data from the database by its ID. + :param conn: database connection object. + :param data: data object to be removed. + :return: None + """ cursor = conn.cursor() cursor.execute("DELETE FROM data WHERE id = %s", (data.id.bytes,)) diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 94b48afd..7d6edeeb 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -1,7 +1,7 @@ import subprocess import time from pathlib import Path -from typing import Tuple +from collections.abc import Generator import pytest @@ -14,7 +14,13 @@ def start_scheduler_workers( storage_url: str, scheduler_port: int -) -> Tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen]: + """ + Start the scheduler and two worker processes. + :param storage_url: + :param scheduler_port: The port for the scheduler to listen on. + :return: scheduler_process, worker_process_0, worker_process_1 + """ # Start the scheduler dir_path = Path(__file__).resolve().parent dir_path = dir_path / ".." / ".." / "src" / "spider" @@ -43,7 +49,13 @@ def start_scheduler_workers( @pytest.fixture(scope="class") -def scheduler_worker(storage): +@pytest.mark.usefixtures("storage") +def scheduler_worker() -> Generator[None, None, None]: + """ + Fixture to start the scheduler and two worker processes. Yields control to the test class, + and then kills the processes after the test class is done. + :return: + """ scheduler_process, worker_process_0, worker_process_1 = start_scheduler_workers( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) @@ -56,7 +68,12 @@ def scheduler_worker(storage): class TestClient: - def test_client(self, scheduler_worker): + @pytest.mark.usefixtures("scheduler_worker") + def test_client(self) -> None: + """ + Test the client_test C++ program and check for successful execution. + :return: None + """ dir_path = Path(__file__).resolve().parent dir_path = dir_path / ".." client_cmds = [ @@ -64,5 +81,5 @@ def test_client(self, scheduler_worker): "--storage_url", g_storage_url, ] - p = subprocess.run(client_cmds, timeout=20) + p = subprocess.run(client_cmds, check=False, timeout=20) assert p.returncode == 0 diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 9dfc359d..f138f9f8 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -2,11 +2,12 @@ import time import uuid from pathlib import Path -from typing import Tuple +from collections.abc import Generator import msgpack import pytest +from tests.integration.client import TaskGraph, Task from .client import ( add_driver, add_driver_data, @@ -29,7 +30,13 @@ def start_scheduler_worker( storage_url: str, scheduler_port: int -) -> Tuple[subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen, subprocess.Popen]: + """ + Start a scheduler and a worker process. + :param storage_url: JDBC storage URL + :param scheduler_port: the port for the scheduler to listen on + :return: scheduler_process, worker_process + """ # Start the scheduler dir_path = Path(__file__).resolve().parent dir_path = dir_path / ".." / ".." / "src" / "spider" @@ -57,7 +64,13 @@ def start_scheduler_worker( @pytest.fixture(scope="class") -def scheduler_worker(storage): +@pytest.mark.usefixtures("storage") +def scheduler_worker() -> Generator[None, None, None]: + """ + Fixture to start a scheduler and a worker process. Yields control to the test function. + After the test function completes, it kills the scheduler and the worker process. + :return: + """ scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) @@ -69,7 +82,13 @@ def scheduler_worker(storage): @pytest.fixture(scope="function") -def success_job(storage): +def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: + """ + Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. + Cleans up the job after the test function completes. + :param storage: + :return: + """ parent_1 = Task( id=uuid.uuid4(), function_name="sum_test", @@ -127,7 +146,13 @@ def success_job(storage): @pytest.fixture(scope="function") -def fail_job(storage): +def fail_job(storage) -> Generator[Task, None, None]: + """ + Fixture to create a job that will fail. The task will raise an error when executed. + Yield the task. Cleanup the job after the test function completes. + :param storage: + :return: + """ task = Task( id=uuid.uuid4(), function_name="error_test", @@ -149,7 +174,13 @@ def fail_job(storage): @pytest.fixture(scope="function") -def data_job(storage): +def data_job(storage) -> Generator[Task, None, None]: + """ + Fixture to create a job that uses data. Yields the task that uses data. + Cleans up the job and data after the test function completes. + :param storage: + :return: + """ data = Data( id=uuid.uuid4(), value=msgpack.packb(2), @@ -181,6 +212,12 @@ def data_job(storage): @pytest.fixture(scope="function") def random_fail_job(storage): + """ + Fixture to create a job that randomly fails. The task will succeed after a few retries. + Yields the task. Cleans up the job after the test function completes. + :param storage: + :return: + """ data = Data( id=uuid.uuid4(), value=msgpack.packb(2), @@ -212,7 +249,14 @@ def random_fail_job(storage): class TestSchedulerWorker: - def test_job_success(self, scheduler_worker, storage, success_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_job_success(self, storage, success_job): + """ + Test the successful execution of a job with two parent tasks and one child task. + :param storage: + :param success_job: + :return: None + """ graph, parent_1, parent_2, child = success_job # Wait for 2 seconds and check task state and output time.sleep(2) @@ -232,14 +276,28 @@ def test_job_success(self, scheduler_worker, storage, success_job): assert len(outputs) == 1 assert outputs[0].value == msgpack.packb(10) - def test_job_failure(self, scheduler_worker, storage, fail_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_job_failure(self, storage, fail_job): + """ + Test the failure of a job that raise an error. + :param storage: + :param fail_job: + :return: None + """ task = fail_job # Wait for 2 seconds and check task output time.sleep(2) state = get_task_state(storage, task.id) assert state == "fail" - def test_data_job(self, scheduler_worker, storage, data_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_data_job(self, storage, data_job): + """ + Test the successful execution of a job that uses data. + :param storage: + :param data_job: + :return: None + """ task = data_job # Wait for 2 seconds and check task output time.sleep(2) @@ -249,7 +307,14 @@ def test_data_job(self, scheduler_worker, storage, data_job): assert len(outputs) == 1 assert outputs[0].value == msgpack.packb(2) - def test_random_fail_job(self, scheduler_worker, storage, random_fail_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_random_fail_job(self, storage, random_fail_job): + """ + Test the successful recovery and execution of a job that randomly fails. + :param storage: + :param random_fail_job: + :return: None + """ task = random_fail_job # Wait for 2 seconds and check task output time.sleep(2) diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index a32f1c3d..012fac7b 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -4,6 +4,7 @@ import time import uuid from pathlib import Path +from collections.abc import Generator import msgpack import pytest @@ -23,7 +24,14 @@ from .utils import g_scheduler_port -def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str): +def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> tuple[subprocess.Popen, subprocess.Popen]: + """ + Create a scheduler and a worker process. + :param storage_url: JDB storage URL. + :param scheduler_port: the port for the scheduler to listen on. + :param lib: Library to load in the worker. + :return: scheduler and worker processes. + """ root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -52,7 +60,12 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str): @pytest.fixture(scope="function") -def scheduler_worker_signal(storage): +@pytest.mark.usefixtures("storage") +def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen], None, None]: + """ + Fixture to start a scheduler and a worker process for testing signal handling. + :return: + """ scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) @@ -65,12 +78,19 @@ def scheduler_worker_signal(storage): class TestWorkerSignal: - # Test that worker propagates the SIGTERM signal to the task executor. - # Submit a task that checks whether the task executor receives the SIGTERM signal. - # The task should return the SIGTERM signal number as the output. - # Later task should not be executed. - # Worker should exit with SIGTERM. - def test_task_signal(self, storage, scheduler_worker_signal): + + def test_task_signal(self, storage, scheduler_worker_signal) -> None: + """ + Test that worker propagates the SIGTERM signal to the task executor. + Submit a task that checks whether the task executor receives the SIGTERM signal. + The task should return the SIGTERM signal number as the output. + Later task should not be executed. + Worker should exit with SIGTERM. + + :param storage: + :param scheduler_worker_signal: + :return: None + """ _, worker_process = scheduler_worker_signal # Submit signal handler task to check for SIGTERM signal in task executor @@ -133,11 +153,17 @@ def test_task_signal(self, storage, scheduler_worker_signal): remove_job(storage, new_graph.id) remove_job(storage, graph.id) - # Test that worker propagates the SIGTERM signal to the task executor. - # Task executor exits immediately after receiving the signal. - # The running task should be marked as failed. - # The worker should exit with SIGTERM. - def test_task_exit(self, storage, scheduler_worker_signal): + def test_task_exit(self, storage, scheduler_worker_signal) -> None: + """ + Test that worker propagates the SIGTERM signal to the task executor. + Task executor exits immediately after receiving the signal. + The running task should be marked as failed. + The worker should exit with SIGTERM. + + :param storage: + :param scheduler_worker_signal: + :return: None + """ _, worker_process = scheduler_worker_signal # Submit a task to sleep for 10 seconds From 573b4487fc933e8cc852e90af321bcf338517a77 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:18:04 -0400 Subject: [PATCH 20/91] Fix ruff lint --- tests/integration/client.py | 8 +++++++- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 3 +-- tests/integration/test_signal.py | 6 ++++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 834ae2b5..99662b54 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -1,7 +1,7 @@ import re import uuid -from dataclasses import dataclass from collections.abc import Generator +from dataclasses import dataclass import mysql.connector import pytest @@ -13,6 +13,7 @@ class TaskInput: TaskInput represents an input to a task. It can either be a direct value, a reference to another task's output, or a reference to data. """ + type: str task_output: tuple[uuid.UUID, int] | None = None value: str | None = None @@ -25,6 +26,7 @@ class TaskOutput: TaskOutput represents an output of a task. It can either be a direct value or a reference to data. """ + type: str value: str | None = None data_id: uuid.UUID | None = None @@ -35,6 +37,7 @@ class Task: """ Task represents a unit of work in the task graph. """ + id: uuid.UUID function_name: str inputs: list[TaskInput] @@ -48,6 +51,7 @@ class TaskGraph: """ TaskGraph represents a directed acyclic graph of tasks. """ + id: uuid.UUID tasks: dict[uuid.UUID, Task] dependencies: list[tuple[uuid.UUID, uuid.UUID]] @@ -58,6 +62,7 @@ class Driver: """ Driver represents a client that can submit jobs to the task graph. """ + id: uuid.UUID @@ -66,6 +71,7 @@ class Data: """ Data represents a Spider Data object. """ + id: uuid.UUID value: str diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 7d6edeeb..c1d15504 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -1,7 +1,7 @@ import subprocess import time -from pathlib import Path from collections.abc import Generator +from pathlib import Path import pytest diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index f138f9f8..9c6ac27e 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -1,13 +1,12 @@ import subprocess import time import uuid -from pathlib import Path from collections.abc import Generator +from pathlib import Path import msgpack import pytest -from tests.integration.client import TaskGraph, Task from .client import ( add_driver, add_driver_data, diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 012fac7b..764bf818 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -3,8 +3,8 @@ import subprocess import time import uuid -from pathlib import Path from collections.abc import Generator +from pathlib import Path import msgpack import pytest @@ -32,6 +32,7 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t :param lib: Library to load in the worker. :return: scheduler and worker processes. """ + root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -61,11 +62,12 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t @pytest.fixture(scope="function") @pytest.mark.usefixtures("storage") -def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen], None, None]: +def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: """ + scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) From 6ad72c3b93e9415b6fddd291956435fda79db005 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:34:04 -0400 Subject: [PATCH 21/91] Bug fix --- ruff.toml | 1 + tests/integration/client.py | 43 +++++++++------------- tests/integration/test_client.py | 4 ++ tests/integration/test_scheduler_worker.py | 3 ++ tests/integration/test_signal.py | 5 ++- tests/integration/utils.py | 2 + 6 files changed, 31 insertions(+), 27 deletions(-) diff --git a/ruff.toml b/ruff.toml index a7a590d3..75462c9c 100644 --- a/ruff.toml +++ b/ruff.toml @@ -11,6 +11,7 @@ ignore = [ "D400", # First line of docstrings may not end in period "D401", # Docstrings should be written in present tense (not imperative) "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations "FBT", # Allow bool positional parameters since other value positions are allowed "FIX002", # Allow todo statements "PERF401", # Allow for loops when creating lists diff --git a/tests/integration/client.py b/tests/integration/client.py index 99662b54..682038e4 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -1,3 +1,5 @@ +"""Simple Spider client for testing purposes.""" + import re import uuid from collections.abc import Generator @@ -34,9 +36,7 @@ class TaskOutput: @dataclass class Task: - """ - Task represents a unit of work in the task graph. - """ + """Task represents a unit of work in the task graph.""" id: uuid.UUID function_name: str @@ -48,9 +48,7 @@ class Task: @dataclass class TaskGraph: - """ - TaskGraph represents a directed acyclic graph of tasks. - """ + """TaskGraph represents a directed acyclic graph of tasks.""" id: uuid.UUID tasks: dict[uuid.UUID, Task] @@ -59,18 +57,14 @@ class TaskGraph: @dataclass class Driver: - """ - Driver represents a client that can submit jobs to the task graph. - """ + """Driver represents a client that can submit jobs to the task graph.""" id: uuid.UUID @dataclass class Data: - """ - Data represents a Spider Data object. - """ + """Data represents a Spider Data object.""" id: uuid.UUID value: str @@ -87,7 +81,7 @@ def create_connection(storage_url: str) -> mysql.connector.MySQLConnection: ) match = pattern.match(storage_url) if not match: - raise ValueError("Invalid JDBC URL format") + raise ValueError(storage_url) connection_params = match.groupdict() return mysql.connector.connect( @@ -103,7 +97,8 @@ def is_head_task(task_id: uuid.UUID, dependencies: list[tuple[uuid.UUID, uuid.UU """ Check if the task is a head task, meaning it has no parent. :param task_id: the ID of the task to check. - :param dependencies: list of dependencies where each dependency is a tuple (parent_id, child_id). + :param dependencies: list of dependencies where each dependency is a tuple + (parent_id, child_id). :return: True if the task has no parent, False otherwise. """ return not any(dependency[1] == task_id for dependency in dependencies) @@ -124,7 +119,8 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: conn.close() -def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: +def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph)\ + -> None: """ Submit a job to the database. :param conn: database connection object. @@ -139,10 +135,7 @@ def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: ) for task_id, task in graph.tasks.items(): - if is_head_task(task_id, graph.dependencies): - state = "ready" - else: - state = "pending" + state = "ready" if is_head_task(task_id, graph.dependencies) else "pending" cursor.execute( "INSERT INTO tasks (id, job_id, func_name, state, timeout, max_retry) VALUES (%s, %s, %s, %s, %s, %s)", ( @@ -185,7 +178,7 @@ def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: cursor.close() -def get_task_outputs(conn, task_id: uuid.UUID) -> list[TaskOutput]: +def get_task_outputs(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> list[TaskOutput]: """ Get the outputs of a task by its ID. :param conn: database connection object. @@ -212,7 +205,7 @@ def get_task_outputs(conn, task_id: uuid.UUID) -> list[TaskOutput]: return outputs -def get_task_state(conn, task_id: uuid.UUID) -> str: +def get_task_state(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> str: """ Get the state of a task by its ID. :param conn: database connection object. @@ -229,7 +222,7 @@ def get_task_state(conn, task_id: uuid.UUID) -> str: return state -def remove_job(conn, job_id: uuid.UUID) -> None: +def remove_job(conn: mysql.connector.MySQLConnection, job_id: uuid.UUID) -> None: """ Remove a job from the database by its ID. :param conn: database connection object. @@ -243,7 +236,7 @@ def remove_job(conn, job_id: uuid.UUID) -> None: cursor.close() -def add_driver(conn, driver: Driver) -> None: +def add_driver(conn: mysql.connector.MySQLConnection, driver: Driver) -> None: """ Register a new driver in the database. :param conn: database connection object. @@ -258,7 +251,7 @@ def add_driver(conn, driver: Driver) -> None: cursor.close() -def add_driver_data(conn, driver: Driver, data: Data) -> None: +def add_driver_data(conn: mysql.connector.MySQLConnection, driver: Driver, data: Data) -> None: """ Add a new data associated with a driver in the database. :param conn: database connection object. @@ -278,7 +271,7 @@ def add_driver_data(conn, driver: Driver, data: Data) -> None: cursor.close() -def remove_data(conn, data: Data) -> None: +def remove_data(conn: mysql.connector.MySQLConnection, data: Data) -> None: """ Remove data from the database by its ID. :param conn: database connection object. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index c1d15504..6fbbd6ad 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -1,3 +1,5 @@ +"""Integration test for the client_test C++ program.""" + import subprocess import time from collections.abc import Generator @@ -68,6 +70,8 @@ def scheduler_worker() -> Generator[None, None, None]: class TestClient: + """Test class for the client_test C++ program.""" + @pytest.mark.usefixtures("scheduler_worker") def test_client(self) -> None: """ diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 9c6ac27e..93f2cde2 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -1,3 +1,5 @@ +"""Integration tests for the scheduler and worker processes.""" + import subprocess import time import uuid @@ -248,6 +250,7 @@ def random_fail_job(storage): class TestSchedulerWorker: + """Test class for the scheduler and worker integration tests.""" @pytest.mark.usefixtures("scheduler_worker") def test_job_success(self, storage, success_job): """ diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 764bf818..c9605b89 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -1,3 +1,5 @@ +"""Integration tests for worker signal handling.""" + import os import signal import subprocess @@ -32,7 +34,6 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t :param lib: Library to load in the worker. :return: scheduler and worker processes. """ - root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -67,7 +68,6 @@ def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Po Fixture to start a scheduler and a worker process for testing signal handling. :return: """ - scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) @@ -79,6 +79,7 @@ def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Po class TestWorkerSignal: + """Test cases for worker signal handling.""" def test_task_signal(self, storage, scheduler_worker_signal) -> None: diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 33aada43..79e4bf95 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -1,3 +1,5 @@ +"""Utilities for the network port.""" + import socket From ba7c6e58e9d6936149bb1d5ba397e0c1c076ec2c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:57:00 -0400 Subject: [PATCH 22/91] Fix ruff --- ruff.toml | 4 +++- tests/integration/client.py | 6 ++++-- tests/integration/test_client.py | 5 ++--- tests/integration/test_scheduler_worker.py | 16 ++++++++-------- tests/integration/test_signal.py | 9 ++++----- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/ruff.toml b/ruff.toml index 75462c9c..d2294f8d 100644 --- a/ruff.toml +++ b/ruff.toml @@ -26,9 +26,11 @@ isort.order-by-type = false [lint.per-file-ignores] "tests/integration/test_*.py" = [ - "S101", # Allow use of `assert` (security warning) "ANN001", # Ignore missing type annotations for arguments "ANN201", # Ignore missing type annotations for parameters + "S101", # Allow use of `assert` (security warning) + "S603", # Allow user of subprocess.Popen (security warning) + "T201", # Allow use of `print` (testing) ] [format] diff --git a/tests/integration/client.py b/tests/integration/client.py index 682038e4..4ed1258d 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -137,7 +137,8 @@ def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, grap for task_id, task in graph.tasks.items(): state = "ready" if is_head_task(task_id, graph.dependencies) else "pending" cursor.execute( - "INSERT INTO tasks (id, job_id, func_name, state, timeout, max_retry) VALUES (%s, %s, %s, %s, %s, %s)", + "INSERT INTO tasks (id, job_id, func_name, state, timeout, max_retry)" + " VALUES (%s, %s, %s, %s, %s, %s)", ( task.id.bytes, graph.id.bytes, @@ -150,7 +151,8 @@ def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, grap for i, task_input in enumerate(task.inputs): cursor.execute( - "INSERT INTO task_inputs (type, task_id, position, output_task_id, output_task_position, value, data_id) VALUES (%s, %s, %s, %s, %s, %s, %s)", + "INSERT INTO task_inputs (type, task_id, position, output_task_id," + " output_task_position, value, data_id) VALUES (%s, %s, %s, %s, %s, %s, %s)", ( task_input.type, task.id.bytes, diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 6fbbd6ad..a00dc8a6 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -51,8 +51,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") -@pytest.mark.usefixtures("storage") -def scheduler_worker() -> Generator[None, None, None]: +def scheduler_worker(storage) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, and then kills the processes after the test class is done. @@ -85,5 +84,5 @@ def test_client(self) -> None: "--storage_url", g_storage_url, ] - p = subprocess.run(client_cmds, check=False, timeout=20) + p = subprocess.run(client_cmds, check=True, timeout=20) assert p.returncode == 0 diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 93f2cde2..1a2f8887 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -65,8 +65,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") -@pytest.mark.usefixtures("storage") -def scheduler_worker() -> Generator[None, None, None]: +def scheduler_worker(storage) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. After the test function completes, it kills the scheduler and the worker process. @@ -82,11 +81,11 @@ def scheduler_worker() -> Generator[None, None, None]: worker_process.kill() -@pytest.fixture(scope="function") +@pytest.fixture def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ - Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. - Cleans up the job after the test function completes. + Fixture to create a job with two parent tasks and one child task. Yields the task graph and + tasks. Cleans up the job after the test function completes. :param storage: :return: """ @@ -146,7 +145,7 @@ def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, remove_job(storage, graph.id) -@pytest.fixture(scope="function") +@pytest.fixture def fail_job(storage) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -174,7 +173,7 @@ def fail_job(storage) -> Generator[Task, None, None]: remove_job(storage, graph.id) -@pytest.fixture(scope="function") +@pytest.fixture def data_job(storage) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -211,7 +210,7 @@ def data_job(storage) -> Generator[Task, None, None]: remove_data(storage, data) -@pytest.fixture(scope="function") +@pytest.fixture def random_fail_job(storage): """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -251,6 +250,7 @@ def random_fail_job(storage): class TestSchedulerWorker: """Test class for the scheduler and worker integration tests.""" + @pytest.mark.usefixtures("scheduler_worker") def test_job_success(self, storage, success_job): """ diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index c9605b89..0edb7164 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -26,7 +26,8 @@ from .utils import g_scheduler_port -def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> tuple[subprocess.Popen, subprocess.Popen]: +def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ + -> tuple[subprocess.Popen, subprocess.Popen]: """ Create a scheduler and a worker process. :param storage_url: JDB storage URL. @@ -61,9 +62,8 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t return scheduler_process, worker_process -@pytest.fixture(scope="function") -@pytest.mark.usefixtures("storage") -def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +@pytest.fixture +def scheduler_worker_signal(storage) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: @@ -81,7 +81,6 @@ def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Po class TestWorkerSignal: """Test cases for worker signal handling.""" - def test_task_signal(self, storage, scheduler_worker_signal) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From 209acb14a38a3f13c2622f2e9c0a210115f5038a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 14:56:59 -0400 Subject: [PATCH 23/91] Fix ruff --- ruff.toml | 2 - tests/integration/client.py | 7 +++- tests/integration/test_client.py | 7 +++- tests/integration/test_scheduler_worker.py | 47 +++++++++++++++++----- tests/integration/test_signal.py | 22 +++++++--- 5 files changed, 64 insertions(+), 21 deletions(-) diff --git a/ruff.toml b/ruff.toml index d2294f8d..6cdab792 100644 --- a/ruff.toml +++ b/ruff.toml @@ -26,8 +26,6 @@ isort.order-by-type = false [lint.per-file-ignores] "tests/integration/test_*.py" = [ - "ANN001", # Ignore missing type annotations for arguments - "ANN201", # Ignore missing type annotations for parameters "S101", # Allow use of `assert` (security warning) "S603", # Allow user of subprocess.Popen (security warning) "T201", # Allow use of `print` (testing) diff --git a/tests/integration/client.py b/tests/integration/client.py index 4ed1258d..886a6c3c 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -119,8 +119,11 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: conn.close() -def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph)\ - -> None: +def submit_job( + conn: mysql.connector.MySQLConnection, + client_id: uuid.UUID, + graph: TaskGraph +) -> None: """ Submit a job to the database. :param conn: database connection object. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index a00dc8a6..aac10a9c 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -5,11 +5,11 @@ from collections.abc import Generator from pathlib import Path +import mysql.connector import pytest from .client import ( g_storage_url, - storage, ) from .utils import g_scheduler_port @@ -51,12 +51,15 @@ def start_scheduler_workers( @pytest.fixture(scope="class") -def scheduler_worker(storage) -> Generator[None, None, None]: +def scheduler_worker( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, and then kills the processes after the test class is done. :return: """ + _ = storage # Avoid ARG001 scheduler_process, worker_process_0, worker_process_1 = start_scheduler_workers( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 1a2f8887..84293fc4 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -7,6 +7,7 @@ from pathlib import Path import msgpack +import mysql.connector import pytest from .client import ( @@ -19,7 +20,6 @@ get_task_state, remove_data, remove_job, - storage, submit_job, Task, TaskGraph, @@ -65,12 +65,15 @@ def start_scheduler_worker( @pytest.fixture(scope="class") -def scheduler_worker(storage) -> Generator[None, None, None]: +def scheduler_worker( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. After the test function completes, it kills the scheduler and the worker process. :return: """ + _ = storage # Avoid ARG001 scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) @@ -82,7 +85,9 @@ def scheduler_worker(storage) -> Generator[None, None, None]: @pytest.fixture -def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: +def success_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +)-> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. Cleans up the job after the test function completes. @@ -146,7 +151,9 @@ def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, @pytest.fixture -def fail_job(storage) -> Generator[Task, None, None]: +def fail_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. Yield the task. Cleanup the job after the test function completes. @@ -174,7 +181,9 @@ def fail_job(storage) -> Generator[Task, None, None]: @pytest.fixture -def data_job(storage) -> Generator[Task, None, None]: +def data_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. Cleans up the job and data after the test function completes. @@ -211,7 +220,9 @@ def data_job(storage) -> Generator[Task, None, None]: @pytest.fixture -def random_fail_job(storage): +def random_fail_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. Yields the task. Cleans up the job after the test function completes. @@ -252,7 +263,11 @@ class TestSchedulerWorker: """Test class for the scheduler and worker integration tests.""" @pytest.mark.usefixtures("scheduler_worker") - def test_job_success(self, storage, success_job): + def test_job_success( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None] + ) -> None: """ Test the successful execution of a job with two parent tasks and one child task. :param storage: @@ -279,7 +294,11 @@ def test_job_success(self, storage, success_job): assert outputs[0].value == msgpack.packb(10) @pytest.mark.usefixtures("scheduler_worker") - def test_job_failure(self, storage, fail_job): + def test_job_failure( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + fail_job: Generator[Task, None, None] + ) -> None: """ Test the failure of a job that raise an error. :param storage: @@ -293,7 +312,11 @@ def test_job_failure(self, storage, fail_job): assert state == "fail" @pytest.mark.usefixtures("scheduler_worker") - def test_data_job(self, storage, data_job): + def test_data_job( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + data_job: Generator[Task, None, None] + ) -> None: """ Test the successful execution of a job that uses data. :param storage: @@ -310,7 +333,11 @@ def test_data_job(self, storage, data_job): assert outputs[0].value == msgpack.packb(2) @pytest.mark.usefixtures("scheduler_worker") - def test_random_fail_job(self, storage, random_fail_job): + def test_random_fail_job( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + random_fail_job: Generator[Task, None, None] + ) -> None: """ Test the successful recovery and execution of a job that randomly fails. :param storage: diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 0edb7164..0da6466a 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -9,6 +9,7 @@ from pathlib import Path import msgpack +import mysql.connector import pytest from .client import ( @@ -16,7 +17,6 @@ get_task_outputs, get_task_state, remove_job, - storage, submit_job, Task, TaskGraph, @@ -37,7 +37,7 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ """ root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" - popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + popen_opts = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "text": True} scheduler_cmds = [ str(bin_dir / "spider_scheduler"), "--host", @@ -63,11 +63,13 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ @pytest.fixture -def scheduler_worker_signal(storage) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +def scheduler_worker_signal(storage: Generator[mysql.connector.MySQLConnection, None, None])\ + -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: """ + _ = storage # Avoid ARG001 scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) @@ -81,7 +83,12 @@ def scheduler_worker_signal(storage) -> Generator[tuple[subprocess.Popen, subpro class TestWorkerSignal: """Test cases for worker signal handling.""" - def test_task_signal(self, storage, scheduler_worker_signal) -> None: + def test_task_signal( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[ + tuple[subprocess.Popen, subprocess.Popen], None, None] + ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. Submit a task that checks whether the task executor receives the SIGTERM signal. @@ -155,7 +162,12 @@ def test_task_signal(self, storage, scheduler_worker_signal) -> None: remove_job(storage, new_graph.id) remove_job(storage, graph.id) - def test_task_exit(self, storage, scheduler_worker_signal) -> None: + def test_task_exit( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[ + tuple[subprocess.Popen, subprocess.Popen], None, None] + ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. Task executor exits immediately after receiving the signal. From 406b514007a05e9701ddcacd0638dd5d20ad4e0e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 14:57:52 -0400 Subject: [PATCH 24/91] Reformat files --- tests/integration/client.py | 4 +-- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 36 +++++++++++----------- tests/integration/test_signal.py | 24 +++++++-------- 4 files changed, 32 insertions(+), 34 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 886a6c3c..721ec244 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -120,9 +120,7 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: def submit_job( - conn: mysql.connector.MySQLConnection, - client_id: uuid.UUID, - graph: TaskGraph + conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph ) -> None: """ Submit a job to the database. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index aac10a9c..f70f5198 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -52,7 +52,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 84293fc4..2f0421c0 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -66,7 +66,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. @@ -86,8 +86,8 @@ def scheduler_worker( @pytest.fixture def success_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] -)-> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: + storage: Generator[mysql.connector.MySQLConnection, None, None], +) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. Cleans up the job after the test function completes. @@ -152,7 +152,7 @@ def success_job( @pytest.fixture def fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -182,7 +182,7 @@ def fail_job( @pytest.fixture def data_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -221,7 +221,7 @@ def data_job( @pytest.fixture def random_fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -264,9 +264,9 @@ class TestSchedulerWorker: @pytest.mark.usefixtures("scheduler_worker") def test_job_success( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None], ) -> None: """ Test the successful execution of a job with two parent tasks and one child task. @@ -295,9 +295,9 @@ def test_job_success( @pytest.mark.usefixtures("scheduler_worker") def test_job_failure( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - fail_job: Generator[Task, None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + fail_job: Generator[Task, None, None], ) -> None: """ Test the failure of a job that raise an error. @@ -313,9 +313,9 @@ def test_job_failure( @pytest.mark.usefixtures("scheduler_worker") def test_data_job( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - data_job: Generator[Task, None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + data_job: Generator[Task, None, None], ) -> None: """ Test the successful execution of a job that uses data. @@ -334,9 +334,9 @@ def test_data_job( @pytest.mark.usefixtures("scheduler_worker") def test_random_fail_job( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - random_fail_job: Generator[Task, None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + random_fail_job: Generator[Task, None, None], ) -> None: """ Test the successful recovery and execution of a job that randomly fails. diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 0da6466a..28b27534 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -26,8 +26,9 @@ from .utils import g_scheduler_port -def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ - -> tuple[subprocess.Popen, subprocess.Popen]: +def start_scheduler_worker( + storage_url: str, scheduler_port: int, lib: str +) -> tuple[subprocess.Popen, subprocess.Popen]: """ Create a scheduler and a worker process. :param storage_url: JDB storage URL. @@ -63,8 +64,9 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ @pytest.fixture -def scheduler_worker_signal(storage: Generator[mysql.connector.MySQLConnection, None, None])\ - -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +def scheduler_worker_signal( + storage: Generator[mysql.connector.MySQLConnection, None, None], +) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: @@ -84,10 +86,9 @@ class TestWorkerSignal: """Test cases for worker signal handling.""" def test_task_signal( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - scheduler_worker_signal: Generator[ - tuple[subprocess.Popen, subprocess.Popen], None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. @@ -163,10 +164,9 @@ def test_task_signal( remove_job(storage, graph.id) def test_task_exit( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - scheduler_worker_signal: Generator[ - tuple[subprocess.Popen, subprocess.Popen], None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From 66892f5e5194490d9dd8af79ecb9db2ac63740fe Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 16:08:45 -0400 Subject: [PATCH 25/91] Remove .inc from cpp linting --- lint-tasks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 659480e5..161c8a19 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -101,7 +101,7 @@ tasks: FLAGS: - "--config-file '{{.ROOT_DIR}}/.clang-tidy'" - "-p '{{.G_EXAMPLES_COMPILE_COMMANDS_DB}}'" - INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] + INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp"] OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" ROOT_PATHS: - "{{.G_EXAMPLES_DIR}}" From 3186a429f9a79ab93750deaa47c3d542a3aaa0af Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 18:41:52 -0400 Subject: [PATCH 26/91] Add mypy and merge lint and test requirements.txt --- lint-tasks.yaml | 5 +++-- lint-requirements.txt => lint-test-requirements.txt | 7 +++++++ mypy.ini | 8 ++++++++ test-requirements.txt | 3 --- test-tasks.yaml | 4 ++-- 5 files changed, 20 insertions(+), 7 deletions(-) rename lint-requirements.txt => lint-test-requirements.txt (57%) create mode 100644 mypy.ini delete mode 100644 test-requirements.txt diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 7822bda7..3739fa12 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -130,6 +130,7 @@ tasks: - "tests/integration" cmd: |- . "{{.G_LINT_VENV_DIR}}/bin/activate" + mypy "{{.ITEM}}" ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}" ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}" @@ -177,7 +178,7 @@ tasks: sources: - "{{.ROOT_DIR}}/taskfile.yaml" - "{{.TASKFILE}}" - - "lint-requirements.txt" + - "lint-test-requirements.txt" generates: ["{{.CHECKSUM_FILE}}"] run: "once" deps: @@ -191,7 +192,7 @@ tasks: vars: LABEL: "lint" OUTPUT_DIR: "{{.OUTPUT_DIR}}" - REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-requirements.txt" + REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-test-requirements.txt" # This command must be last - task: ":utils:checksum:compute" vars: diff --git a/lint-requirements.txt b/lint-test-requirements.txt similarity index 57% rename from lint-requirements.txt rename to lint-test-requirements.txt index 690b9797..7ac75af2 100644 --- a/lint-requirements.txt +++ b/lint-test-requirements.txt @@ -1,6 +1,13 @@ +# Lint dependencies clang-format>=20.1.0 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 ruff>=0.4.4 +mypy>=1.12.0 gersemi>=0.16.2 yamllint>=1.35.1 + +# Test dependencies +msgpack>=1.1.0 +mysql-connector-python>=8.0.26 +pytest>=8.3.4 diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..4c66b111 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +strict = true + +# Additional output +pretty = true +show_error_code_links = true +show_error_context = true +show_error_end = true diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index ec24b0b2..00000000 --- a/test-requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -msgpack>=1.1.0 -mysql-connector-python>=8.0.26 -pytest>=8.3.4 \ No newline at end of file diff --git a/test-tasks.yaml b/test-tasks.yaml index 116ae85b..41dfc457 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -56,7 +56,7 @@ tasks: sources: - "{{.ROOT_DIR}}/taskfile.yaml" - "{{.TASKFILE}}" - - "test-requirements.txt" + - "lint-test-requirements.txt" generates: ["{{.CHECKSUM_FILE}}"] run: "once" deps: @@ -70,7 +70,7 @@ tasks: vars: LABEL: "test" OUTPUT_DIR: "{{.OUTPUT_DIR}}" - REQUIREMENTS_FILE: "{{.ROOT_DIR}}/test-requirements.txt" + REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-test-requirements.txt" # This command must be last - task: ":utils:checksum:compute" vars: From 092593844041d72e6da23797a7c8fb16fbf1d258 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:34:15 -0400 Subject: [PATCH 27/91] Fix mysql connection type --- tests/integration/client.py | 19 ++++++++++--------- tests/integration/test_client.py | 4 ++-- tests/integration/test_scheduler_worker.py | 20 ++++++++++---------- tests/integration/test_signal.py | 8 ++++---- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 721ec244..f0b7f0cf 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -69,8 +69,9 @@ class Data: id: uuid.UUID value: str +SQLConnection = mysql.connector.abstracts.MySQLConnectionAbstract | mysql.connector.pooling.PooledMySQLConnection -def create_connection(storage_url: str) -> mysql.connector.MySQLConnection: +def create_connection(storage_url: str) -> SQLConnection : """ Creation a MariaDB connection from a JDBC URL. :param storage_url: JDBC URL for the MariaDB database. @@ -108,7 +109,7 @@ def is_head_task(task_id: uuid.UUID, dependencies: list[tuple[uuid.UUID, uuid.UU @pytest.fixture(scope="session") -def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: +def storage() -> Generator[SQLConnection, None, None]: """ Fixture to create a database connection for the test session. Yields a connection object and ensures it is closed after the tests are done. @@ -120,7 +121,7 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: def submit_job( - conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph + conn: SQLConnection, client_id: uuid.UUID, graph: TaskGraph ) -> None: """ Submit a job to the database. @@ -181,7 +182,7 @@ def submit_job( cursor.close() -def get_task_outputs(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> list[TaskOutput]: +def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput]: """ Get the outputs of a task by its ID. :param conn: database connection object. @@ -208,7 +209,7 @@ def get_task_outputs(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) return outputs -def get_task_state(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> str: +def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: """ Get the state of a task by its ID. :param conn: database connection object. @@ -225,7 +226,7 @@ def get_task_state(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> return state -def remove_job(conn: mysql.connector.MySQLConnection, job_id: uuid.UUID) -> None: +def remove_job(conn: SQLConnection, job_id: uuid.UUID) -> None: """ Remove a job from the database by its ID. :param conn: database connection object. @@ -239,7 +240,7 @@ def remove_job(conn: mysql.connector.MySQLConnection, job_id: uuid.UUID) -> None cursor.close() -def add_driver(conn: mysql.connector.MySQLConnection, driver: Driver) -> None: +def add_driver(conn: SQLConnection, driver: Driver) -> None: """ Register a new driver in the database. :param conn: database connection object. @@ -254,7 +255,7 @@ def add_driver(conn: mysql.connector.MySQLConnection, driver: Driver) -> None: cursor.close() -def add_driver_data(conn: mysql.connector.MySQLConnection, driver: Driver, data: Data) -> None: +def add_driver_data(conn: SQLConnection, driver: Driver, data: Data) -> None: """ Add a new data associated with a driver in the database. :param conn: database connection object. @@ -274,7 +275,7 @@ def add_driver_data(conn: mysql.connector.MySQLConnection, driver: Driver, data: cursor.close() -def remove_data(conn: mysql.connector.MySQLConnection, data: Data) -> None: +def remove_data(conn: SQLConnection, data: Data) -> None: """ Remove data from the database by its ID. :param conn: database connection object. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index f70f5198..80908592 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -5,11 +5,11 @@ from collections.abc import Generator from pathlib import Path -import mysql.connector import pytest from .client import ( g_storage_url, + SQLConnection ) from .utils import g_scheduler_port @@ -52,7 +52,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 2f0421c0..19087568 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -7,7 +7,6 @@ from pathlib import Path import msgpack -import mysql.connector import pytest from .client import ( @@ -20,6 +19,7 @@ get_task_state, remove_data, remove_job, + SQLConnection, submit_job, Task, TaskGraph, @@ -66,7 +66,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. @@ -86,7 +86,7 @@ def scheduler_worker( @pytest.fixture def success_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and @@ -152,7 +152,7 @@ def success_job( @pytest.fixture def fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -182,7 +182,7 @@ def fail_job( @pytest.fixture def data_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -221,7 +221,7 @@ def data_job( @pytest.fixture def random_fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -265,7 +265,7 @@ class TestSchedulerWorker: @pytest.mark.usefixtures("scheduler_worker") def test_job_success( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None], ) -> None: """ @@ -296,7 +296,7 @@ def test_job_success( @pytest.mark.usefixtures("scheduler_worker") def test_job_failure( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], fail_job: Generator[Task, None, None], ) -> None: """ @@ -314,7 +314,7 @@ def test_job_failure( @pytest.mark.usefixtures("scheduler_worker") def test_data_job( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], data_job: Generator[Task, None, None], ) -> None: """ @@ -335,7 +335,7 @@ def test_data_job( @pytest.mark.usefixtures("scheduler_worker") def test_random_fail_job( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], random_fail_job: Generator[Task, None, None], ) -> None: """ diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 28b27534..42eee6ba 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -9,7 +9,6 @@ from pathlib import Path import msgpack -import mysql.connector import pytest from .client import ( @@ -17,6 +16,7 @@ get_task_outputs, get_task_state, remove_job, + SQLConnection, submit_job, Task, TaskGraph, @@ -65,7 +65,7 @@ def start_scheduler_worker( @pytest.fixture def scheduler_worker_signal( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. @@ -87,7 +87,7 @@ class TestWorkerSignal: def test_task_signal( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ @@ -165,7 +165,7 @@ def test_task_signal( def test_task_exit( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ From 6936a9b448a6ce4ececbc97d53e23b67569850e4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:39:24 -0400 Subject: [PATCH 28/91] Fix socket name type --- tests/integration/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 79e4bf95..cae3f173 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -2,12 +2,16 @@ import socket +IPv4Addr = tuple[str, int] +IPv6Addr = tuple[str, int, int, int] +AddrType = IPv4Addr | IPv6Addr def _get_free_tcp_port() -> int: """Returns a free TCP port.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(("127.0.0.1", 0)) - return s.getsockname()[1] + addr: AddrType = s.getsockname() + return addr[1] g_scheduler_port = _get_free_tcp_port() From 74070908318632b16e7dde6800994676d19e8982 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:43:07 -0400 Subject: [PATCH 29/91] Fix return type from db cursor --- tests/integration/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index f0b7f0cf..b4369fe3 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -196,7 +196,8 @@ def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput (task_id.bytes,), ) outputs = [] - for output_type, value, data_id in cursor.fetchall(): + rows: list[tuple[str, str | None, bytes | None]] = cursor.fetchall() + for output_type, value, data_id in rows: if value is not None: outputs.append(TaskOutput(type=output_type, value=value)) elif data_id is not None: @@ -219,7 +220,7 @@ def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) - state = cursor.fetchone()[0] + state: str = cursor.fetchone()[0] conn.commit() cursor.close() From 6a95b24912c69807c9cc66172f97a5441beea8e6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:50:34 -0400 Subject: [PATCH 30/91] Fix db cursor return type --- tests/integration/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index b4369fe3..13f9666a 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -4,6 +4,7 @@ import uuid from collections.abc import Generator from dataclasses import dataclass +from typing import cast import mysql.connector import pytest @@ -196,7 +197,7 @@ def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput (task_id.bytes,), ) outputs = [] - rows: list[tuple[str, str | None, bytes | None]] = cursor.fetchall() + rows = cast(list[tuple[str, str | None, bytes | None]], cursor.fetchall()) for output_type, value, data_id in rows: if value is not None: outputs.append(TaskOutput(type=output_type, value=value)) @@ -220,7 +221,7 @@ def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) - state: str = cursor.fetchone()[0] + state = cast(tuple[str], cursor.fetchone())[0] conn.commit() cursor.close() From f29dcba479f5f80d66c99aa94fbf7ab93ecfc4c2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:55:24 -0400 Subject: [PATCH 31/91] Fix mypy import untyped --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 4c66b111..932c4e40 100644 --- a/mypy.ini +++ b/mypy.ini @@ -6,3 +6,6 @@ pretty = true show_error_code_links = true show_error_context = true show_error_end = true + +[mypy-msgpack.*] +ignore_missing_imports = true From 453f20908e8f566d7561e1a3c178c713e206e915 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:17:20 -0400 Subject: [PATCH 32/91] Fix mypy and Popen --- tests/integration/test_signal.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 42eee6ba..47848f63 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -7,6 +7,7 @@ import uuid from collections.abc import Generator from pathlib import Path +from typing import TypedDict import msgpack import pytest @@ -25,10 +26,16 @@ ) from .utils import g_scheduler_port +class PopenOpts(TypedDict, total=False): + stdin: int + stdout: int + stderr: int + text: bool + def start_scheduler_worker( storage_url: str, scheduler_port: int, lib: str -) -> tuple[subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]]: """ Create a scheduler and a worker process. :param storage_url: JDB storage URL. @@ -38,7 +45,11 @@ def start_scheduler_worker( """ root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" - popen_opts = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "text": True} + popen_opts: PopenOpts = { + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "text": True, + } scheduler_cmds = [ str(bin_dir / "spider_scheduler"), "--host", @@ -66,7 +77,7 @@ def start_scheduler_worker( @pytest.fixture def scheduler_worker_signal( storage: Generator[SQLConnection, None, None], -) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +) -> Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: @@ -88,7 +99,7 @@ class TestWorkerSignal: def test_task_signal( self, storage: Generator[SQLConnection, None, None], - scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], + scheduler_worker_signal: Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From d1011c70eb908fad0247f84d845be67c89d7bbd4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:22:31 -0400 Subject: [PATCH 33/91] Fix generator type hint --- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 26 +++++++++++----------- tests/integration/test_signal.py | 10 ++++----- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 80908592..c96b731d 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -52,7 +52,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 19087568..2aa59ed5 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -66,7 +66,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. @@ -86,7 +86,7 @@ def scheduler_worker( @pytest.fixture def success_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and @@ -152,7 +152,7 @@ def success_job( @pytest.fixture def fail_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -182,7 +182,7 @@ def fail_job( @pytest.fixture def data_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -221,7 +221,7 @@ def data_job( @pytest.fixture def random_fail_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -265,8 +265,8 @@ class TestSchedulerWorker: @pytest.mark.usefixtures("scheduler_worker") def test_job_success( self, - storage: Generator[SQLConnection, None, None], - success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None], + storage: SQLConnection, + success_job: tuple[TaskGraph, Task, Task, Task], ) -> None: """ Test the successful execution of a job with two parent tasks and one child task. @@ -296,8 +296,8 @@ def test_job_success( @pytest.mark.usefixtures("scheduler_worker") def test_job_failure( self, - storage: Generator[SQLConnection, None, None], - fail_job: Generator[Task, None, None], + storage: SQLConnection, + fail_job: Task, ) -> None: """ Test the failure of a job that raise an error. @@ -314,8 +314,8 @@ def test_job_failure( @pytest.mark.usefixtures("scheduler_worker") def test_data_job( self, - storage: Generator[SQLConnection, None, None], - data_job: Generator[Task, None, None], + storage: SQLConnection, + data_job: Task, ) -> None: """ Test the successful execution of a job that uses data. @@ -335,8 +335,8 @@ def test_data_job( @pytest.mark.usefixtures("scheduler_worker") def test_random_fail_job( self, - storage: Generator[SQLConnection, None, None], - random_fail_job: Generator[Task, None, None], + storage: SQLConnection, + random_fail_job: Task, ) -> None: """ Test the successful recovery and execution of a job that randomly fails. diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 47848f63..667e23d5 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -76,7 +76,7 @@ def start_scheduler_worker( @pytest.fixture def scheduler_worker_signal( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. @@ -98,8 +98,8 @@ class TestWorkerSignal: def test_task_signal( self, - storage: Generator[SQLConnection, None, None], - scheduler_worker_signal: Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None], + storage: SQLConnection, + scheduler_worker_signal: tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. @@ -176,8 +176,8 @@ def test_task_signal( def test_task_exit( self, - storage: Generator[SQLConnection, None, None], - scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], + storage: SQLConnection, + scheduler_worker_signal: tuple[subprocess.Popen, subprocess.Popen], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From a7dc642bcb68e83f52152b27bd811f358843ad2e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:24:47 -0400 Subject: [PATCH 34/91] Fix mypy --- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 2 +- tests/integration/test_signal.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index c96b731d..e5219835 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -16,7 +16,7 @@ def start_scheduler_workers( storage_url: str, scheduler_port: int -) -> tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen[bytes], subprocess.Popen[bytes], subprocess.Popen[bytes]]: """ Start the scheduler and two worker processes. :param storage_url: diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 2aa59ed5..ae841332 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -31,7 +31,7 @@ def start_scheduler_worker( storage_url: str, scheduler_port: int -) -> tuple[subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]]: """ Start a scheduler and a worker process. :param storage_url: JDBC storage URL diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 667e23d5..f42fdefc 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -177,7 +177,7 @@ def test_task_signal( def test_task_exit( self, storage: SQLConnection, - scheduler_worker_signal: tuple[subprocess.Popen, subprocess.Popen], + scheduler_worker_signal: tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From a7d92c2255faf5c3b336ca90069aecec7d70b600 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:28:12 -0400 Subject: [PATCH 35/91] Fix ruff --- tests/integration/client.py | 17 ++++++++++------- tests/integration/test_client.py | 5 +---- tests/integration/test_signal.py | 3 +++ tests/integration/utils.py | 1 + 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 13f9666a..b1e5a30e 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -70,9 +70,14 @@ class Data: id: uuid.UUID value: str -SQLConnection = mysql.connector.abstracts.MySQLConnectionAbstract | mysql.connector.pooling.PooledMySQLConnection -def create_connection(storage_url: str) -> SQLConnection : +SQLConnection = ( + mysql.connector.abstracts.MySQLConnectionAbstract + | mysql.connector.pooling.PooledMySQLConnection +) + + +def create_connection(storage_url: str) -> SQLConnection: """ Creation a MariaDB connection from a JDBC URL. :param storage_url: JDBC URL for the MariaDB database. @@ -121,9 +126,7 @@ def storage() -> Generator[SQLConnection, None, None]: conn.close() -def submit_job( - conn: SQLConnection, client_id: uuid.UUID, graph: TaskGraph -) -> None: +def submit_job(conn: SQLConnection, client_id: uuid.UUID, graph: TaskGraph) -> None: """ Submit a job to the database. :param conn: database connection object. @@ -197,7 +200,7 @@ def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput (task_id.bytes,), ) outputs = [] - rows = cast(list[tuple[str, str | None, bytes | None]], cursor.fetchall()) + rows = cast("list[tuple[str, str | None, bytes | None]]", cursor.fetchall()) for output_type, value, data_id in rows: if value is not None: outputs.append(TaskOutput(type=output_type, value=value)) @@ -221,7 +224,7 @@ def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) - state = cast(tuple[str], cursor.fetchone())[0] + state = cast("tuple[str]", cursor.fetchone())[0] conn.commit() cursor.close() diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index e5219835..2b2ca00a 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -7,10 +7,7 @@ import pytest -from .client import ( - g_storage_url, - SQLConnection -) +from .client import g_storage_url, SQLConnection from .utils import g_scheduler_port diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index f42fdefc..89191621 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -26,7 +26,10 @@ ) from .utils import g_scheduler_port + class PopenOpts(TypedDict, total=False): + """Options for subprocess.Popen.""" + stdin: int stdout: int stderr: int diff --git a/tests/integration/utils.py b/tests/integration/utils.py index cae3f173..7f163539 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -6,6 +6,7 @@ IPv6Addr = tuple[str, int, int, int] AddrType = IPv4Addr | IPv6Addr + def _get_free_tcp_port() -> int: """Returns a free TCP port.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: From 1c6213bb4a8d5d229bfacc89f84020c9463f2857 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:40:58 -0400 Subject: [PATCH 36/91] Simply socket return types. Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- tests/integration/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 7f163539..64c134ce 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -2,9 +2,7 @@ import socket -IPv4Addr = tuple[str, int] -IPv6Addr = tuple[str, int, int, int] -AddrType = IPv4Addr | IPv6Addr +AddrType = tuple[str, int] def _get_free_tcp_port() -> int: From 55a727ae8012f724a95d25360079b76ad1353ae5 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 23:41:32 -0400 Subject: [PATCH 37/91] Add tombi lint tasks --- lint-tasks.yaml | 24 +++++++++++++++++++++++ lint-test-requirements.txt | 1 + ruff.toml | 40 +++++++++++++++++++------------------- tombi.toml | 34 ++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 tombi.toml diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 2bfe9064..40259b21 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -11,12 +11,14 @@ tasks: - task: "cpp-check" - task: "py-check" - task: "yml-check" + - task: "toml-check" fix: cmds: - task: "cmake-fix" - task: "cpp-fix" - task: "yml-fix" + - task: "toml-fix" cmake-check: deps: ["venv"] @@ -154,6 +156,28 @@ tasks: taskfile.yaml \ test-tasks.yaml + toml: + aliases: + - "toml-check" + - "toml-fix" + cmds: + - task: "toml-lint" + - task: "toml-format" + + toml-lint: + deps: ["venv"] + cmds: + - |- + . "{{.G_LINT_VENV_DIR}}/bin/activate" + tombi lint "{{.ROOT_DIR}}" + + toml-format: + deps: ["venv"] + cmds: + - |- + . "{{.G_LINT_VENV_DIR}}/bin/activate" + tombi format "{{.ROOT_DIR}}" + cmake: internal: true requires: diff --git a/lint-test-requirements.txt b/lint-test-requirements.txt index 7ac75af2..879c198b 100644 --- a/lint-test-requirements.txt +++ b/lint-test-requirements.txt @@ -6,6 +6,7 @@ ruff>=0.4.4 mypy>=1.12.0 gersemi>=0.16.2 yamllint>=1.35.1 +tombi>=0.4.0 # Test dependencies msgpack>=1.1.0 diff --git a/ruff.toml b/ruff.toml index 6cdab792..fde4359e 100644 --- a/ruff.toml +++ b/ruff.toml @@ -4,31 +4,31 @@ line-length = 100 select = ["ALL"] extend-select = ["PT"] ignore = [ - "COM812", # Redundant and conflicts with ruff format - "D203", # No blank line before docstrings (D211) - "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists - "D212", # Enforce docstring summary line on the next line after quotes (D213) - "D400", # First line of docstrings may not end in period - "D401", # Docstrings should be written in present tense (not imperative) - "D415", # First line of docstrings may not end in a period, question mark, or exclamation point - "FA102", # Allow use of PEP 604 union in type annotations - "FBT", # Allow bool positional parameters since other value positions are allowed - "FIX002", # Allow todo statements - "PERF401", # Allow for loops when creating lists - "PERF403", # Allow for loops when creating dicts - "S311", # Allow usage of `random` package - "SIM102", # Allow collapsible if statements for readability - "TD002", # Author unnecessary for todo statement - "TD003", # Issue link unnecessary for todo statement - "UP015", # Explicit open modes are helpful + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful ] isort.order-by-type = false [lint.per-file-ignores] "tests/integration/test_*.py" = [ - "S101", # Allow use of `assert` (security warning) - "S603", # Allow user of subprocess.Popen (security warning) - "T201", # Allow use of `print` (testing) + "S101", # Allow use of `assert` (security warning) + "S603", # Allow user of subprocess.Popen (security warning) + "T201", # Allow use of `print` (testing) ] [format] diff --git a/tombi.toml b/tombi.toml new file mode 100644 index 00000000..ad86d3c0 --- /dev/null +++ b/tombi.toml @@ -0,0 +1,34 @@ +toml-version = "v1.0.0" + +[files] +include = ["**/*.toml"] +exclude = ["tools/yscope-dev-utils/**", "build/**"] + +[format] + +[lint] +[lint.rules] +dotted-keys-out-of-order = "warn" +key-empty = "warn" +tables-out-of-order = "warn" + +[schema] +enabled = true +catalog = { + paths = [ + "tombi://json.schemastore.org/api/json/catalog.json", + "https://json.schemastore.org/api/json/catalog.json", + ], +} + +[[schemas]] +path = "tombi://json.schemastore.org/tombi.json" +include = ["tombi.toml"] + +[[schemas]] +path = "https://json.schemastore.org/pyproject.json" +include = ["pyproject.toml"] + +[[schemas]] +path = "https://json.schemastore.org/ruff.json" +include = ["ruff.toml"] From c98fea18e1a3f4134067c8e5648f970bdaba6fe2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 10:07:59 -0400 Subject: [PATCH 38/91] Rename cpp build tasks --- build-tasks.yaml | 4 ++-- test-tasks.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build-tasks.yaml b/build-tasks.yaml index 666576f2..53828027 100644 --- a/build-tasks.yaml +++ b/build-tasks.yaml @@ -1,7 +1,7 @@ version: "3" tasks: - target: + cpp-target: internal: true vars: TARGETS: @@ -14,7 +14,7 @@ tasks: --parallel {{numCPU}} --target {{range .TARGETS}}{{.}} {{end}} - clean: + cpp-clean: internal: true deps: [":config-cmake-project"] cmds: diff --git a/test-tasks.yaml b/test-tasks.yaml index 41dfc457..58a1ff19 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -27,7 +27,7 @@ tasks: build-unit-test: internal: true deps: - - task: ":build:target" + - task: ":build:cpp-target" vars: TARGETS: ["spider_task_executor", "unitTest", "worker_test"] @@ -35,7 +35,7 @@ tasks: dir: "{{.G_BUILD_SPIDER_DIR}}" deps: - "venv" - - task: ":build:target" + - task: ":build:cpp-target" vars: TARGETS: [ "spider_task_executor", From 4e7237ea5775299f94728d674c4675d9b5dae6b2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 14:54:01 -0400 Subject: [PATCH 39/91] Add basic python structure --- python/.python-version | 1 + python/README.md | 0 python/pyproject.toml | 72 +++++++ python/src/__init__.py | 0 python/src/client/__init__.py | 0 python/src/core/__init__.py | 0 python/src/core/task.py | 0 python/src/task_executor/__init__.py | 0 python/src/task_executor/task_executor.py | 6 + python/uv.lock | 242 ++++++++++++++++++++++ 10 files changed, 321 insertions(+) create mode 100644 python/.python-version create mode 100644 python/README.md create mode 100644 python/pyproject.toml create mode 100644 python/src/__init__.py create mode 100644 python/src/client/__init__.py create mode 100644 python/src/core/__init__.py create mode 100644 python/src/core/task.py create mode 100644 python/src/task_executor/__init__.py create mode 100644 python/src/task_executor/task_executor.py create mode 100644 python/uv.lock diff --git a/python/.python-version b/python/.python-version new file mode 100644 index 00000000..24ee5b1b --- /dev/null +++ b/python/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/python/README.md b/python/README.md new file mode 100644 index 00000000..e69de29b diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 00000000..c8d87580 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,72 @@ +[project] +name = "spider" +version = "0.0.1" +description = "Spider is a distributed task execution framework" +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "mariadb>=1.1.13", + "msgpack>=1.1.1", +] + +[project.scritps] +spider_task_executor = "src.task_executor.task_executor:main" + +[build-system] +requires = ["hatchling>=1.18.0"] +build-backend = "hatchling.build" + +[tool.hatch] + +[tool.hatch.build.targets.wheel] +packages = ["src/client", "src/task_executor"] + +[dependency-groups] +dev = [ + "mypy>=1.12.0", + "ruff>=0.4.4", +] + +[tool.mypy] +strict = true + +# Additional output +pretty = true +show_error_code_links = true +show_error_context = true +show_error_end = true + +[[tool.mypy.overrides]] +module = "msgpack.*" +ignore_missing_imports = true + +[tool.ruff] +line-length = 100 + +[tool.ruff.lint] +select = ["ALL"] +extend-select = ["PT"] +ignore = [ + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful +] +isort.order-by-type = false + +[tool.ruff.format] +docstring-code-format = true +docstring-code-line-length = 100 diff --git a/python/src/__init__.py b/python/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/src/client/__init__.py b/python/src/client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/src/core/__init__.py b/python/src/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/src/core/task.py b/python/src/core/task.py new file mode 100644 index 00000000..e69de29b diff --git a/python/src/task_executor/__init__.py b/python/src/task_executor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/src/task_executor/task_executor.py b/python/src/task_executor/task_executor.py new file mode 100644 index 00000000..b0c61da3 --- /dev/null +++ b/python/src/task_executor/task_executor.py @@ -0,0 +1,6 @@ +def main() -> None: + """Main function to execute the task.""" + + +if __name__ == "__main__": + main() diff --git a/python/uv.lock b/python/uv.lock new file mode 100644 index 00000000..31a72ace --- /dev/null +++ b/python/uv.lock @@ -0,0 +1,242 @@ +version = 1 +revision = 2 +requires-python = ">=3.10" + +[[package]] +name = "mariadb" +version = "1.1.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/a0/0498c90f46940b6fd7f5c76d22df9f011849964e94e0b22fde26c42191c7/mariadb-1.1.13.tar.gz", hash = "sha256:3a0fd24fae2b9990dc13a0a427a43d4a5434cc2a8c3b260a27f40b7824719037", size = 111126, upload-time = "2025-07-11T09:55:11.147Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/e2/5e4068b225ccbd6fb7ea2b8f5ba52060e48bf2e63448206670cdfe73a1c5/mariadb-1.1.13-cp310-cp310-win32.whl", hash = "sha256:f49467e7076819ccf21a7e5ef030761af2160b23ce4484d88d260459809d90ff", size = 185138, upload-time = "2025-07-11T09:54:53.334Z" }, + { url = "https://files.pythonhosted.org/packages/33/8d/65b28ce61c422aba68c052a3cdb23772003267f47438a27a6e2cbf58c1c9/mariadb-1.1.13-cp310-cp310-win_amd64.whl", hash = "sha256:1c9aa9a31f943737c3a58bb4dc9deb9cf2b50b3ec807f90c5c99c7b35f8442bd", size = 202048, upload-time = "2025-07-11T09:54:54.978Z" }, + { url = "https://files.pythonhosted.org/packages/09/71/c23f6ffb06ba2c0ad2e90f4236b1541f9005e24eaeb166800b19c4f584f6/mariadb-1.1.13-cp311-cp311-win32.whl", hash = "sha256:c15ab872f1e951dd1bd4006d2ee2b942156f66c1cd724a1b2992dc02c47dff6a", size = 185136, upload-time = "2025-07-11T09:54:56.791Z" }, + { url = "https://files.pythonhosted.org/packages/c8/08/10308cb9bbccf83ff72caa1146014445a79dfeffb4eba9a85f73575c5959/mariadb-1.1.13-cp311-cp311-win_amd64.whl", hash = "sha256:d6fa7f2cdfae762d2d69b71e51f61415d0251118a622a6d06c7f05261579100a", size = 202046, upload-time = "2025-07-11T09:54:58.365Z" }, + { url = "https://files.pythonhosted.org/packages/ad/b4/d961a5dfa8a7a04ede70f76f85b951b8d0ec28d79b86ced829a67af78bdd/mariadb-1.1.13-cp312-cp312-win32.whl", hash = "sha256:358ecdc8775f4a97357fc233bb84a1c8d8d52866ee5fe98f2d13e9e58a04b6fb", size = 185334, upload-time = "2025-07-11T09:54:59.901Z" }, + { url = "https://files.pythonhosted.org/packages/0a/3b/16cc69a9c90f91b74c096c0cb9634e14692486beb28b82ec5efd97866de7/mariadb-1.1.13-cp312-cp312-win_amd64.whl", hash = "sha256:0604c57e6f90ff7ef190a56894d2b32ca5f0182924b5159be9319ba243ffbf98", size = 202109, upload-time = "2025-07-11T09:55:01.846Z" }, + { url = "https://files.pythonhosted.org/packages/02/16/583719e127774294d4af085a063b38f20407801cd9194a676334fb121edd/mariadb-1.1.13-cp313-cp313-win32.whl", hash = "sha256:36c644c13ecf3e38cb89f37f9bd054703760b94c881476492fe789801a063ce3", size = 185301, upload-time = "2025-07-11T09:55:03.486Z" }, + { url = "https://files.pythonhosted.org/packages/1b/da/4170f5532813ffba2641f821ed23ed1f22fa3ff8d603e47c91b0776ff9b7/mariadb-1.1.13-cp313-cp313-win_amd64.whl", hash = "sha256:6ec51ad2e33312c397cfcc5e2bca59861f572d5e8880e025d43db9a5d472a3a9", size = 202105, upload-time = "2025-07-11T09:55:05.681Z" }, +] + +[[package]] +name = "msgpack" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/b1/ea4f68038a18c77c9467400d166d74c4ffa536f34761f7983a104357e614/msgpack-1.1.1.tar.gz", hash = "sha256:77b79ce34a2bdab2594f490c8e80dd62a02d650b91a75159a63ec413b8d104cd", size = 173555, upload-time = "2025-06-13T06:52:51.324Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/52/f30da112c1dc92cf64f57d08a273ac771e7b29dea10b4b30369b2d7e8546/msgpack-1.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:353b6fc0c36fde68b661a12949d7d49f8f51ff5fa019c1e47c87c4ff34b080ed", size = 81799, upload-time = "2025-06-13T06:51:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/e4/35/7bfc0def2f04ab4145f7f108e3563f9b4abae4ab0ed78a61f350518cc4d2/msgpack-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:79c408fcf76a958491b4e3b103d1c417044544b68e96d06432a189b43d1215c8", size = 78278, upload-time = "2025-06-13T06:51:38.534Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c5/df5d6c1c39856bc55f800bf82778fd4c11370667f9b9e9d51b2f5da88f20/msgpack-1.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78426096939c2c7482bf31ef15ca219a9e24460289c00dd0b94411040bb73ad2", size = 402805, upload-time = "2025-06-13T06:51:39.538Z" }, + { url = "https://files.pythonhosted.org/packages/20/8e/0bb8c977efecfe6ea7116e2ed73a78a8d32a947f94d272586cf02a9757db/msgpack-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b17ba27727a36cb73aabacaa44b13090feb88a01d012c0f4be70c00f75048b4", size = 408642, upload-time = "2025-06-13T06:51:41.092Z" }, + { url = "https://files.pythonhosted.org/packages/59/a1/731d52c1aeec52006be6d1f8027c49fdc2cfc3ab7cbe7c28335b2910d7b6/msgpack-1.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a17ac1ea6ec3c7687d70201cfda3b1e8061466f28f686c24f627cae4ea8efd0", size = 395143, upload-time = "2025-06-13T06:51:42.575Z" }, + { url = "https://files.pythonhosted.org/packages/2b/92/b42911c52cda2ba67a6418ffa7d08969edf2e760b09015593c8a8a27a97d/msgpack-1.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:88d1e966c9235c1d4e2afac21ca83933ba59537e2e2727a999bf3f515ca2af26", size = 395986, upload-time = "2025-06-13T06:51:43.807Z" }, + { url = "https://files.pythonhosted.org/packages/61/dc/8ae165337e70118d4dab651b8b562dd5066dd1e6dd57b038f32ebc3e2f07/msgpack-1.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f6d58656842e1b2ddbe07f43f56b10a60f2ba5826164910968f5933e5178af75", size = 402682, upload-time = "2025-06-13T06:51:45.534Z" }, + { url = "https://files.pythonhosted.org/packages/58/27/555851cb98dcbd6ce041df1eacb25ac30646575e9cd125681aa2f4b1b6f1/msgpack-1.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:96decdfc4adcbc087f5ea7ebdcfd3dee9a13358cae6e81d54be962efc38f6338", size = 406368, upload-time = "2025-06-13T06:51:46.97Z" }, + { url = "https://files.pythonhosted.org/packages/d4/64/39a26add4ce16f24e99eabb9005e44c663db00e3fce17d4ae1ae9d61df99/msgpack-1.1.1-cp310-cp310-win32.whl", hash = "sha256:6640fd979ca9a212e4bcdf6eb74051ade2c690b862b679bfcb60ae46e6dc4bfd", size = 65004, upload-time = "2025-06-13T06:51:48.582Z" }, + { url = "https://files.pythonhosted.org/packages/7d/18/73dfa3e9d5d7450d39debde5b0d848139f7de23bd637a4506e36c9800fd6/msgpack-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:8b65b53204fe1bd037c40c4148d00ef918eb2108d24c9aaa20bc31f9810ce0a8", size = 71548, upload-time = "2025-06-13T06:51:49.558Z" }, + { url = "https://files.pythonhosted.org/packages/7f/83/97f24bf9848af23fe2ba04380388216defc49a8af6da0c28cc636d722502/msgpack-1.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:71ef05c1726884e44f8b1d1773604ab5d4d17729d8491403a705e649116c9558", size = 82728, upload-time = "2025-06-13T06:51:50.68Z" }, + { url = "https://files.pythonhosted.org/packages/aa/7f/2eaa388267a78401f6e182662b08a588ef4f3de6f0eab1ec09736a7aaa2b/msgpack-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:36043272c6aede309d29d56851f8841ba907a1a3d04435e43e8a19928e243c1d", size = 79279, upload-time = "2025-06-13T06:51:51.72Z" }, + { url = "https://files.pythonhosted.org/packages/f8/46/31eb60f4452c96161e4dfd26dbca562b4ec68c72e4ad07d9566d7ea35e8a/msgpack-1.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a32747b1b39c3ac27d0670122b57e6e57f28eefb725e0b625618d1b59bf9d1e0", size = 423859, upload-time = "2025-06-13T06:51:52.749Z" }, + { url = "https://files.pythonhosted.org/packages/45/16/a20fa8c32825cc7ae8457fab45670c7a8996d7746ce80ce41cc51e3b2bd7/msgpack-1.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a8b10fdb84a43e50d38057b06901ec9da52baac6983d3f709d8507f3889d43f", size = 429975, upload-time = "2025-06-13T06:51:53.97Z" }, + { url = "https://files.pythonhosted.org/packages/86/ea/6c958e07692367feeb1a1594d35e22b62f7f476f3c568b002a5ea09d443d/msgpack-1.1.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba0c325c3f485dc54ec298d8b024e134acf07c10d494ffa24373bea729acf704", size = 413528, upload-time = "2025-06-13T06:51:55.507Z" }, + { url = "https://files.pythonhosted.org/packages/75/05/ac84063c5dae79722bda9f68b878dc31fc3059adb8633c79f1e82c2cd946/msgpack-1.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:88daaf7d146e48ec71212ce21109b66e06a98e5e44dca47d853cbfe171d6c8d2", size = 413338, upload-time = "2025-06-13T06:51:57.023Z" }, + { url = "https://files.pythonhosted.org/packages/69/e8/fe86b082c781d3e1c09ca0f4dacd457ede60a13119b6ce939efe2ea77b76/msgpack-1.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d8b55ea20dc59b181d3f47103f113e6f28a5e1c89fd5b67b9140edb442ab67f2", size = 422658, upload-time = "2025-06-13T06:51:58.419Z" }, + { url = "https://files.pythonhosted.org/packages/3b/2b/bafc9924df52d8f3bb7c00d24e57be477f4d0f967c0a31ef5e2225e035c7/msgpack-1.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4a28e8072ae9779f20427af07f53bbb8b4aa81151054e882aee333b158da8752", size = 427124, upload-time = "2025-06-13T06:51:59.969Z" }, + { url = "https://files.pythonhosted.org/packages/a2/3b/1f717e17e53e0ed0b68fa59e9188f3f610c79d7151f0e52ff3cd8eb6b2dc/msgpack-1.1.1-cp311-cp311-win32.whl", hash = "sha256:7da8831f9a0fdb526621ba09a281fadc58ea12701bc709e7b8cbc362feabc295", size = 65016, upload-time = "2025-06-13T06:52:01.294Z" }, + { url = "https://files.pythonhosted.org/packages/48/45/9d1780768d3b249accecc5a38c725eb1e203d44a191f7b7ff1941f7df60c/msgpack-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:5fd1b58e1431008a57247d6e7cc4faa41c3607e8e7d4aaf81f7c29ea013cb458", size = 72267, upload-time = "2025-06-13T06:52:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/389b9c593eda2b8551b2e7126ad3a06af6f9b44274eb3a4f054d48ff7e47/msgpack-1.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ae497b11f4c21558d95de9f64fff7053544f4d1a17731c866143ed6bb4591238", size = 82359, upload-time = "2025-06-13T06:52:03.909Z" }, + { url = "https://files.pythonhosted.org/packages/ab/65/7d1de38c8a22cf8b1551469159d4b6cf49be2126adc2482de50976084d78/msgpack-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:33be9ab121df9b6b461ff91baac6f2731f83d9b27ed948c5b9d1978ae28bf157", size = 79172, upload-time = "2025-06-13T06:52:05.246Z" }, + { url = "https://files.pythonhosted.org/packages/0f/bd/cacf208b64d9577a62c74b677e1ada005caa9b69a05a599889d6fc2ab20a/msgpack-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f64ae8fe7ffba251fecb8408540c34ee9df1c26674c50c4544d72dbf792e5ce", size = 425013, upload-time = "2025-06-13T06:52:06.341Z" }, + { url = "https://files.pythonhosted.org/packages/4d/ec/fd869e2567cc9c01278a736cfd1697941ba0d4b81a43e0aa2e8d71dab208/msgpack-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a494554874691720ba5891c9b0b39474ba43ffb1aaf32a5dac874effb1619e1a", size = 426905, upload-time = "2025-06-13T06:52:07.501Z" }, + { url = "https://files.pythonhosted.org/packages/55/2a/35860f33229075bce803a5593d046d8b489d7ba2fc85701e714fc1aaf898/msgpack-1.1.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb643284ab0ed26f6957d969fe0dd8bb17beb567beb8998140b5e38a90974f6c", size = 407336, upload-time = "2025-06-13T06:52:09.047Z" }, + { url = "https://files.pythonhosted.org/packages/8c/16/69ed8f3ada150bf92745fb4921bd621fd2cdf5a42e25eb50bcc57a5328f0/msgpack-1.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d275a9e3c81b1093c060c3837e580c37f47c51eca031f7b5fb76f7b8470f5f9b", size = 409485, upload-time = "2025-06-13T06:52:10.382Z" }, + { url = "https://files.pythonhosted.org/packages/c6/b6/0c398039e4c6d0b2e37c61d7e0e9d13439f91f780686deb8ee64ecf1ae71/msgpack-1.1.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fd6b577e4541676e0cc9ddc1709d25014d3ad9a66caa19962c4f5de30fc09ef", size = 412182, upload-time = "2025-06-13T06:52:11.644Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d0/0cf4a6ecb9bc960d624c93effaeaae75cbf00b3bc4a54f35c8507273cda1/msgpack-1.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb29aaa613c0a1c40d1af111abf025f1732cab333f96f285d6a93b934738a68a", size = 419883, upload-time = "2025-06-13T06:52:12.806Z" }, + { url = "https://files.pythonhosted.org/packages/62/83/9697c211720fa71a2dfb632cad6196a8af3abea56eece220fde4674dc44b/msgpack-1.1.1-cp312-cp312-win32.whl", hash = "sha256:870b9a626280c86cff9c576ec0d9cbcc54a1e5ebda9cd26dab12baf41fee218c", size = 65406, upload-time = "2025-06-13T06:52:14.271Z" }, + { url = "https://files.pythonhosted.org/packages/c0/23/0abb886e80eab08f5e8c485d6f13924028602829f63b8f5fa25a06636628/msgpack-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:5692095123007180dca3e788bb4c399cc26626da51629a31d40207cb262e67f4", size = 72558, upload-time = "2025-06-13T06:52:15.252Z" }, + { url = "https://files.pythonhosted.org/packages/a1/38/561f01cf3577430b59b340b51329803d3a5bf6a45864a55f4ef308ac11e3/msgpack-1.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3765afa6bd4832fc11c3749be4ba4b69a0e8d7b728f78e68120a157a4c5d41f0", size = 81677, upload-time = "2025-06-13T06:52:16.64Z" }, + { url = "https://files.pythonhosted.org/packages/09/48/54a89579ea36b6ae0ee001cba8c61f776451fad3c9306cd80f5b5c55be87/msgpack-1.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8ddb2bcfd1a8b9e431c8d6f4f7db0773084e107730ecf3472f1dfe9ad583f3d9", size = 78603, upload-time = "2025-06-13T06:52:17.843Z" }, + { url = "https://files.pythonhosted.org/packages/a0/60/daba2699b308e95ae792cdc2ef092a38eb5ee422f9d2fbd4101526d8a210/msgpack-1.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:196a736f0526a03653d829d7d4c5500a97eea3648aebfd4b6743875f28aa2af8", size = 420504, upload-time = "2025-06-13T06:52:18.982Z" }, + { url = "https://files.pythonhosted.org/packages/20/22/2ebae7ae43cd8f2debc35c631172ddf14e2a87ffcc04cf43ff9df9fff0d3/msgpack-1.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d592d06e3cc2f537ceeeb23d38799c6ad83255289bb84c2e5792e5a8dea268a", size = 423749, upload-time = "2025-06-13T06:52:20.211Z" }, + { url = "https://files.pythonhosted.org/packages/40/1b/54c08dd5452427e1179a40b4b607e37e2664bca1c790c60c442c8e972e47/msgpack-1.1.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4df2311b0ce24f06ba253fda361f938dfecd7b961576f9be3f3fbd60e87130ac", size = 404458, upload-time = "2025-06-13T06:52:21.429Z" }, + { url = "https://files.pythonhosted.org/packages/2e/60/6bb17e9ffb080616a51f09928fdd5cac1353c9becc6c4a8abd4e57269a16/msgpack-1.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e4141c5a32b5e37905b5940aacbc59739f036930367d7acce7a64e4dec1f5e0b", size = 405976, upload-time = "2025-06-13T06:52:22.995Z" }, + { url = "https://files.pythonhosted.org/packages/ee/97/88983e266572e8707c1f4b99c8fd04f9eb97b43f2db40e3172d87d8642db/msgpack-1.1.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b1ce7f41670c5a69e1389420436f41385b1aa2504c3b0c30620764b15dded2e7", size = 408607, upload-time = "2025-06-13T06:52:24.152Z" }, + { url = "https://files.pythonhosted.org/packages/bc/66/36c78af2efaffcc15a5a61ae0df53a1d025f2680122e2a9eb8442fed3ae4/msgpack-1.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4147151acabb9caed4e474c3344181e91ff7a388b888f1e19ea04f7e73dc7ad5", size = 424172, upload-time = "2025-06-13T06:52:25.704Z" }, + { url = "https://files.pythonhosted.org/packages/8c/87/a75eb622b555708fe0427fab96056d39d4c9892b0c784b3a721088c7ee37/msgpack-1.1.1-cp313-cp313-win32.whl", hash = "sha256:500e85823a27d6d9bba1d057c871b4210c1dd6fb01fbb764e37e4e8847376323", size = 65347, upload-time = "2025-06-13T06:52:26.846Z" }, + { url = "https://files.pythonhosted.org/packages/ca/91/7dc28d5e2a11a5ad804cf2b7f7a5fcb1eb5a4966d66a5d2b41aee6376543/msgpack-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:6d489fba546295983abd142812bda76b57e33d0b9f5d5b71c09a583285506f69", size = 72341, upload-time = "2025-06-13T06:52:27.835Z" }, +] + +[[package]] +name = "mypy" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/22/ea637422dedf0bf36f3ef238eab4e455e2a0dcc3082b5cc067615347ab8e/mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01", size = 3352570, upload-time = "2025-07-31T07:54:19.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/a9/3d7aa83955617cdf02f94e50aab5c830d205cfa4320cf124ff64acce3a8e/mypy-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3fbe6d5555bf608c47203baa3e72dbc6ec9965b3d7c318aa9a4ca76f465bd972", size = 11003299, upload-time = "2025-07-31T07:54:06.425Z" }, + { url = "https://files.pythonhosted.org/packages/83/e8/72e62ff837dd5caaac2b4a5c07ce769c8e808a00a65e5d8f94ea9c6f20ab/mypy-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80ef5c058b7bce08c83cac668158cb7edea692e458d21098c7d3bce35a5d43e7", size = 10125451, upload-time = "2025-07-31T07:53:52.974Z" }, + { url = "https://files.pythonhosted.org/packages/7d/10/f3f3543f6448db11881776f26a0ed079865926b0c841818ee22de2c6bbab/mypy-1.17.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a580f8a70c69e4a75587bd925d298434057fe2a428faaf927ffe6e4b9a98df", size = 11916211, upload-time = "2025-07-31T07:53:18.879Z" }, + { url = "https://files.pythonhosted.org/packages/06/bf/63e83ed551282d67bb3f7fea2cd5561b08d2bb6eb287c096539feb5ddbc5/mypy-1.17.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd86bb649299f09d987a2eebb4d52d10603224500792e1bee18303bbcc1ce390", size = 12652687, upload-time = "2025-07-31T07:53:30.544Z" }, + { url = "https://files.pythonhosted.org/packages/69/66/68f2eeef11facf597143e85b694a161868b3b006a5fbad50e09ea117ef24/mypy-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a76906f26bd8d51ea9504966a9c25419f2e668f012e0bdf3da4ea1526c534d94", size = 12896322, upload-time = "2025-07-31T07:53:50.74Z" }, + { url = "https://files.pythonhosted.org/packages/a3/87/8e3e9c2c8bd0d7e071a89c71be28ad088aaecbadf0454f46a540bda7bca6/mypy-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:e79311f2d904ccb59787477b7bd5d26f3347789c06fcd7656fa500875290264b", size = 9507962, upload-time = "2025-07-31T07:53:08.431Z" }, + { url = "https://files.pythonhosted.org/packages/46/cf/eadc80c4e0a70db1c08921dcc220357ba8ab2faecb4392e3cebeb10edbfa/mypy-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad37544be07c5d7fba814eb370e006df58fed8ad1ef33ed1649cb1889ba6ff58", size = 10921009, upload-time = "2025-07-31T07:53:23.037Z" }, + { url = "https://files.pythonhosted.org/packages/5d/c1/c869d8c067829ad30d9bdae051046561552516cfb3a14f7f0347b7d973ee/mypy-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:064e2ff508e5464b4bd807a7c1625bc5047c5022b85c70f030680e18f37273a5", size = 10047482, upload-time = "2025-07-31T07:53:26.151Z" }, + { url = "https://files.pythonhosted.org/packages/98/b9/803672bab3fe03cee2e14786ca056efda4bb511ea02dadcedde6176d06d0/mypy-1.17.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70401bbabd2fa1aa7c43bb358f54037baf0586f41e83b0ae67dd0534fc64edfd", size = 11832883, upload-time = "2025-07-31T07:53:47.948Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/fcdac695beca66800918c18697b48833a9a6701de288452b6715a98cfee1/mypy-1.17.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e92bdc656b7757c438660f775f872a669b8ff374edc4d18277d86b63edba6b8b", size = 12566215, upload-time = "2025-07-31T07:54:04.031Z" }, + { url = "https://files.pythonhosted.org/packages/7f/37/a932da3d3dace99ee8eb2043b6ab03b6768c36eb29a02f98f46c18c0da0e/mypy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c1fdf4abb29ed1cb091cf432979e162c208a5ac676ce35010373ff29247bcad5", size = 12751956, upload-time = "2025-07-31T07:53:36.263Z" }, + { url = "https://files.pythonhosted.org/packages/8c/cf/6438a429e0f2f5cab8bc83e53dbebfa666476f40ee322e13cac5e64b79e7/mypy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:ff2933428516ab63f961644bc49bc4cbe42bbffb2cd3b71cc7277c07d16b1a8b", size = 9507307, upload-time = "2025-07-31T07:53:59.734Z" }, + { url = "https://files.pythonhosted.org/packages/17/a2/7034d0d61af8098ec47902108553122baa0f438df8a713be860f7407c9e6/mypy-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:69e83ea6553a3ba79c08c6e15dbd9bfa912ec1e493bf75489ef93beb65209aeb", size = 11086295, upload-time = "2025-07-31T07:53:28.124Z" }, + { url = "https://files.pythonhosted.org/packages/14/1f/19e7e44b594d4b12f6ba8064dbe136505cec813549ca3e5191e40b1d3cc2/mypy-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b16708a66d38abb1e6b5702f5c2c87e133289da36f6a1d15f6a5221085c6403", size = 10112355, upload-time = "2025-07-31T07:53:21.121Z" }, + { url = "https://files.pythonhosted.org/packages/5b/69/baa33927e29e6b4c55d798a9d44db5d394072eef2bdc18c3e2048c9ed1e9/mypy-1.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89e972c0035e9e05823907ad5398c5a73b9f47a002b22359b177d40bdaee7056", size = 11875285, upload-time = "2025-07-31T07:53:55.293Z" }, + { url = "https://files.pythonhosted.org/packages/90/13/f3a89c76b0a41e19490b01e7069713a30949d9a6c147289ee1521bcea245/mypy-1.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03b6d0ed2b188e35ee6d5c36b5580cffd6da23319991c49ab5556c023ccf1341", size = 12737895, upload-time = "2025-07-31T07:53:43.623Z" }, + { url = "https://files.pythonhosted.org/packages/23/a1/c4ee79ac484241301564072e6476c5a5be2590bc2e7bfd28220033d2ef8f/mypy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c837b896b37cd103570d776bda106eabb8737aa6dd4f248451aecf53030cdbeb", size = 12931025, upload-time = "2025-07-31T07:54:17.125Z" }, + { url = "https://files.pythonhosted.org/packages/89/b8/7409477be7919a0608900e6320b155c72caab4fef46427c5cc75f85edadd/mypy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:665afab0963a4b39dff7c1fa563cc8b11ecff7910206db4b2e64dd1ba25aed19", size = 9584664, upload-time = "2025-07-31T07:54:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/5b/82/aec2fc9b9b149f372850291827537a508d6c4d3664b1750a324b91f71355/mypy-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93378d3203a5c0800c6b6d850ad2f19f7a3cdf1a3701d3416dbf128805c6a6a7", size = 11075338, upload-time = "2025-07-31T07:53:38.873Z" }, + { url = "https://files.pythonhosted.org/packages/07/ac/ee93fbde9d2242657128af8c86f5d917cd2887584cf948a8e3663d0cd737/mypy-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15d54056f7fe7a826d897789f53dd6377ec2ea8ba6f776dc83c2902b899fee81", size = 10113066, upload-time = "2025-07-31T07:54:14.707Z" }, + { url = "https://files.pythonhosted.org/packages/5a/68/946a1e0be93f17f7caa56c45844ec691ca153ee8b62f21eddda336a2d203/mypy-1.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:209a58fed9987eccc20f2ca94afe7257a8f46eb5df1fb69958650973230f91e6", size = 11875473, upload-time = "2025-07-31T07:53:14.504Z" }, + { url = "https://files.pythonhosted.org/packages/9f/0f/478b4dce1cb4f43cf0f0d00fba3030b21ca04a01b74d1cd272a528cf446f/mypy-1.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:099b9a5da47de9e2cb5165e581f158e854d9e19d2e96b6698c0d64de911dd849", size = 12744296, upload-time = "2025-07-31T07:53:03.896Z" }, + { url = "https://files.pythonhosted.org/packages/ca/70/afa5850176379d1b303f992a828de95fc14487429a7139a4e0bdd17a8279/mypy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ffadfbe6994d724c5a1bb6123a7d27dd68fc9c059561cd33b664a79578e14", size = 12914657, upload-time = "2025-07-31T07:54:08.576Z" }, + { url = "https://files.pythonhosted.org/packages/53/f9/4a83e1c856a3d9c8f6edaa4749a4864ee98486e9b9dbfbc93842891029c2/mypy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:9a2b7d9180aed171f033c9f2fc6c204c1245cf60b0cb61cf2e7acc24eea78e0a", size = 9593320, upload-time = "2025-07-31T07:53:01.341Z" }, + { url = "https://files.pythonhosted.org/packages/38/56/79c2fac86da57c7d8c48622a05873eaab40b905096c33597462713f5af90/mypy-1.17.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:15a83369400454c41ed3a118e0cc58bd8123921a602f385cb6d6ea5df050c733", size = 11040037, upload-time = "2025-07-31T07:54:10.942Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c3/adabe6ff53638e3cad19e3547268482408323b1e68bf082c9119000cd049/mypy-1.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b918670f692fc9fba55c3298d8a3beae295c5cded0a55dccdc5bbead814acd", size = 10131550, upload-time = "2025-07-31T07:53:41.307Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c5/2e234c22c3bdeb23a7817af57a58865a39753bde52c74e2c661ee0cfc640/mypy-1.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62761474061feef6f720149d7ba876122007ddc64adff5ba6f374fda35a018a0", size = 11872963, upload-time = "2025-07-31T07:53:16.878Z" }, + { url = "https://files.pythonhosted.org/packages/ab/26/c13c130f35ca8caa5f2ceab68a247775648fdcd6c9a18f158825f2bc2410/mypy-1.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c49562d3d908fd49ed0938e5423daed8d407774a479b595b143a3d7f87cdae6a", size = 12710189, upload-time = "2025-07-31T07:54:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/c7d79d09f6de8383fe800521d066d877e54d30b4fb94281c262be2df84ef/mypy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:397fba5d7616a5bc60b45c7ed204717eaddc38f826e3645402c426057ead9a91", size = 12900322, upload-time = "2025-07-31T07:53:10.551Z" }, + { url = "https://files.pythonhosted.org/packages/b8/98/3d5a48978b4f708c55ae832619addc66d677f6dc59f3ebad71bae8285ca6/mypy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:9d6b20b97d373f41617bd0708fd46aa656059af57f2ef72aa8c7d6a2b73b74ed", size = 9751879, upload-time = "2025-07-31T07:52:56.683Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f3/8fcd2af0f5b806f6cf463efaffd3c9548a28f84220493ecd38d127b6b66d/mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9", size = 2283411, upload-time = "2025-07-31T07:53:24.664Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + +[[package]] +name = "ruff" +version = "0.12.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/81/0bd3594fa0f690466e41bd033bdcdf86cba8288345ac77ad4afbe5ec743a/ruff-0.12.7.tar.gz", hash = "sha256:1fc3193f238bc2d7968772c82831a4ff69252f673be371fb49663f0068b7ec71", size = 5197814, upload-time = "2025-07-29T22:32:35.877Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/d2/6cb35e9c85e7a91e8d22ab32ae07ac39cc34a71f1009a6f9e4a2a019e602/ruff-0.12.7-py3-none-linux_armv6l.whl", hash = "sha256:76e4f31529899b8c434c3c1dede98c4483b89590e15fb49f2d46183801565303", size = 11852189, upload-time = "2025-07-29T22:31:41.281Z" }, + { url = "https://files.pythonhosted.org/packages/63/5b/a4136b9921aa84638f1a6be7fb086f8cad0fde538ba76bda3682f2599a2f/ruff-0.12.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:789b7a03e72507c54fb3ba6209e4bb36517b90f1a3569ea17084e3fd295500fb", size = 12519389, upload-time = "2025-07-29T22:31:54.265Z" }, + { url = "https://files.pythonhosted.org/packages/a8/c9/3e24a8472484269b6b1821794141f879c54645a111ded4b6f58f9ab0705f/ruff-0.12.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2e1c2a3b8626339bb6369116e7030a4cf194ea48f49b64bb505732a7fce4f4e3", size = 11743384, upload-time = "2025-07-29T22:31:59.575Z" }, + { url = "https://files.pythonhosted.org/packages/26/7c/458dd25deeb3452c43eaee853c0b17a1e84169f8021a26d500ead77964fd/ruff-0.12.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32dec41817623d388e645612ec70d5757a6d9c035f3744a52c7b195a57e03860", size = 11943759, upload-time = "2025-07-29T22:32:01.95Z" }, + { url = "https://files.pythonhosted.org/packages/7f/8b/658798472ef260ca050e400ab96ef7e85c366c39cf3dfbef4d0a46a528b6/ruff-0.12.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47ef751f722053a5df5fa48d412dbb54d41ab9b17875c6840a58ec63ff0c247c", size = 11654028, upload-time = "2025-07-29T22:32:04.367Z" }, + { url = "https://files.pythonhosted.org/packages/a8/86/9c2336f13b2a3326d06d39178fd3448dcc7025f82514d1b15816fe42bfe8/ruff-0.12.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a828a5fc25a3efd3e1ff7b241fd392686c9386f20e5ac90aa9234a5faa12c423", size = 13225209, upload-time = "2025-07-29T22:32:06.952Z" }, + { url = "https://files.pythonhosted.org/packages/76/69/df73f65f53d6c463b19b6b312fd2391dc36425d926ec237a7ed028a90fc1/ruff-0.12.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5726f59b171111fa6a69d82aef48f00b56598b03a22f0f4170664ff4d8298efb", size = 14182353, upload-time = "2025-07-29T22:32:10.053Z" }, + { url = "https://files.pythonhosted.org/packages/58/1e/de6cda406d99fea84b66811c189b5ea139814b98125b052424b55d28a41c/ruff-0.12.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:74e6f5c04c4dd4aba223f4fe6e7104f79e0eebf7d307e4f9b18c18362124bccd", size = 13631555, upload-time = "2025-07-29T22:32:12.644Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ae/625d46d5164a6cc9261945a5e89df24457dc8262539ace3ac36c40f0b51e/ruff-0.12.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d0bfe4e77fba61bf2ccadf8cf005d6133e3ce08793bbe870dd1c734f2699a3e", size = 12667556, upload-time = "2025-07-29T22:32:15.312Z" }, + { url = "https://files.pythonhosted.org/packages/55/bf/9cb1ea5e3066779e42ade8d0cd3d3b0582a5720a814ae1586f85014656b6/ruff-0.12.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06bfb01e1623bf7f59ea749a841da56f8f653d641bfd046edee32ede7ff6c606", size = 12939784, upload-time = "2025-07-29T22:32:17.69Z" }, + { url = "https://files.pythonhosted.org/packages/55/7f/7ead2663be5627c04be83754c4f3096603bf5e99ed856c7cd29618c691bd/ruff-0.12.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e41df94a957d50083fd09b916d6e89e497246698c3f3d5c681c8b3e7b9bb4ac8", size = 11771356, upload-time = "2025-07-29T22:32:20.134Z" }, + { url = "https://files.pythonhosted.org/packages/17/40/a95352ea16edf78cd3a938085dccc55df692a4d8ba1b3af7accbe2c806b0/ruff-0.12.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:4000623300563c709458d0ce170c3d0d788c23a058912f28bbadc6f905d67afa", size = 11612124, upload-time = "2025-07-29T22:32:22.645Z" }, + { url = "https://files.pythonhosted.org/packages/4d/74/633b04871c669e23b8917877e812376827c06df866e1677f15abfadc95cb/ruff-0.12.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:69ffe0e5f9b2cf2b8e289a3f8945b402a1b19eff24ec389f45f23c42a3dd6fb5", size = 12479945, upload-time = "2025-07-29T22:32:24.765Z" }, + { url = "https://files.pythonhosted.org/packages/be/34/c3ef2d7799c9778b835a76189c6f53c179d3bdebc8c65288c29032e03613/ruff-0.12.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a07a5c8ffa2611a52732bdc67bf88e243abd84fe2d7f6daef3826b59abbfeda4", size = 12998677, upload-time = "2025-07-29T22:32:27.022Z" }, + { url = "https://files.pythonhosted.org/packages/77/ab/aca2e756ad7b09b3d662a41773f3edcbd262872a4fc81f920dc1ffa44541/ruff-0.12.7-py3-none-win32.whl", hash = "sha256:c928f1b2ec59fb77dfdf70e0419408898b63998789cc98197e15f560b9e77f77", size = 11756687, upload-time = "2025-07-29T22:32:29.381Z" }, + { url = "https://files.pythonhosted.org/packages/b4/71/26d45a5042bc71db22ddd8252ca9d01e9ca454f230e2996bb04f16d72799/ruff-0.12.7-py3-none-win_amd64.whl", hash = "sha256:9c18f3d707ee9edf89da76131956aba1270c6348bfee8f6c647de841eac7194f", size = 12912365, upload-time = "2025-07-29T22:32:31.517Z" }, + { url = "https://files.pythonhosted.org/packages/4c/9b/0b8aa09817b63e78d94b4977f18b1fcaead3165a5ee49251c5d5c245bb2d/ruff-0.12.7-py3-none-win_arm64.whl", hash = "sha256:dfce05101dbd11833a0776716d5d1578641b7fddb537fe7fa956ab85d1769b69", size = 11982083, upload-time = "2025-07-29T22:32:33.881Z" }, +] + +[[package]] +name = "spider" +version = "0.0.1" +source = { virtual = "." } +dependencies = [ + { name = "mariadb" }, + { name = "msgpack" }, +] + +[package.dev-dependencies] +dev = [ + { name = "mypy" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "mariadb", specifier = ">=1.1.13" }, + { name = "msgpack", specifier = ">=1.1.1" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "mypy", specifier = ">=1.12.0" }, + { name = "ruff", specifier = ">=0.4.4" }, +] + +[[package]] +name = "tomli" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077, upload-time = "2024-11-27T22:37:54.956Z" }, + { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429, upload-time = "2024-11-27T22:37:56.698Z" }, + { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067, upload-time = "2024-11-27T22:37:57.63Z" }, + { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030, upload-time = "2024-11-27T22:37:59.344Z" }, + { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898, upload-time = "2024-11-27T22:38:00.429Z" }, + { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894, upload-time = "2024-11-27T22:38:02.094Z" }, + { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319, upload-time = "2024-11-27T22:38:03.206Z" }, + { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273, upload-time = "2024-11-27T22:38:04.217Z" }, + { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310, upload-time = "2024-11-27T22:38:05.908Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309, upload-time = "2024-11-27T22:38:06.812Z" }, + { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762, upload-time = "2024-11-27T22:38:07.731Z" }, + { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453, upload-time = "2024-11-27T22:38:09.384Z" }, + { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486, upload-time = "2024-11-27T22:38:10.329Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349, upload-time = "2024-11-27T22:38:11.443Z" }, + { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159, upload-time = "2024-11-27T22:38:13.099Z" }, + { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243, upload-time = "2024-11-27T22:38:14.766Z" }, + { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645, upload-time = "2024-11-27T22:38:15.843Z" }, + { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584, upload-time = "2024-11-27T22:38:17.645Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875, upload-time = "2024-11-27T22:38:19.159Z" }, + { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418, upload-time = "2024-11-27T22:38:20.064Z" }, + { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708, upload-time = "2024-11-27T22:38:21.659Z" }, + { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582, upload-time = "2024-11-27T22:38:22.693Z" }, + { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543, upload-time = "2024-11-27T22:38:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691, upload-time = "2024-11-27T22:38:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170, upload-time = "2024-11-27T22:38:27.921Z" }, + { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530, upload-time = "2024-11-27T22:38:29.591Z" }, + { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666, upload-time = "2024-11-27T22:38:30.639Z" }, + { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954, upload-time = "2024-11-27T22:38:31.702Z" }, + { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724, upload-time = "2024-11-27T22:38:32.837Z" }, + { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383, upload-time = "2024-11-27T22:38:34.455Z" }, + { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673, upload-time = "2025-07-04T13:28:34.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" }, +] From 1f944d058288587c891c5910c41d4469a4d7f5cd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 14:56:46 -0400 Subject: [PATCH 40/91] Fix code structure --- python/src/__init__.py | 0 python/uv.lock | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 python/src/__init__.py diff --git a/python/src/__init__.py b/python/src/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/python/uv.lock b/python/uv.lock index 31a72ace..330a5aaa 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -169,7 +169,7 @@ wheels = [ [[package]] name = "spider" version = "0.0.1" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "mariadb" }, { name = "msgpack" }, From 68992fcd4daac3ca8e93b897aa8b8fa8f921e469 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 15:00:38 -0400 Subject: [PATCH 41/91] Fix ruff lint --- python/src/client/__init__.py | 1 + python/src/core/__init__.py | 1 + python/src/core/task.py | 5 +++++ python/src/task_executor/__init__.py | 1 + python/src/task_executor/task_executor.py | 3 +++ 5 files changed, 11 insertions(+) diff --git a/python/src/client/__init__.py b/python/src/client/__init__.py index e69de29b..03845614 100644 --- a/python/src/client/__init__.py +++ b/python/src/client/__init__.py @@ -0,0 +1 @@ +"""Spider python client.""" diff --git a/python/src/core/__init__.py b/python/src/core/__init__.py index e69de29b..e4e40894 100644 --- a/python/src/core/__init__.py +++ b/python/src/core/__init__.py @@ -0,0 +1 @@ +"""Spider core module.""" diff --git a/python/src/core/task.py b/python/src/core/task.py index e69de29b..855f2a72 100644 --- a/python/src/core/task.py +++ b/python/src/core/task.py @@ -0,0 +1,5 @@ +"""Task module for Spider.""" + + +class Task: + """Represents a task in Spider.""" diff --git a/python/src/task_executor/__init__.py b/python/src/task_executor/__init__.py index e69de29b..03195404 100644 --- a/python/src/task_executor/__init__.py +++ b/python/src/task_executor/__init__.py @@ -0,0 +1 @@ +"""Python task executor for running Python task function.""" diff --git a/python/src/task_executor/task_executor.py b/python/src/task_executor/task_executor.py index b0c61da3..c8a0593d 100644 --- a/python/src/task_executor/task_executor.py +++ b/python/src/task_executor/task_executor.py @@ -1,3 +1,6 @@ +"""Executes a Spider Python task.""" + + def main() -> None: """Main function to execute the task.""" From 1127b2215bd735aa8b8205d5e1443d17efafcb1e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 15:06:00 -0400 Subject: [PATCH 42/91] Extend lint tasks to python directory --- lint-tasks.yaml | 5 +++++ taskfile.yaml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 40259b21..270a121c 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -135,6 +135,11 @@ tasks: mypy "{{.ITEM}}" ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}" ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}" + - cmd: |- + cd "{{.G_SRC_PYTHON_DIR}}" + uv run mypy . + uv run ruff check {{.RUFF_CHECK_FLAGS}} + uv run ruff format {{.RUFF_FORMAT_FLAGS}} yml: aliases: diff --git a/taskfile.yaml b/taskfile.yaml index 470edd58..4588d5bd 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,6 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + G_SRC_PYTHON_DIR: "{{.ROOT_DIR}}/python" + # Build parameters # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. G_DEPS_MAX_PARALLELISM_PER_TASK: >- From 4c51c916abbc549b8b6a395b6d9085bc5bb7e587 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 15:13:01 -0400 Subject: [PATCH 43/91] Add python build tasks --- build-tasks.yaml | 4 ++++ taskfile.yaml | 1 + 2 files changed, 5 insertions(+) diff --git a/build-tasks.yaml b/build-tasks.yaml index 53828027..7363d4d2 100644 --- a/build-tasks.yaml +++ b/build-tasks.yaml @@ -19,3 +19,7 @@ tasks: deps: [":config-cmake-project"] cmds: - "cmake --build {{.G_BUILD_SPIDER_DIR}} --target clean --parallel {{numCPU}}" + + py-build: + cmds: + - uv build --directory "{{.G_SRC_PYTHON_DIR}}" -o "{{.G_BUILD_PYTHON_DIR}}" diff --git a/taskfile.yaml b/taskfile.yaml index 4588d5bd..1a94fdc3 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -21,6 +21,7 @@ vars: G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" G_SRC_PYTHON_DIR: "{{.ROOT_DIR}}/python" + G_BUILD_PYTHON_DIR: "{{.G_BUILD_DIR}}/spider-py" # Build parameters # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. From 6004d6455b56ae4bdd1f30878254ddbe65fe1b9a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 17:51:53 -0400 Subject: [PATCH 44/91] Fix yaml lint error --- build-tasks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tasks.yaml b/build-tasks.yaml index 7363d4d2..1ce1efb9 100644 --- a/build-tasks.yaml +++ b/build-tasks.yaml @@ -22,4 +22,4 @@ tasks: py-build: cmds: - - uv build --directory "{{.G_SRC_PYTHON_DIR}}" -o "{{.G_BUILD_PYTHON_DIR}}" + - "uv build --directory {{.G_SRC_PYTHON_DIR}} -o {{.G_BUILD_PYTHON_DIR}}" From bcab5db89a55c3bbf7086718e1e1a89adcc7adba Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 18:33:58 -0400 Subject: [PATCH 45/91] Fix tombi lint --- python/pyproject.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index c8d87580..235370dd 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -16,17 +16,17 @@ spider_task_executor = "src.task_executor.task_executor:main" requires = ["hatchling>=1.18.0"] build-backend = "hatchling.build" -[tool.hatch] - -[tool.hatch.build.targets.wheel] -packages = ["src/client", "src/task_executor"] - [dependency-groups] dev = [ "mypy>=1.12.0", "ruff>=0.4.4", ] +[tool.hatch] + +[tool.hatch.build.targets.wheel] +packages = ["src/client", "src/task_executor"] + [tool.mypy] strict = true @@ -43,6 +43,10 @@ ignore_missing_imports = true [tool.ruff] line-length = 100 +[tool.ruff.format] +docstring-code-format = true +docstring-code-line-length = 100 + [tool.ruff.lint] select = ["ALL"] extend-select = ["PT"] @@ -66,7 +70,3 @@ ignore = [ "UP015", # Explicit open modes are helpful ] isort.order-by-type = false - -[tool.ruff.format] -docstring-code-format = true -docstring-code-line-length = 100 From a50d25a36e71146abb374328dbf99c910e70d038 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 18:38:25 -0400 Subject: [PATCH 46/91] Remove wrong mypy config --- python/pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 235370dd..09c1e0f1 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -34,7 +34,6 @@ strict = true pretty = true show_error_code_links = true show_error_context = true -show_error_end = true [[tool.mypy.overrides]] module = "msgpack.*" From fbebb963eecfef0b7073ae5aad6311b68fb5fd05 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 18:48:37 -0400 Subject: [PATCH 47/91] Fix typo and format file --- python/pyproject.toml | 54 +++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 09c1e0f1..be9486a1 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -5,31 +5,29 @@ description = "Spider is a distributed task execution framework" readme = "README.md" requires-python = ">=3.10" dependencies = [ - "mariadb>=1.1.13", - "msgpack>=1.1.1", + "mariadb>=1.1.13", + "msgpack>=1.1.1", ] -[project.scritps] +[project.scripts] spider_task_executor = "src.task_executor.task_executor:main" -[build-system] -requires = ["hatchling>=1.18.0"] -build-backend = "hatchling.build" - [dependency-groups] dev = [ - "mypy>=1.12.0", - "ruff>=0.4.4", + "mypy>=1.12.0", + "ruff>=0.4.4", ] -[tool.hatch] +[build-system] +requires = ["hatchling>=1.18.0"] +build-backend = "hatchling.build" +[tool.hatch] [tool.hatch.build.targets.wheel] packages = ["src/client", "src/task_executor"] [tool.mypy] strict = true - # Additional output pretty = true show_error_code_links = true @@ -50,22 +48,22 @@ docstring-code-line-length = 100 select = ["ALL"] extend-select = ["PT"] ignore = [ - "COM812", # Redundant and conflicts with ruff format - "D203", # No blank line before docstrings (D211) - "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists - "D212", # Enforce docstring summary line on the next line after quotes (D213) - "D400", # First line of docstrings may not end in period - "D401", # Docstrings should be written in present tense (not imperative) - "D415", # First line of docstrings may not end in a period, question mark, or exclamation point - "FA102", # Allow use of PEP 604 union in type annotations - "FBT", # Allow bool positional parameters since other value positions are allowed - "FIX002", # Allow todo statements - "PERF401", # Allow for loops when creating lists - "PERF403", # Allow for loops when creating dicts - "S311", # Allow usage of `random` package - "SIM102", # Allow collapsible if statements for readability - "TD002", # Author unnecessary for todo statement - "TD003", # Issue link unnecessary for todo statement - "UP015", # Explicit open modes are helpful + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful ] isort.order-by-type = false From 0322bd528424d820a755284e9f8d86a42788c854 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 21:07:18 -0400 Subject: [PATCH 48/91] Use typed msgpack and remove mypy config for msgpack from pyproject --- python/pyproject.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index be9486a1..9b69cfde 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" requires-python = ">=3.10" dependencies = [ "mariadb>=1.1.13", - "msgpack>=1.1.1", + "msgpack-types>=0.5.0", ] [project.scripts] @@ -33,10 +33,6 @@ pretty = true show_error_code_links = true show_error_context = true -[[tool.mypy.overrides]] -module = "msgpack.*" -ignore_missing_imports = true - [tool.ruff] line-length = 100 From 2a0815558de44be469e2e4a1581b91683d621b3a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 23:55:16 -0400 Subject: [PATCH 49/91] Update uv lock --- python/uv.lock | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/python/uv.lock b/python/uv.lock index 330a5aaa..6dc97a77 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -69,6 +69,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/91/7dc28d5e2a11a5ad804cf2b7f7a5fcb1eb5a4966d66a5d2b41aee6376543/msgpack-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:6d489fba546295983abd142812bda76b57e33d0b9f5d5b71c09a583285506f69", size = 72341, upload-time = "2025-06-13T06:52:27.835Z" }, ] +[[package]] +name = "msgpack-types" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msgpack" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/26/a15707f2af5681333cd598724bedd1948844ac2af45eafc4175af0671a8d/msgpack_types-0.5.0.tar.gz", hash = "sha256:aebd1b8da23f8f9966d66ebb1a43bd261b95751c6a267bd21a124d2ccac84201", size = 6702, upload-time = "2024-09-21T13:55:05.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/dd/cd9d2b0ef506f6164cd81d4e92e408095041f28523d751b9f7dabdc244eb/msgpack_types-0.5.0-py3-none-any.whl", hash = "sha256:8b633ed75e495a555fa0615843de559a74b1d176828d59bb393d266e51f6bda7", size = 8182, upload-time = "2024-09-21T13:55:04.232Z" }, +] + [[package]] name = "mypy" version = "1.17.1" @@ -172,7 +185,7 @@ version = "0.0.1" source = { editable = "." } dependencies = [ { name = "mariadb" }, - { name = "msgpack" }, + { name = "msgpack-types" }, ] [package.dev-dependencies] @@ -184,7 +197,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "mariadb", specifier = ">=1.1.13" }, - { name = "msgpack", specifier = ">=1.1.1" }, + { name = "msgpack-types", specifier = ">=0.5.0" }, ] [package.metadata.requires-dev] From b09515c0d5a815c6e72a25e4275ddf1d464c4937 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 10:16:51 -0400 Subject: [PATCH 50/91] Increase min version of tombi --- lint-test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-test-requirements.txt b/lint-test-requirements.txt index 879c198b..ffbf8992 100644 --- a/lint-test-requirements.txt +++ b/lint-test-requirements.txt @@ -6,7 +6,7 @@ ruff>=0.4.4 mypy>=1.12.0 gersemi>=0.16.2 yamllint>=1.35.1 -tombi>=0.4.0 +tombi>=0.4.35 # Test dependencies msgpack>=1.1.0 From b4d657637fa5da39f8dd1cdd25f67348371eec43 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 16:34:55 -0400 Subject: [PATCH 51/91] Add uv in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2ccfff88..54cf6bd6 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Follow the steps below to develop and contribute to the project. ## Requirements * Python 3.10 or higher * [Task] 3.40.0 or higher +* [uv] 0.7.0 or higher ## Set up Run dependency installation task: From f447614c9ba65d9e2ddd42e69a194120e7168d1f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 17:35:09 -0400 Subject: [PATCH 52/91] Fix redenduncy caused by merge --- lint-tasks.yaml | 2 -- requirements-dev.txt | 1 - 2 files changed, 3 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 4e941803..54ca4611 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -12,7 +12,6 @@ tasks: - task: "py-check" - task: "toml-check" - task: "yml-check" - - task: "toml-check" fix: cmds: @@ -20,7 +19,6 @@ tasks: - task: "cpp-fix" - task: "toml-fix" - task: "yml-fix" - - task: "toml-fix" cmake-check: deps: ["venv"] diff --git a/requirements-dev.txt b/requirements-dev.txt index d1379209..a5e8717f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,6 @@ mypy>=1.12.0 ruff>=0.4.4 tombi>=0.4.35 yamllint>=1.35.1 -tombi>=0.4.35 # Test dependencies msgpack-types>=0.5.0 From de149ee82fcad48b70f5f128c01a0d860449663c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 19:51:06 -0400 Subject: [PATCH 53/91] Add integral types --- python/src/type/__init__.py | 1 + python/src/type/type.py | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 python/src/type/__init__.py create mode 100644 python/src/type/type.py diff --git a/python/src/type/__init__.py b/python/src/type/__init__.py new file mode 100644 index 00000000..f147ce3e --- /dev/null +++ b/python/src/type/__init__.py @@ -0,0 +1 @@ +"""Spider type module.""" diff --git a/python/src/type/type.py b/python/src/type/type.py new file mode 100644 index 00000000..1a14dfda --- /dev/null +++ b/python/src/type/type.py @@ -0,0 +1,42 @@ +"""Custom type module for Spider.""" + +from typing import cast + + +class BoundedInt(int): + """Bounded integer type.""" + + def __new__(cls, value: int, bits: int = 32) -> "BoundedInt": + """Creates a bounded integer.""" + min_val = -(1 << (bits - 1)) + max_val = (1 << (bits - 1)) - 1 + + if not (min_val <= value <= max_val): + msg = f"Bounded integer value ({value}) must be between {min_val} and {max_val}" + raise ValueError(msg) + + return super().__new__(cls, value) + + +class Int8(BoundedInt): + """8 bits integer type.""" + + def __new__(cls, value: int) -> "Int8": + """Creates an int8 integer.""" + return cast("Int8", super().__new__(cls, value, bits=8)) + + +class Int16(BoundedInt): + """16 bits integer type.""" + + def __new__(cls, value: int) -> "Int16": + """Creates an int16 integer.""" + return cast("Int16", super().__new__(cls, value, bits=16)) + + +class Int32(BoundedInt): + """32 bits integer type.""" + + def __new__(cls, value: int) -> "Int32": + """Creates an int32 integer.""" + return cast("Int32", super().__new__(cls, value, bits=8)) From d332dbe3ed95b3a4a422d44248c054f3200aeb6b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 19:52:13 -0400 Subject: [PATCH 54/91] Revert "Add integral types" This reverts commit de149ee82fcad48b70f5f128c01a0d860449663c. --- python/src/type/__init__.py | 1 - python/src/type/type.py | 42 ------------------------------------- 2 files changed, 43 deletions(-) delete mode 100644 python/src/type/__init__.py delete mode 100644 python/src/type/type.py diff --git a/python/src/type/__init__.py b/python/src/type/__init__.py deleted file mode 100644 index f147ce3e..00000000 --- a/python/src/type/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Spider type module.""" diff --git a/python/src/type/type.py b/python/src/type/type.py deleted file mode 100644 index 1a14dfda..00000000 --- a/python/src/type/type.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Custom type module for Spider.""" - -from typing import cast - - -class BoundedInt(int): - """Bounded integer type.""" - - def __new__(cls, value: int, bits: int = 32) -> "BoundedInt": - """Creates a bounded integer.""" - min_val = -(1 << (bits - 1)) - max_val = (1 << (bits - 1)) - 1 - - if not (min_val <= value <= max_val): - msg = f"Bounded integer value ({value}) must be between {min_val} and {max_val}" - raise ValueError(msg) - - return super().__new__(cls, value) - - -class Int8(BoundedInt): - """8 bits integer type.""" - - def __new__(cls, value: int) -> "Int8": - """Creates an int8 integer.""" - return cast("Int8", super().__new__(cls, value, bits=8)) - - -class Int16(BoundedInt): - """16 bits integer type.""" - - def __new__(cls, value: int) -> "Int16": - """Creates an int16 integer.""" - return cast("Int16", super().__new__(cls, value, bits=16)) - - -class Int32(BoundedInt): - """32 bits integer type.""" - - def __new__(cls, value: int) -> "Int32": - """Creates an int32 integer.""" - return cast("Int32", super().__new__(cls, value, bits=8)) From 80b67726082556e8c02d768ab54778d92a7b7a59 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 19:57:46 -0400 Subject: [PATCH 55/91] Add package export control for type --- python/src/type/__init__.py | 4 ++++ python/src/type/type.py | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/python/src/type/__init__.py b/python/src/type/__init__.py index f147ce3e..f0f38119 100644 --- a/python/src/type/__init__.py +++ b/python/src/type/__init__.py @@ -1 +1,5 @@ """Spider type module.""" + +from type.type import Int8, Int16, Int32, Int64 + +__all__ = ["Int8", "Int16", "Int32", "Int64"] diff --git a/python/src/type/type.py b/python/src/type/type.py index 1a14dfda..a7d9ad41 100644 --- a/python/src/type/type.py +++ b/python/src/type/type.py @@ -39,4 +39,12 @@ class Int32(BoundedInt): def __new__(cls, value: int) -> "Int32": """Creates an int32 integer.""" - return cast("Int32", super().__new__(cls, value, bits=8)) + return cast("Int32", super().__new__(cls, value, bits=32)) + + +class Int64(BoundedInt): + """64 bits integer type.""" + + def __new__(cls, value: int) -> "Int64": + """Creates an int64 integer.""" + return cast("Int64", super().__new__(cls, value, bits=64)) From f2ebaa67020d3525cdf705470cb2be5f7884270c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 20:02:45 -0400 Subject: [PATCH 56/91] Add floating point type --- python/src/type/__init__.py | 4 ++-- python/src/type/type.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/python/src/type/__init__.py b/python/src/type/__init__.py index f0f38119..f8dba98e 100644 --- a/python/src/type/__init__.py +++ b/python/src/type/__init__.py @@ -1,5 +1,5 @@ """Spider type module.""" -from type.type import Int8, Int16, Int32, Int64 +from type.type import Double, Float, Int8, Int16, Int32, Int64 -__all__ = ["Int8", "Int16", "Int32", "Int64"] +__all__ = ["Double", "Float", "Int8", "Int16", "Int32", "Int64"] diff --git a/python/src/type/type.py b/python/src/type/type.py index a7d9ad41..318cc90c 100644 --- a/python/src/type/type.py +++ b/python/src/type/type.py @@ -48,3 +48,7 @@ class Int64(BoundedInt): def __new__(cls, value: int) -> "Int64": """Creates an int64 integer.""" return cast("Int64", super().__new__(cls, value, bits=64)) + + +Float = float +Double = float From 8c27b1696ede2bb099693d1536e94930de2bac9f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 20:57:38 -0400 Subject: [PATCH 57/91] Restructure under src/spider --- python/pyproject.toml | 6 +----- python/src/core/__init__.py | 1 - python/src/spider/__init__.py | 1 + python/src/{ => spider}/client/__init__.py | 0 python/src/spider/core/__init__.py | 1 + python/src/{ => spider}/core/task.py | 0 python/src/{ => spider}/task_executor/__init__.py | 0 python/src/{ => spider}/task_executor/task_executor.py | 0 8 files changed, 3 insertions(+), 6 deletions(-) delete mode 100644 python/src/core/__init__.py create mode 100644 python/src/spider/__init__.py rename python/src/{ => spider}/client/__init__.py (100%) create mode 100644 python/src/spider/core/__init__.py rename python/src/{ => spider}/core/task.py (100%) rename python/src/{ => spider}/task_executor/__init__.py (100%) rename python/src/{ => spider}/task_executor/task_executor.py (100%) diff --git a/python/pyproject.toml b/python/pyproject.toml index 9b69cfde..658b2143 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ ] [project.scripts] -spider_task_executor = "src.task_executor.task_executor:main" +spider_task_executor = "src.spider.task_executor.task_executor:main" [dependency-groups] dev = [ @@ -22,10 +22,6 @@ dev = [ requires = ["hatchling>=1.18.0"] build-backend = "hatchling.build" -[tool.hatch] -[tool.hatch.build.targets.wheel] -packages = ["src/client", "src/task_executor"] - [tool.mypy] strict = true # Additional output diff --git a/python/src/core/__init__.py b/python/src/core/__init__.py deleted file mode 100644 index e4e40894..00000000 --- a/python/src/core/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Spider core module.""" diff --git a/python/src/spider/__init__.py b/python/src/spider/__init__.py new file mode 100644 index 00000000..c3628976 --- /dev/null +++ b/python/src/spider/__init__.py @@ -0,0 +1 @@ +"""Spider package root.""" diff --git a/python/src/client/__init__.py b/python/src/spider/client/__init__.py similarity index 100% rename from python/src/client/__init__.py rename to python/src/spider/client/__init__.py diff --git a/python/src/spider/core/__init__.py b/python/src/spider/core/__init__.py new file mode 100644 index 00000000..d417d95e --- /dev/null +++ b/python/src/spider/core/__init__.py @@ -0,0 +1 @@ +"""Spider core package.""" diff --git a/python/src/core/task.py b/python/src/spider/core/task.py similarity index 100% rename from python/src/core/task.py rename to python/src/spider/core/task.py diff --git a/python/src/task_executor/__init__.py b/python/src/spider/task_executor/__init__.py similarity index 100% rename from python/src/task_executor/__init__.py rename to python/src/spider/task_executor/__init__.py diff --git a/python/src/task_executor/task_executor.py b/python/src/spider/task_executor/task_executor.py similarity index 100% rename from python/src/task_executor/task_executor.py rename to python/src/spider/task_executor/task_executor.py From a7aa6b096bbf69ac99d97d331ca65ea4bca4ae1f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 21:02:07 -0400 Subject: [PATCH 58/91] Re-export spider.type under spider --- python/src/spider/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/src/spider/__init__.py b/python/src/spider/__init__.py index c3628976..929a7aaa 100644 --- a/python/src/spider/__init__.py +++ b/python/src/spider/__init__.py @@ -1 +1,12 @@ """Spider package root.""" + +from spider.type import Double, Float, Int8, Int16, Int32, Int64 + +__all__ = [ + "Double", + "Float", + "Int8", + "Int16", + "Int32", + "Int64", +] From 5765acf6b059e05c036f5c7eb1a337f73c60a82b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 00:50:38 -0400 Subject: [PATCH 59/91] Add tdl types; Allow no docstring for override function --- python/pyproject.toml | 3 + python/src/spider/type/tdl_type.py | 130 +++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 python/src/spider/type/tdl_type.py diff --git a/python/pyproject.toml b/python/pyproject.toml index 658b2143..5da3702d 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -59,3 +59,6 @@ ignore = [ "UP015", # Explicit open modes are helpful ] isort.order-by-type = false + +[tool.ruff.lint.pydocstyle] +ignore-decorators = ["typing.override"] diff --git a/python/src/spider/type/tdl_type.py b/python/src/spider/type/tdl_type.py new file mode 100644 index 00000000..5f484ca8 --- /dev/null +++ b/python/src/spider/type/tdl_type.py @@ -0,0 +1,130 @@ +"""Spider TDL types.""" + +from abc import ABC, abstractmethod + +from typing_extensions import override + + +class TdlType(ABC): + """Abstract base class for all TDL types.""" + + @abstractmethod + def type_str(self) -> str: + """:return: String representation of the TDL type.""" + + +class DoubleType(TdlType): + """TDL double type.""" + + @override + def type_str(self) -> str: + return "double" + + +class FloatType(TdlType): + """TDL float type.""" + + @override + def type_str(self) -> str: + return "float" + + +class Int8Type(TdlType): + """TDL int8 type.""" + + @override + def type_str(self) -> str: + return "int8" + + +class Int16Type(TdlType): + """TDL int16 type.""" + + @override + def type_str(self) -> str: + return "int16" + + +class Int32Type(TdlType): + """TDL int32 type.""" + + @override + def type_str(self) -> str: + return "int32" + + +class Int64Type(TdlType): + """TDL int64 type.""" + + @override + def type_str(self) -> str: + return "int64" + + +class BoolType(TdlType): + """TDL bool type.""" + + @override + def type_str(self) -> str: + return "bool" + + +class ClassType(TdlType): + """TDL Custom class type.""" + + def __init__(self, name: str) -> None: + """ + Creates a TDL custom class type. + :param name: The name of the class. + """ + self.name = name + + @override + def type_str(self) -> str: + return self.name + + +class ListType(TdlType): + """TDL List type.""" + + def __init__(self, key: TdlType) -> None: + """Creates a TDL list type.""" + self.key = key + + @override + def type_str(self) -> str: + return f"List<{self.key.type_str()}>" + + +def is_integral(tdl_type: TdlType) -> bool: + """:return: If TDL type is an integral type.""" + return isinstance(tdl_type, (Int8Type, Int16Type, Int32Type, Int64Type)) + + +def is_string(tdl_type: TdlType) -> bool: + """:return: If TDL type is a string type, i.e. List.""" + return isinstance(tdl_type, ListType) and isinstance(tdl_type.key, Int8Type) + + +def is_map_key(tdl_type: TdlType) -> bool: + """:return: If TDL type is a valid type for map key.""" + return is_integral(tdl_type) or is_string(tdl_type) + + +class MapType(TdlType): + """TDL Map type.""" + + def __init__(self, key: TdlType, value: TdlType) -> None: + """ + Creates a TDL map type. + :raises TypeError: If key is not a supported type. + """ + if not is_map_key(key): + msg = f"{key} is not a supported type for map key." + raise TypeError(msg) + self.key = key + self.value = value + + @override + def type_str(self) -> str: + return f"Map<{self.key.type_str()},{self.value.type_str()}>" From 213d2fcd5e69099f68d43953ebc40e0797156232 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 13:28:19 -0400 Subject: [PATCH 60/91] Add type conversion to tdl type --- python/src/spider/type/tdl_convert.py | 84 +++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 python/src/spider/type/tdl_convert.py diff --git a/python/src/spider/type/tdl_convert.py b/python/src/spider/type/tdl_convert.py new file mode 100644 index 00000000..82c99d89 --- /dev/null +++ b/python/src/spider/type/tdl_convert.py @@ -0,0 +1,84 @@ +"""Converts native types to TDL types.""" + +from collections.abc import Collection +from typing import get_args + +from spider import Double, Float, Int8, Int16, Int32, Int64 +from spider.type.tdl_type import ( + BoolType, + ClassType, + DoubleType, + FloatType, + Int8Type, + Int16Type, + Int32Type, + Int64Type, + ListType, + MapType, + TdlType, +) + + +def to_primitive_tdl_type(native_type: type) -> TdlType | None: + """ + Converts a native type to primitive TDL type. + :param native_type: + :return: Converted TDL primitive. None if `native_type` is not a supported primitive type. + """ + tdl_type: TdlType | None = None + if native_type is Int8: + tdl_type = Int8Type() + elif native_type is Int16: + tdl_type = Int16Type() + elif native_type is Int32: + tdl_type = Int32Type() + elif native_type is Int64: + tdl_type = Int64Type() + elif native_type is Float: + tdl_type = FloatType() + elif native_type is Double: + tdl_type = DoubleType() + elif native_type is bool: + tdl_type = BoolType() + return tdl_type + + +def to_tdl_type(native_type: type) -> TdlType: + """ + Converts a Python type to TDL type. + :param native_type: + :return: + :raise: TypeError if `native_type` is not a valid TDL type. + """ + primitive_tdl_type = to_primitive_tdl_type(native_type) + if primitive_tdl_type is not None: + return primitive_tdl_type + + if native_type is list: + arg = get_args(native_type) + if arg is None: + msg = "List does not have a key type." + raise TypeError(msg) + arg = arg[0] + return ListType(to_tdl_type(arg)) + + if native_type is dict: + arg = get_args(native_type) + msg = "Dict does not have a key/value type." + if arg is None: + raise TypeError(msg) + if len(arg) != 2: # noqa: PLR2004 + raise TypeError(msg) + key = arg[0] + value = arg[1] + return MapType(to_tdl_type(key), to_tdl_type(value)) + + if native_type in (int, float, str, complex, bytes): + msg = f"{native_type} is not a valid TDL type." + raise TypeError(msg) + + if issubclass(native_type, Collection): + msg = f"{native_type} is not a valid TDL type." + raise TypeError(msg) + + return ClassType(native_type.__name__) From a1a86f329c370a8bc1b9063276e4af56e9a5ecc2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 13:31:34 -0400 Subject: [PATCH 61/91] Add to_tdl_str --- python/src/spider/type/tdl_convert.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/src/spider/type/tdl_convert.py b/python/src/spider/type/tdl_convert.py index 82c99d89..eb2a8ad8 100644 --- a/python/src/spider/type/tdl_convert.py +++ b/python/src/spider/type/tdl_convert.py @@ -82,3 +82,13 @@ def to_tdl_type(native_type: type) -> TdlType: raise TypeError(msg) return ClassType(native_type.__name__) + + +def to_tdl_type_str(native_type: type) -> str: + """ + Converts a Python type to TDL type string. + :param native_type: + :return: + :raise: TypeError if `native_type` is not a valid TDL type. + """ + return to_tdl_type(native_type).type_str() From 97af71bb7d0e0e86a7a34ec2a3dd7b45a1cb9898 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 13:33:10 -0400 Subject: [PATCH 62/91] Add to_tdl_type_str to type package export --- python/src/spider/type/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/src/spider/type/__init__.py b/python/src/spider/type/__init__.py index 70ff91d9..e0c873db 100644 --- a/python/src/spider/type/__init__.py +++ b/python/src/spider/type/__init__.py @@ -1,5 +1,6 @@ """Spider type package.""" +from spider.type.tdl_convert import to_tdl_type_str from spider.type.type import Double, Float, Int8, Int16, Int32, Int64 -__all__ = ["Double", "Float", "Int8", "Int16", "Int32", "Int64"] +__all__ = ["Double", "Float", "Int8", "Int16", "Int32", "Int64", "to_tdl_type_str"] From 74f31e26e4e50f9b09253e769b1eb501b718e741 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 16:28:35 -0400 Subject: [PATCH 63/91] Add pytest and basic test structure --- python/pyproject.toml | 11 ++++++ python/tests/__init__.py | 1 + python/tests/core/__init__.py | 1 + python/tests/core/test_task.py | 9 +++++ python/uv.lock | 68 ++++++++++++++++++++++++++++++++++ 5 files changed, 90 insertions(+) create mode 100644 python/tests/__init__.py create mode 100644 python/tests/core/__init__.py create mode 100644 python/tests/core/test_task.py diff --git a/python/pyproject.toml b/python/pyproject.toml index 658b2143..f0c2b768 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -15,6 +15,7 @@ spider_task_executor = "src.spider.task_executor.task_executor:main" [dependency-groups] dev = [ "mypy>=1.12.0", + "pytest>=8.4.1", "ruff>=0.4.4", ] @@ -29,6 +30,9 @@ pretty = true show_error_code_links = true show_error_context = true +[tool.pytest.ini_options] +testpaths = ["tests"] + [tool.ruff] line-length = 100 @@ -59,3 +63,10 @@ ignore = [ "UP015", # Explicit open modes are helpful ] isort.order-by-type = false + +[tool.ruff.lint.per-file-ignores] +"tests/**" = [ + "S101", # Allow use of `assert` (security warning) + "S603", # Allow use of `subprocess.Popen` (security warning) + "T201", # Allow use of `print` (testing) +] diff --git a/python/tests/__init__.py b/python/tests/__init__.py new file mode 100644 index 00000000..ab1be478 --- /dev/null +++ b/python/tests/__init__.py @@ -0,0 +1 @@ +"""Spider tests.""" diff --git a/python/tests/core/__init__.py b/python/tests/core/__init__.py new file mode 100644 index 00000000..4835acd8 --- /dev/null +++ b/python/tests/core/__init__.py @@ -0,0 +1 @@ +"""Spider core package tests.""" diff --git a/python/tests/core/test_task.py b/python/tests/core/test_task.py new file mode 100644 index 00000000..a6d4834e --- /dev/null +++ b/python/tests/core/test_task.py @@ -0,0 +1,9 @@ +"""Test core task.""" + +from spider.core.task import Task + + +def test_task() -> None: + """Tests task created is not None.""" + task = Task() + assert task is not None diff --git a/python/uv.lock b/python/uv.lock index 6dc97a77..ae2f0e96 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -2,6 +2,36 @@ version = 1 revision = 2 requires-python = ">=3.10" +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, +] + [[package]] name = "mariadb" version = "1.1.13" @@ -154,6 +184,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, ] +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714, upload-time = "2025-06-18T05:48:06.109Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, +] + [[package]] name = "ruff" version = "0.12.7" @@ -191,6 +257,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "mypy" }, + { name = "pytest" }, { name = "ruff" }, ] @@ -203,6 +270,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "mypy", specifier = ">=1.12.0" }, + { name = "pytest", specifier = ">=8.4.1" }, { name = "ruff", specifier = ">=0.4.4" }, ] From 7d9570663a0e6452b40f7fa50eb1169c926ea7a8 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 16:28:57 -0400 Subject: [PATCH 64/91] Add python test tasks --- test-tasks.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test-tasks.yaml b/test-tasks.yaml index 936211e5..0b6e4aa4 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -76,3 +76,8 @@ tasks: vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + + python-tests: + dir: "{{.G_SRC_PYTHON_DIR}}" + cmd: |- + uv run pytest From 96b5324a642087d75ec757b2908e489b6cd87d1d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 16:33:24 -0400 Subject: [PATCH 65/91] Rename some cpp tests and add python tests to GH workflow and doc --- .github/workflows/code-linting-checks.yaml | 1 + .github/workflows/unit-tests.yaml | 3 +++ docs/src/dev-docs/testing.md | 17 +++++++++-------- test-tasks.yaml | 4 ++-- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index 48f8eacd..f8dd495a 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -44,6 +44,7 @@ jobs: python --version tar --version task --version + uv --version - name: "Install project dependencies " timeout-minutes: 10 diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 1ced6a88..05877926 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -47,6 +47,7 @@ jobs: python --version tar --version task --version + uv --version - name: "Install project dependencies " timeout-minutes: 10 @@ -55,3 +56,5 @@ jobs: run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" + + - run: "task test:python-tests" diff --git a/docs/src/dev-docs/testing.md b/docs/src/dev-docs/testing.md index 873ef7b6..759cf502 100644 --- a/docs/src/dev-docs/testing.md +++ b/docs/src/dev-docs/testing.md @@ -30,11 +30,12 @@ require this storage backend. You can use the following tasks to run the set of unit tests that's appropriate. -| Task | Description | -|-------------------------------|-------------------------------------------------------------------| -| `test:all` | Runs all unit tests. | -| `test:non-storage-unit-tests` | Runs all unit tests which don't require a storage backend to run. | -| `test:storage-unit-tests` | Runs all unit tests which require a storage backend to run. | +| Task | Description | +|-------------------------------|-----------------------------------------------------------------------| +| `test:cpp-all` | Runs all C++ unit tests. | +| `test:non-storage-unit-tests` | Runs all C++ unit tests which don't require a storage backend to run. | +| `test:storage-unit-tests` | Runs all C++ unit tests which require a storage backend to run. | +| `test:python-tests` | Runs all Python tests. | If any tests show error messages for the connection function below, revisit the [setup section](#set-up-mysql-as-storage-backend) and verify that `cStorageUrl` was set correctly. @@ -52,9 +53,9 @@ pull requests, and daily. Currently, it only runs unit tests that don't require You can use the following tasks to run integration tests. -| Task | Description | -|-------------------------------|-------------------------------------------------------------------| -| `test:integration` | Runs all integration tests. | +| Task | Description | +|------------------------|---------------------------------| +| `test:cpp-integration` | Runs all C++ integration tests. | [gh-workflow-unit-tests]: https://github.com/y-scope/spider/blob/main/.github/workflows/unit-tests.yaml diff --git a/test-tasks.yaml b/test-tasks.yaml index 0b6e4aa4..1d4c88fb 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -18,7 +18,7 @@ tasks: cmds: - "{{.G_UNIT_TEST_BINARY}} \"[storage]\"" - all: + cpp-all: deps: - "build-unit-test" cmds: @@ -31,7 +31,7 @@ tasks: vars: TARGETS: ["spider_task_executor", "unitTest", "worker_test"] - integration: + cpp-integration: dir: "{{.G_BUILD_SPIDER_DIR}}" deps: - "venv" From 14ea6fb93a7eebda34b29920b245897715f059c6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 16:44:22 -0400 Subject: [PATCH 66/91] Don't create __pycache__ when running pytest --- test-tasks.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test-tasks.yaml b/test-tasks.yaml index 1d4c88fb..cb8bbdde 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -79,5 +79,6 @@ tasks: python-tests: dir: "{{.G_SRC_PYTHON_DIR}}" - cmd: |- - uv run pytest + cmds: + # Don't create __pycache__ directories in source tree. + - "uv run env PYTHONDONTWRITEBYTECODE=1 pytest" From 854cd11dbfcc009b144c374b493973a52c66524b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 17:19:25 -0400 Subject: [PATCH 67/91] Fix missing link --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8ac64e9e..11aec22f 100644 --- a/README.md +++ b/README.md @@ -68,3 +68,4 @@ in the table below. [spider-docs]: https://docs.yscope.com/spider/main/ [Task]: https://taskfile.dev +[uv]: https://docs.astral.sh/uv/ \ No newline at end of file From 7ea49287700fbab6d7d1ac0344e42e4f5297a60c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 17:21:16 -0400 Subject: [PATCH 68/91] Use task env --- test-tasks.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test-tasks.yaml b/test-tasks.yaml index cb8bbdde..88a2c838 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -79,6 +79,8 @@ tasks: python-tests: dir: "{{.G_SRC_PYTHON_DIR}}" + env: + PYTHONDONTWRITEBYTECODE: "1" cmds: # Don't create __pycache__ directories in source tree. - - "uv run env PYTHONDONTWRITEBYTECODE=1 pytest" + - "uv run env pytest" From 01dd0c6b6fb4ca99f7b045bfb13d70c697c0d8ca Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 17:26:30 -0400 Subject: [PATCH 69/91] Fix task executor path --- python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 658b2143..0c9da379 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ ] [project.scripts] -spider_task_executor = "src.spider.task_executor.task_executor:main" +spider_task_executor = "spider.task_executor.task_executor:main" [dependency-groups] dev = [ From 438f6e770c4a92492e1208144fb2e5f211fabe08 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 22:11:10 -0400 Subject: [PATCH 70/91] Remove unnecessary __init__.py files --- python/tests/__init__.py | 1 - python/tests/core/__init__.py | 1 - 2 files changed, 2 deletions(-) delete mode 100644 python/tests/__init__.py delete mode 100644 python/tests/core/__init__.py diff --git a/python/tests/__init__.py b/python/tests/__init__.py deleted file mode 100644 index ab1be478..00000000 --- a/python/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Spider tests.""" diff --git a/python/tests/core/__init__.py b/python/tests/core/__init__.py deleted file mode 100644 index 4835acd8..00000000 --- a/python/tests/core/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Spider core package tests.""" From 61e0651d0d74e49bfc3ae10658ea0c5b6b33a2e7 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 22:51:30 -0400 Subject: [PATCH 71/91] Fix bugs --- python/src/spider/type/tdl_convert.py | 46 +++++++++++++++------------ python/src/spider/type/type.py | 16 ++++++++-- python/tests/core/test_task.py | 9 ------ 3 files changed, 40 insertions(+), 31 deletions(-) delete mode 100644 python/tests/core/test_task.py diff --git a/python/src/spider/type/tdl_convert.py b/python/src/spider/type/tdl_convert.py index eb2a8ad8..35bff266 100644 --- a/python/src/spider/type/tdl_convert.py +++ b/python/src/spider/type/tdl_convert.py @@ -1,9 +1,9 @@ """Converts native types to TDL types.""" +import types from collections.abc import Collection -from typing import get_args +from typing import get_args, get_origin -from spider import Double, Float, Int8, Int16, Int32, Int64 from spider.type.tdl_type import ( BoolType, ClassType, @@ -17,6 +17,7 @@ MapType, TdlType, ) +from spider.type.type import Double, Float, Int8, Int16, Int32, Int64 def to_primitive_tdl_type(native_type: type) -> TdlType | None: @@ -54,26 +55,31 @@ def to_tdl_type(native_type: type) -> TdlType: if primitive_tdl_type is not None: return primitive_tdl_type - if native_type is list: - arg = get_args(native_type) - if arg is None: - msg = "List does not have a key type." - raise TypeError(msg) - arg = arg[0] - return ListType(to_tdl_type(arg)) + if native_type in (int, float, str, complex, bytes): + msg = f"{native_type} is not a valid TDL type." + raise TypeError(msg) - if native_type is dict: - arg = get_args(native_type) - msg = "Dict does not have a key/value type." - if arg is None: - raise TypeError(msg) - if len(arg) != 2: # noqa: PLR2004 - raise TypeError(msg) - key = arg[0] - value = arg[1] - return MapType(to_tdl_type(key), to_tdl_type(value)) + if isinstance(native_type, types.GenericAlias): + origin = get_origin(native_type) + if origin is list: + arg = get_args(native_type) + if arg is None: + msg = "List does not have a key type." + raise TypeError(msg) + arg = arg[0] + return ListType(to_tdl_type(arg)) + + if origin is dict: + arg = get_args(native_type) + msg = "Dict does not have a key/value type." + if arg is None: + raise TypeError(msg) + if len(arg) != 2: # noqa: PLR2004 + raise TypeError(msg) + key = arg[0] + value = arg[1] + return MapType(to_tdl_type(key), to_tdl_type(value)) - if native_type in (int, float, str, complex, bytes): msg = f"{native_type} is not a valid TDL type." raise TypeError(msg) diff --git a/python/src/spider/type/type.py b/python/src/spider/type/type.py index 318cc90c..903a9972 100644 --- a/python/src/spider/type/type.py +++ b/python/src/spider/type/type.py @@ -50,5 +50,17 @@ def __new__(cls, value: int) -> "Int64": return cast("Int64", super().__new__(cls, value, bits=64)) -Float = float -Double = float +class Float(float): + """Float type.""" + + def __new__(cls, value: float) -> "Float": + """Creates a float number.""" + return super().__new__(cls, value) + + +class Double(float): + """Double type.""" + + def __new__(cls, value: float) -> "Double": + """Creates a double number.""" + return super().__new__(cls, value) diff --git a/python/tests/core/test_task.py b/python/tests/core/test_task.py deleted file mode 100644 index a6d4834e..00000000 --- a/python/tests/core/test_task.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Test core task.""" - -from spider.core.task import Task - - -def test_task() -> None: - """Tests task created is not None.""" - task = Task() - assert task is not None From 8b13abdb591703623ef9d39dfabfc377db29c1d0 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 22:55:59 -0400 Subject: [PATCH 72/91] Add some type convertion tests --- python/pyproject.toml | 1 + python/tests/type/test_to_tdl.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 python/tests/type/test_to_tdl.py diff --git a/python/pyproject.toml b/python/pyproject.toml index ab0c7ae3..74c43b23 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -66,6 +66,7 @@ isort.order-by-type = false [tool.ruff.lint.per-file-ignores] "tests/**" = [ + "INP001", # Allow implicit namespace package for tests "S101", # Allow use of `assert` (security warning) "S603", # Allow use of `subprocess.Popen` (security warning) "T201", # Allow use of `print` (testing) diff --git a/python/tests/type/test_to_tdl.py b/python/tests/type/test_to_tdl.py new file mode 100644 index 00000000..a0d2db3c --- /dev/null +++ b/python/tests/type/test_to_tdl.py @@ -0,0 +1,23 @@ +"""Unit tests for converting to TDL.""" + +from spider import Double, Float, Int8, Int16, Int32, Int64 +from spider.type import to_tdl_type_str + + +class TestToTDL: + """Unit tests for converting to TDL.""" + + def test_to_tdl_primitive(self) -> None: + """Test converting primitive types to TDL Types.""" + assert to_tdl_type_str(bool) == "bool" + assert to_tdl_type_str(Double) == "double" + assert to_tdl_type_str(Float) == "float" + assert to_tdl_type_str(Int8) == "int8" + assert to_tdl_type_str(Int16) == "int16" + assert to_tdl_type_str(Int32) == "int32" + assert to_tdl_type_str(Int64) == "int64" + + def test_to_tdl_list(self) -> None: + """Test converting lists to TDL Types.""" + assert to_tdl_type_str(list[Int8]) == "List" + assert to_tdl_type_str(list[list[Int8]]) == "List>" From 50b2b25cd13bb3236bf8617ef731f50a41624ac4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 23:02:20 -0400 Subject: [PATCH 73/91] Bug fix --- python/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index e3308eb5..db8a9932 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -66,6 +66,7 @@ isort.order-by-type = false [tool.ruff.lint.per-file-ignores] "tests/**" = [ + "INP001", # Allow implicit namespace package for tests "S101", # Allow use of `assert` (security warning) "S603", # Allow use of `subprocess.Popen` (security warning) "T201", # Allow use of `print` (testing) From 8e2786bda5bab979967e5037e1765cf96b3fedb0 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 23:40:25 -0400 Subject: [PATCH 74/91] Add more tests --- python/tests/type/test_to_tdl.py | 46 ++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/python/tests/type/test_to_tdl.py b/python/tests/type/test_to_tdl.py index a0d2db3c..6ffb98a5 100644 --- a/python/tests/type/test_to_tdl.py +++ b/python/tests/type/test_to_tdl.py @@ -1,5 +1,7 @@ """Unit tests for converting to TDL.""" +import pytest + from spider import Double, Float, Int8, Int16, Int32, Int64 from spider.type import to_tdl_type_str @@ -21,3 +23,47 @@ def test_to_tdl_list(self) -> None: """Test converting lists to TDL Types.""" assert to_tdl_type_str(list[Int8]) == "List" assert to_tdl_type_str(list[list[Int8]]) == "List>" + + def test_to_tdl_map(self) -> None: + """Test converting maps to TDL Types.""" + assert to_tdl_type_str(dict[Int8, dict[Int16, Float]]) == "Map>" + assert to_tdl_type_str(dict[list[Int8], Double]) == "Map,double>" + + def test_to_tdl_class(self) -> None: + """Test converting class to TDL Types.""" + assert to_tdl_type_str(TestToTDL) == "TestToTDL" + + def test_to_tdl_primitive_exception(self) -> None: + """Test converting unsupported primitive types to TDL Types.""" + with pytest.raises(TypeError): + to_tdl_type_str(int) + with pytest.raises(TypeError): + to_tdl_type_str(float) + with pytest.raises(TypeError): + to_tdl_type_str(str) + with pytest.raises(TypeError): + to_tdl_type_str(bytes) + with pytest.raises(TypeError): + to_tdl_type_str(list) + with pytest.raises(TypeError): + to_tdl_type_str(dict) + with pytest.raises(TypeError): + to_tdl_type_str(tuple) + + def test_to_tdl_list_exception(self) -> None: + """Test converting unsupported lists to TDL Types.""" + with pytest.raises(TypeError): + to_tdl_type_str(list[int]) + + def test_to_tdl_map_exception(self) -> None: + """Test converting unsupported maps to TDL Types.""" + with pytest.raises(TypeError): + to_tdl_type_str(dict[Int8, int]) + with pytest.raises(TypeError): + to_tdl_type_str(dict[int, Int8]) + with pytest.raises(TypeError): + to_tdl_type_str(dict[list[Int16], Int8]) + with pytest.raises(TypeError): + to_tdl_type_str(dict[Float, Int8]) + with pytest.raises(TypeError): + to_tdl_type_str(dict[dict[Int8, Float], Int8]) From a8c8641fc8f524902d784395bfc737e6600a1d0a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 9 Aug 2025 17:47:36 -0400 Subject: [PATCH 75/91] Fix pytest --- test-tasks.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-tasks.yaml b/test-tasks.yaml index 88a2c838..ca635301 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -80,7 +80,7 @@ tasks: python-tests: dir: "{{.G_SRC_PYTHON_DIR}}" env: + # Don't create __pycache__ directories in source tree. PYTHONDONTWRITEBYTECODE: "1" cmds: - # Don't create __pycache__ directories in source tree. - - "uv run env pytest" + - "uv run pytest" From e558f67a068fff8ef135a4006ee644970c97d393 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 9 Aug 2025 18:46:12 -0400 Subject: [PATCH 76/91] Improve code according to coderabbit --- python/src/spider/type/tdl_convert.py | 13 +++++++------ python/src/spider/type/utils.py | 9 +++++++++ python/tests/type/test_to_tdl.py | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) create mode 100644 python/src/spider/type/utils.py diff --git a/python/src/spider/type/tdl_convert.py b/python/src/spider/type/tdl_convert.py index 35bff266..09a2f416 100644 --- a/python/src/spider/type/tdl_convert.py +++ b/python/src/spider/type/tdl_convert.py @@ -1,7 +1,7 @@ """Converts native types to TDL types.""" -import types from collections.abc import Collection +from types import GenericAlias from typing import get_args, get_origin from spider.type.tdl_type import ( @@ -18,9 +18,10 @@ TdlType, ) from spider.type.type import Double, Float, Int8, Int16, Int32, Int64 +from spider.type.utils import get_class_name -def to_primitive_tdl_type(native_type: type) -> TdlType | None: +def to_primitive_tdl_type(native_type: type | GenericAlias) -> TdlType | None: """ Converts a native type to primitive TDL type. :param native_type: @@ -44,7 +45,7 @@ def to_primitive_tdl_type(native_type: type) -> TdlType | None: return tdl_type -def to_tdl_type(native_type: type) -> TdlType: +def to_tdl_type(native_type: type | GenericAlias) -> TdlType: """ Converts a Python type to TDL type. :param native_type: @@ -59,7 +60,7 @@ def to_tdl_type(native_type: type) -> TdlType: msg = f"{native_type} is not a valid TDL type." raise TypeError(msg) - if isinstance(native_type, types.GenericAlias): + if isinstance(native_type, GenericAlias): origin = get_origin(native_type) if origin is list: arg = get_args(native_type) @@ -87,10 +88,10 @@ def to_tdl_type(native_type: type) -> TdlType: msg = f"{native_type} is not a valid TDL type." raise TypeError(msg) - return ClassType(native_type.__name__) + return ClassType(get_class_name(native_type)) -def to_tdl_type_str(native_type: type) -> str: +def to_tdl_type_str(native_type: type | GenericAlias) -> str: """ Converts a Python type to TDL type string. :param native_type: diff --git a/python/src/spider/type/utils.py b/python/src/spider/type/utils.py new file mode 100644 index 00000000..f1d3ae11 --- /dev/null +++ b/python/src/spider/type/utils.py @@ -0,0 +1,9 @@ +"""Utility for TDL types.""" + + +def get_class_name(cls: type) -> str: + """ + :param cls: + :return: Full name of `cls`. + """ + return f"{cls.__module__}.{cls.__qualname__}" diff --git a/python/tests/type/test_to_tdl.py b/python/tests/type/test_to_tdl.py index 6ffb98a5..c0fa734d 100644 --- a/python/tests/type/test_to_tdl.py +++ b/python/tests/type/test_to_tdl.py @@ -31,7 +31,7 @@ def test_to_tdl_map(self) -> None: def test_to_tdl_class(self) -> None: """Test converting class to TDL Types.""" - assert to_tdl_type_str(TestToTDL) == "TestToTDL" + assert to_tdl_type_str(TestToTDL) == "test_to_tdl.TestToTDL" def test_to_tdl_primitive_exception(self) -> None: """Test converting unsupported primitive types to TDL Types.""" From 0eea922c31628b07a843efcb08f71e19df3b8e9b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 19 Aug 2025 15:24:42 -0400 Subject: [PATCH 77/91] Restructure files --- python/{ => spider-py}/README.md | 0 python/{ => spider-py}/pyproject.toml | 0 python/{src/spider => spider-py/src/spider_py}/__init__.py | 2 +- .../spider => spider-py/src/spider_py}/client/__init__.py | 0 .../spider => spider-py/src/spider_py}/core/__init__.py | 0 python/{src/spider => spider-py/src/spider_py}/core/task.py | 0 .../src/spider_py}/task_executor/__init__.py | 0 .../src/spider_py}/task_executor/task_executor.py | 0 python/spider-py/src/spider_py/type/__init__.py | 6 ++++++ .../spider => spider-py/src/spider_py}/type/tdl_convert.py | 6 +++--- .../spider => spider-py/src/spider_py}/type/tdl_type.py | 0 python/{src/spider => spider-py/src/spider_py}/type/type.py | 0 .../{src/spider => spider-py/src/spider_py}/type/utils.py | 0 python/{ => spider-py}/tests/type/test_to_tdl.py | 4 ++-- python/{ => spider-py}/uv.lock | 0 python/src/spider/type/__init__.py | 6 ------ 16 files changed, 12 insertions(+), 12 deletions(-) rename python/{ => spider-py}/README.md (100%) rename python/{ => spider-py}/pyproject.toml (100%) rename python/{src/spider => spider-py/src/spider_py}/__init__.py (64%) rename python/{src/spider => spider-py/src/spider_py}/client/__init__.py (100%) rename python/{src/spider => spider-py/src/spider_py}/core/__init__.py (100%) rename python/{src/spider => spider-py/src/spider_py}/core/task.py (100%) rename python/{src/spider => spider-py/src/spider_py}/task_executor/__init__.py (100%) rename python/{src/spider => spider-py/src/spider_py}/task_executor/task_executor.py (100%) create mode 100644 python/spider-py/src/spider_py/type/__init__.py rename python/{src/spider => spider-py/src/spider_py}/type/tdl_convert.py (94%) rename python/{src/spider => spider-py/src/spider_py}/type/tdl_type.py (100%) rename python/{src/spider => spider-py/src/spider_py}/type/type.py (100%) rename python/{src/spider => spider-py/src/spider_py}/type/utils.py (100%) rename python/{ => spider-py}/tests/type/test_to_tdl.py (96%) rename python/{ => spider-py}/uv.lock (100%) delete mode 100644 python/src/spider/type/__init__.py diff --git a/python/README.md b/python/spider-py/README.md similarity index 100% rename from python/README.md rename to python/spider-py/README.md diff --git a/python/pyproject.toml b/python/spider-py/pyproject.toml similarity index 100% rename from python/pyproject.toml rename to python/spider-py/pyproject.toml diff --git a/python/src/spider/__init__.py b/python/spider-py/src/spider_py/__init__.py similarity index 64% rename from python/src/spider/__init__.py rename to python/spider-py/src/spider_py/__init__.py index 929a7aaa..905b7a0e 100644 --- a/python/src/spider/__init__.py +++ b/python/spider-py/src/spider_py/__init__.py @@ -1,6 +1,6 @@ """Spider package root.""" -from spider.type import Double, Float, Int8, Int16, Int32, Int64 +from spider_py.type import Double, Float, Int8, Int16, Int32, Int64 __all__ = [ "Double", diff --git a/python/src/spider/client/__init__.py b/python/spider-py/src/spider_py/client/__init__.py similarity index 100% rename from python/src/spider/client/__init__.py rename to python/spider-py/src/spider_py/client/__init__.py diff --git a/python/src/spider/core/__init__.py b/python/spider-py/src/spider_py/core/__init__.py similarity index 100% rename from python/src/spider/core/__init__.py rename to python/spider-py/src/spider_py/core/__init__.py diff --git a/python/src/spider/core/task.py b/python/spider-py/src/spider_py/core/task.py similarity index 100% rename from python/src/spider/core/task.py rename to python/spider-py/src/spider_py/core/task.py diff --git a/python/src/spider/task_executor/__init__.py b/python/spider-py/src/spider_py/task_executor/__init__.py similarity index 100% rename from python/src/spider/task_executor/__init__.py rename to python/spider-py/src/spider_py/task_executor/__init__.py diff --git a/python/src/spider/task_executor/task_executor.py b/python/spider-py/src/spider_py/task_executor/task_executor.py similarity index 100% rename from python/src/spider/task_executor/task_executor.py rename to python/spider-py/src/spider_py/task_executor/task_executor.py diff --git a/python/spider-py/src/spider_py/type/__init__.py b/python/spider-py/src/spider_py/type/__init__.py new file mode 100644 index 00000000..c943af9d --- /dev/null +++ b/python/spider-py/src/spider_py/type/__init__.py @@ -0,0 +1,6 @@ +"""Spider type package.""" + +from spider_py.type.tdl_convert import to_tdl_type_str +from spider_py.type.type import Double, Float, Int8, Int16, Int32, Int64 + +__all__ = ["Double", "Float", "Int8", "Int16", "Int32", "Int64", "to_tdl_type_str"] diff --git a/python/src/spider/type/tdl_convert.py b/python/spider-py/src/spider_py/type/tdl_convert.py similarity index 94% rename from python/src/spider/type/tdl_convert.py rename to python/spider-py/src/spider_py/type/tdl_convert.py index 09a2f416..0d449823 100644 --- a/python/src/spider/type/tdl_convert.py +++ b/python/spider-py/src/spider_py/type/tdl_convert.py @@ -4,7 +4,7 @@ from types import GenericAlias from typing import get_args, get_origin -from spider.type.tdl_type import ( +from spider_py.type.tdl_type import ( BoolType, ClassType, DoubleType, @@ -17,8 +17,8 @@ MapType, TdlType, ) -from spider.type.type import Double, Float, Int8, Int16, Int32, Int64 -from spider.type.utils import get_class_name +from spider_py.type.type import Double, Float, Int8, Int16, Int32, Int64 +from spider_py.type.utils import get_class_name def to_primitive_tdl_type(native_type: type | GenericAlias) -> TdlType | None: diff --git a/python/src/spider/type/tdl_type.py b/python/spider-py/src/spider_py/type/tdl_type.py similarity index 100% rename from python/src/spider/type/tdl_type.py rename to python/spider-py/src/spider_py/type/tdl_type.py diff --git a/python/src/spider/type/type.py b/python/spider-py/src/spider_py/type/type.py similarity index 100% rename from python/src/spider/type/type.py rename to python/spider-py/src/spider_py/type/type.py diff --git a/python/src/spider/type/utils.py b/python/spider-py/src/spider_py/type/utils.py similarity index 100% rename from python/src/spider/type/utils.py rename to python/spider-py/src/spider_py/type/utils.py diff --git a/python/tests/type/test_to_tdl.py b/python/spider-py/tests/type/test_to_tdl.py similarity index 96% rename from python/tests/type/test_to_tdl.py rename to python/spider-py/tests/type/test_to_tdl.py index c0fa734d..f3caf3a0 100644 --- a/python/tests/type/test_to_tdl.py +++ b/python/spider-py/tests/type/test_to_tdl.py @@ -2,8 +2,8 @@ import pytest -from spider import Double, Float, Int8, Int16, Int32, Int64 -from spider.type import to_tdl_type_str +from spider_py import Double, Float, Int8, Int16, Int32, Int64 +from spider_py.type import to_tdl_type_str class TestToTDL: diff --git a/python/uv.lock b/python/spider-py/uv.lock similarity index 100% rename from python/uv.lock rename to python/spider-py/uv.lock diff --git a/python/src/spider/type/__init__.py b/python/src/spider/type/__init__.py deleted file mode 100644 index e0c873db..00000000 --- a/python/src/spider/type/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Spider type package.""" - -from spider.type.tdl_convert import to_tdl_type_str -from spider.type.type import Double, Float, Int8, Int16, Int32, Int64 - -__all__ = ["Double", "Float", "Int8", "Int16", "Int32", "Int64", "to_tdl_type_str"] From 6eda1bdef03b9e0b860adefc0faf4452461e7153 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 19 Aug 2025 15:34:44 -0400 Subject: [PATCH 78/91] Clean up merge --- lint-tasks.yaml | 5 ----- python/.python-version | 1 - 2 files changed, 6 deletions(-) delete mode 100644 python/.python-version diff --git a/lint-tasks.yaml b/lint-tasks.yaml index bad088de..d194d951 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -139,11 +139,6 @@ tasks: mypy "{{.ITEM}}" ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}" ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}" - - cmd: |- - cd "{{.G_SRC_PYTHON_DIR}}" - uv run mypy . - uv run ruff check {{.RUFF_CHECK_FLAGS}} - uv run ruff format {{.RUFF_FORMAT_FLAGS}} spider-py-check: cmds: diff --git a/python/.python-version b/python/.python-version deleted file mode 100644 index 24ee5b1b..00000000 --- a/python/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.13 From 0713c0d089f137d1864e02b95c3a5ce4497da6b6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 21 Aug 2025 22:01:10 -0400 Subject: [PATCH 79/91] Use future annotation and disable Self return type check --- python/spider-py/pyproject.toml | 1 + .../spider-py/src/spider_py/type/tdl_type.py | 24 +++++++++---------- python/spider-py/src/spider_py/type/type.py | 16 +++++++------ 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/python/spider-py/pyproject.toml b/python/spider-py/pyproject.toml index 730ec260..b173ae24 100644 --- a/python/spider-py/pyproject.toml +++ b/python/spider-py/pyproject.toml @@ -67,6 +67,7 @@ ignore = [ "UP015", # Explicit open modes are helpful ] isort.order-by-type = false +typing-extensions = false # Disable type extensions for 3.10 compatibility [tool.ruff.lint.per-file-ignores] "tests/**" = [ diff --git a/python/spider-py/src/spider_py/type/tdl_type.py b/python/spider-py/src/spider_py/type/tdl_type.py index 5f484ca8..f72f6f33 100644 --- a/python/spider-py/src/spider_py/type/tdl_type.py +++ b/python/spider-py/src/spider_py/type/tdl_type.py @@ -77,23 +77,23 @@ def __init__(self, name: str) -> None: Creates a TDL custom class type. :param name: The name of the class. """ - self.name = name + self._name = name @override def type_str(self) -> str: - return self.name + return self._name class ListType(TdlType): """TDL List type.""" - def __init__(self, key: TdlType) -> None: + def __init__(self, element_type: TdlType) -> None: """Creates a TDL list type.""" - self.key = key + self.element_type = element_type @override def type_str(self) -> str: - return f"List<{self.key.type_str()}>" + return f"List<{self.element_type.type_str()}>" def is_integral(tdl_type: TdlType) -> bool: @@ -103,7 +103,7 @@ def is_integral(tdl_type: TdlType) -> bool: def is_string(tdl_type: TdlType) -> bool: """:return: If TDL type is a string type, i.e. List.""" - return isinstance(tdl_type, ListType) and isinstance(tdl_type.key, Int8Type) + return isinstance(tdl_type, ListType) and isinstance(tdl_type.element_type, Int8Type) def is_map_key(tdl_type: TdlType) -> bool: @@ -114,17 +114,17 @@ def is_map_key(tdl_type: TdlType) -> bool: class MapType(TdlType): """TDL Map type.""" - def __init__(self, key: TdlType, value: TdlType) -> None: + def __init__(self, key_type: TdlType, value_type: TdlType) -> None: """ Creates a TDL map type. :raises TypeError: If key is not a supported type. """ - if not is_map_key(key): - msg = f"{key} is not a supported type for map key." + if not is_map_key(key_type): + msg = f"{key_type} is not a supported type for map key." raise TypeError(msg) - self.key = key - self.value = value + self.key_type = key_type + self.value_type = value_type @override def type_str(self) -> str: - return f"Map<{self.key.type_str()},{self.value.type_str()}>" + return f"Map<{self.key_type.type_str()},{self.value_type.type_str()}>" diff --git a/python/spider-py/src/spider_py/type/type.py b/python/spider-py/src/spider_py/type/type.py index 903a9972..b8fa16a4 100644 --- a/python/spider-py/src/spider_py/type/type.py +++ b/python/spider-py/src/spider_py/type/type.py @@ -1,12 +1,14 @@ """Custom type module for Spider.""" +from __future__ import annotations + from typing import cast class BoundedInt(int): """Bounded integer type.""" - def __new__(cls, value: int, bits: int = 32) -> "BoundedInt": + def __new__(cls, value: int, bits: int = 32) -> BoundedInt: """Creates a bounded integer.""" min_val = -(1 << (bits - 1)) max_val = (1 << (bits - 1)) - 1 @@ -21,7 +23,7 @@ def __new__(cls, value: int, bits: int = 32) -> "BoundedInt": class Int8(BoundedInt): """8 bits integer type.""" - def __new__(cls, value: int) -> "Int8": + def __new__(cls, value: int) -> Int8: """Creates an int8 integer.""" return cast("Int8", super().__new__(cls, value, bits=8)) @@ -29,7 +31,7 @@ def __new__(cls, value: int) -> "Int8": class Int16(BoundedInt): """16 bits integer type.""" - def __new__(cls, value: int) -> "Int16": + def __new__(cls, value: int) -> Int16: """Creates an int16 integer.""" return cast("Int16", super().__new__(cls, value, bits=16)) @@ -37,7 +39,7 @@ def __new__(cls, value: int) -> "Int16": class Int32(BoundedInt): """32 bits integer type.""" - def __new__(cls, value: int) -> "Int32": + def __new__(cls, value: int) -> Int32: """Creates an int32 integer.""" return cast("Int32", super().__new__(cls, value, bits=32)) @@ -45,7 +47,7 @@ def __new__(cls, value: int) -> "Int32": class Int64(BoundedInt): """64 bits integer type.""" - def __new__(cls, value: int) -> "Int64": + def __new__(cls, value: int) -> Int64: """Creates an int64 integer.""" return cast("Int64", super().__new__(cls, value, bits=64)) @@ -53,7 +55,7 @@ def __new__(cls, value: int) -> "Int64": class Float(float): """Float type.""" - def __new__(cls, value: float) -> "Float": + def __new__(cls, value: float) -> Float: """Creates a float number.""" return super().__new__(cls, value) @@ -61,6 +63,6 @@ def __new__(cls, value: float) -> "Float": class Double(float): """Double type.""" - def __new__(cls, value: float) -> "Double": + def __new__(cls, value: float) -> Double: """Creates a double number.""" return super().__new__(cls, value) From 3b8cfe9e19def469ed060b4b02666aaf0ff0964a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 21 Aug 2025 22:02:34 -0400 Subject: [PATCH 80/91] Avoid using chain comparision --- python/spider-py/src/spider_py/type/type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/spider-py/src/spider_py/type/type.py b/python/spider-py/src/spider_py/type/type.py index b8fa16a4..dabefc4e 100644 --- a/python/spider-py/src/spider_py/type/type.py +++ b/python/spider-py/src/spider_py/type/type.py @@ -13,7 +13,7 @@ def __new__(cls, value: int, bits: int = 32) -> BoundedInt: min_val = -(1 << (bits - 1)) max_val = (1 << (bits - 1)) - 1 - if not (min_val <= value <= max_val): + if not min_val <= value and value <= max_val: msg = f"Bounded integer value ({value}) must be between {min_val} and {max_val}" raise ValueError(msg) From c9cb4b68b2f819fd920e73d6f8fcd85c3132b2e6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 22 Aug 2025 00:44:12 -0400 Subject: [PATCH 81/91] Apply suggestions from code review Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- python/spider-py/src/spider_py/type/tdl_convert.py | 5 ++--- python/spider-py/src/spider_py/type/type.py | 6 +++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/python/spider-py/src/spider_py/type/tdl_convert.py b/python/spider-py/src/spider_py/type/tdl_convert.py index 0d449823..6ef2c8a8 100644 --- a/python/spider-py/src/spider_py/type/tdl_convert.py +++ b/python/spider-py/src/spider_py/type/tdl_convert.py @@ -93,9 +93,8 @@ def to_tdl_type(native_type: type | GenericAlias) -> TdlType: def to_tdl_type_str(native_type: type | GenericAlias) -> str: """ - Converts a Python type to TDL type string. - :param native_type: - :return: + :param native_type: A Python native type. + :return: A string representation of the TDL type. :raise: TypeError if `native_type` is not a valid TDL type. """ return to_tdl_type(native_type).type_str() diff --git a/python/spider-py/src/spider_py/type/type.py b/python/spider-py/src/spider_py/type/type.py index dabefc4e..f1257ccf 100644 --- a/python/spider-py/src/spider_py/type/type.py +++ b/python/spider-py/src/spider_py/type/type.py @@ -13,7 +13,11 @@ def __new__(cls, value: int, bits: int = 32) -> BoundedInt: min_val = -(1 << (bits - 1)) max_val = (1 << (bits - 1)) - 1 - if not min_val <= value and value <= max_val: + if bits not in (8, 16, 32, 64): + msg = f"Unsupported bits size: {bits}. Supported sizes are 8, 16, 32, or 64." + raise ValueError(msg) + + if not (min_val <= value and value <= max_val): msg = f"Bounded integer value ({value}) must be between {min_val} and {max_val}" raise ValueError(msg) From 412a2b58037d553d7c950d86fb4bd2124a1285ca Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 22 Aug 2025 09:34:20 -0400 Subject: [PATCH 82/91] Disable chained boolean comparison in ruff --- python/spider-py/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/spider-py/pyproject.toml b/python/spider-py/pyproject.toml index b173ae24..fe4217fb 100644 --- a/python/spider-py/pyproject.toml +++ b/python/spider-py/pyproject.toml @@ -60,6 +60,7 @@ ignore = [ "FIX002", # Allow todo statements "PERF401", # Allow for loops when creating lists "PERF403", # Allow for loops when creating dicts + "PLR1716", # Disable chained boolean comparisons "S311", # Allow usage of `random` package "SIM102", # Allow collapsible if statements for readability "TD002", # Author unnecessary for todo statement From bab48068e859cfccbcdd45e1c74989ddc22d1798 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 22 Aug 2025 20:51:09 -0400 Subject: [PATCH 83/91] Apply suggestions from code review Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- .../src/spider_py/type/tdl_convert.py | 8 ++++--- .../spider-py/src/spider_py/type/tdl_type.py | 22 +++++++++++++++---- python/spider-py/src/spider_py/type/type.py | 2 +- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/python/spider-py/src/spider_py/type/tdl_convert.py b/python/spider-py/src/spider_py/type/tdl_convert.py index 6ef2c8a8..5dabdfdc 100644 --- a/python/spider-py/src/spider_py/type/tdl_convert.py +++ b/python/spider-py/src/spider_py/type/tdl_convert.py @@ -25,7 +25,9 @@ def to_primitive_tdl_type(native_type: type | GenericAlias) -> TdlType | None: """ Converts a native type to primitive TDL type. :param native_type: - :return: Converted TDL primitive. None if `native_type` is not a supported primitive type. + :return: + - The converted TDL primitive if `native_type` is supported. + - None otherwise. """ tdl_type: TdlType | None = None if native_type is Int8: @@ -49,7 +51,7 @@ def to_tdl_type(native_type: type | GenericAlias) -> TdlType: """ Converts a Python type to TDL type. :param native_type: - :return: + :return: The converted TDL type. :raise: TypeError if `native_type` is not a valid TDL type. """ primitive_tdl_type = to_primitive_tdl_type(native_type) @@ -65,7 +67,7 @@ def to_tdl_type(native_type: type | GenericAlias) -> TdlType: if origin is list: arg = get_args(native_type) if arg is None: - msg = "List does not have a key type." + msg = "List does not have an element type." raise TypeError(msg) arg = arg[0] return ListType(to_tdl_type(arg)) diff --git a/python/spider-py/src/spider_py/type/tdl_type.py b/python/spider-py/src/spider_py/type/tdl_type.py index f72f6f33..cc90e8e6 100644 --- a/python/spider-py/src/spider_py/type/tdl_type.py +++ b/python/spider-py/src/spider_py/type/tdl_type.py @@ -88,7 +88,10 @@ class ListType(TdlType): """TDL List type.""" def __init__(self, element_type: TdlType) -> None: - """Creates a TDL list type.""" + """ + Creates a TDL list type. + :param element_type: + """ self.element_type = element_type @override @@ -97,17 +100,26 @@ def type_str(self) -> str: def is_integral(tdl_type: TdlType) -> bool: - """:return: If TDL type is an integral type.""" + """ + :param tdl_type: + :return: If `tdl_type` is a TDL integral type. + """ return isinstance(tdl_type, (Int8Type, Int16Type, Int32Type, Int64Type)) def is_string(tdl_type: TdlType) -> bool: - """:return: If TDL type is a string type, i.e. List.""" + """ + :param tdl_type: + :return: Whether `tdl_type` is a TDL string type, i.e. `List`. + """ return isinstance(tdl_type, ListType) and isinstance(tdl_type.element_type, Int8Type) def is_map_key(tdl_type: TdlType) -> bool: - """:return: If TDL type is a valid type for map key.""" + """ + :param tdl_type: + :return: Whether `tdl_type` is a supported key type of a map. + """ return is_integral(tdl_type) or is_string(tdl_type) @@ -117,6 +129,8 @@ class MapType(TdlType): def __init__(self, key_type: TdlType, value_type: TdlType) -> None: """ Creates a TDL map type. + :param key_type: + :param value_type: :raises TypeError: If key is not a supported type. """ if not is_map_key(key_type): diff --git a/python/spider-py/src/spider_py/type/type.py b/python/spider-py/src/spider_py/type/type.py index f1257ccf..484868a3 100644 --- a/python/spider-py/src/spider_py/type/type.py +++ b/python/spider-py/src/spider_py/type/type.py @@ -18,7 +18,7 @@ def __new__(cls, value: int, bits: int = 32) -> BoundedInt: raise ValueError(msg) if not (min_val <= value and value <= max_val): - msg = f"Bounded integer value ({value}) must be between {min_val} and {max_val}" + msg = f"Bounded integer value ({value}) must be between {min_val} and {max_val}." raise ValueError(msg) return super().__new__(cls, value) From 096eda9c14e8c8379f29b6490e6989e59cadb737 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 22 Aug 2025 20:52:30 -0400 Subject: [PATCH 84/91] Use whether for boolean return type docstring --- python/spider-py/src/spider_py/type/tdl_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/spider-py/src/spider_py/type/tdl_type.py b/python/spider-py/src/spider_py/type/tdl_type.py index cc90e8e6..f5c4e15b 100644 --- a/python/spider-py/src/spider_py/type/tdl_type.py +++ b/python/spider-py/src/spider_py/type/tdl_type.py @@ -102,7 +102,7 @@ def type_str(self) -> str: def is_integral(tdl_type: TdlType) -> bool: """ :param tdl_type: - :return: If `tdl_type` is a TDL integral type. + :return: Whether `tdl_type` is a TDL integral type. """ return isinstance(tdl_type, (Int8Type, Int16Type, Int32Type, Int64Type)) From be05c3845a4b237e9fd9dd5f9985d393fd2b34d1 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 22 Aug 2025 20:55:07 -0400 Subject: [PATCH 85/91] Fix BoundedInt --- python/spider-py/src/spider_py/type/type.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/python/spider-py/src/spider_py/type/type.py b/python/spider-py/src/spider_py/type/type.py index 484868a3..fcf5a584 100644 --- a/python/spider-py/src/spider_py/type/type.py +++ b/python/spider-py/src/spider_py/type/type.py @@ -10,15 +10,17 @@ class BoundedInt(int): def __new__(cls, value: int, bits: int = 32) -> BoundedInt: """Creates a bounded integer.""" - min_val = -(1 << (bits - 1)) - max_val = (1 << (bits - 1)) - 1 - if bits not in (8, 16, 32, 64): msg = f"Unsupported bits size: {bits}. Supported sizes are 8, 16, 32, or 64." raise ValueError(msg) - if not (min_val <= value and value <= max_val): - msg = f"Bounded integer value ({value}) must be between {min_val} and {max_val}." + lower_bound = -(1 << (bits - 1)) + upper_bound = (1 << (bits - 1)) - 1 + + if not (lower_bound <= value and value <= upper_bound): + msg = ( + f"Bounded integer value ({value}) must be between {lower_bound} and {upper_bound}." + ) raise ValueError(msg) return super().__new__(cls, value) From 5cd5699bae56f3369cc02d0bcc716ae2823023de Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 22 Aug 2025 21:07:15 -0400 Subject: [PATCH 86/91] Rename private functions --- python/spider-py/src/spider_py/type/tdl_convert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/spider-py/src/spider_py/type/tdl_convert.py b/python/spider-py/src/spider_py/type/tdl_convert.py index 5dabdfdc..45a44aa2 100644 --- a/python/spider-py/src/spider_py/type/tdl_convert.py +++ b/python/spider-py/src/spider_py/type/tdl_convert.py @@ -21,7 +21,7 @@ from spider_py.type.utils import get_class_name -def to_primitive_tdl_type(native_type: type | GenericAlias) -> TdlType | None: +def _to_primitive_tdl_type(native_type: type | GenericAlias) -> TdlType | None: """ Converts a native type to primitive TDL type. :param native_type: @@ -54,7 +54,7 @@ def to_tdl_type(native_type: type | GenericAlias) -> TdlType: :return: The converted TDL type. :raise: TypeError if `native_type` is not a valid TDL type. """ - primitive_tdl_type = to_primitive_tdl_type(native_type) + primitive_tdl_type = _to_primitive_tdl_type(native_type) if primitive_tdl_type is not None: return primitive_tdl_type From bdf1d8fb29e9382531de4c90ae51aa04b6be434e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 22 Aug 2025 21:12:56 -0400 Subject: [PATCH 87/91] Fix get_args --- .../spider-py/src/spider_py/type/tdl_convert.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/python/spider-py/src/spider_py/type/tdl_convert.py b/python/spider-py/src/spider_py/type/tdl_convert.py index 45a44aa2..e19b70e2 100644 --- a/python/spider-py/src/spider_py/type/tdl_convert.py +++ b/python/spider-py/src/spider_py/type/tdl_convert.py @@ -65,22 +65,20 @@ def to_tdl_type(native_type: type | GenericAlias) -> TdlType: if isinstance(native_type, GenericAlias): origin = get_origin(native_type) if origin is list: - arg = get_args(native_type) - if arg is None: + args = get_args(native_type) + if len(args) == 0: msg = "List does not have an element type." raise TypeError(msg) - arg = arg[0] + arg = args[0] return ListType(to_tdl_type(arg)) if origin is dict: - arg = get_args(native_type) + args = get_args(native_type) msg = "Dict does not have a key/value type." - if arg is None: + if len(args) != 2: # noqa: PLR2004 raise TypeError(msg) - if len(arg) != 2: # noqa: PLR2004 - raise TypeError(msg) - key = arg[0] - value = arg[1] + key = args[0] + value = args[1] return MapType(to_tdl_type(key), to_tdl_type(value)) msg = f"{native_type} is not a valid TDL type." From 5602b11d92b0cd96c9ae5913589c109fd78bbbe3 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 22 Aug 2025 21:19:25 -0400 Subject: [PATCH 88/91] Update python/spider-py/pyproject.toml Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- python/spider-py/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/spider-py/pyproject.toml b/python/spider-py/pyproject.toml index fe4217fb..47bac154 100644 --- a/python/spider-py/pyproject.toml +++ b/python/spider-py/pyproject.toml @@ -60,7 +60,7 @@ ignore = [ "FIX002", # Allow todo statements "PERF401", # Allow for loops when creating lists "PERF403", # Allow for loops when creating dicts - "PLR1716", # Disable chained boolean comparisons + "PLR1716", # Allow chained boolean comparisons "S311", # Allow usage of `random` package "SIM102", # Allow collapsible if statements for readability "TD002", # Author unnecessary for todo statement From 0696cacfc991bc88e3e02bf51fc09b70ab390677 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 22 Aug 2025 21:27:25 -0400 Subject: [PATCH 89/91] Fix exception docstring --- python/spider-py/src/spider_py/type/tdl_convert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/spider-py/src/spider_py/type/tdl_convert.py b/python/spider-py/src/spider_py/type/tdl_convert.py index e19b70e2..468e379e 100644 --- a/python/spider-py/src/spider_py/type/tdl_convert.py +++ b/python/spider-py/src/spider_py/type/tdl_convert.py @@ -52,7 +52,7 @@ def to_tdl_type(native_type: type | GenericAlias) -> TdlType: Converts a Python type to TDL type. :param native_type: :return: The converted TDL type. - :raise: TypeError if `native_type` is not a valid TDL type. + :raises TypeError: If `native_type` is not a valid TDL type. """ primitive_tdl_type = _to_primitive_tdl_type(native_type) if primitive_tdl_type is not None: @@ -95,6 +95,6 @@ def to_tdl_type_str(native_type: type | GenericAlias) -> str: """ :param native_type: A Python native type. :return: A string representation of the TDL type. - :raise: TypeError if `native_type` is not a valid TDL type. + :raises TypeError: If `native_type` is not a valid TDL type. """ return to_tdl_type(native_type).type_str() From d6e32b35dac62aad306656bbd341f5fe3d3557b8 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 22 Aug 2025 23:29:03 -0400 Subject: [PATCH 90/91] Move native primitive type check inside _to_primitive_tdl_type --- python/spider-py/src/spider_py/type/tdl_convert.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/python/spider-py/src/spider_py/type/tdl_convert.py b/python/spider-py/src/spider_py/type/tdl_convert.py index 468e379e..c6fa40e5 100644 --- a/python/spider-py/src/spider_py/type/tdl_convert.py +++ b/python/spider-py/src/spider_py/type/tdl_convert.py @@ -27,7 +27,8 @@ def _to_primitive_tdl_type(native_type: type | GenericAlias) -> TdlType | None: :param native_type: :return: - The converted TDL primitive if `native_type` is supported. - - None otherwise. + - None if `native_type` is not a primitive Python type. + :raises TypeError: If `native_type` is a primitive Python type not supported by TDL. """ tdl_type: TdlType | None = None if native_type is Int8: @@ -44,6 +45,11 @@ def _to_primitive_tdl_type(native_type: type | GenericAlias) -> TdlType | None: tdl_type = DoubleType() elif native_type is bool: tdl_type = BoolType() + + if native_type in (int, float, str, complex, bytes): + msg = f"{native_type} is not a TDL type. Please use the corresponding TDL primitive type." + raise TypeError(msg) + return tdl_type @@ -58,10 +64,6 @@ def to_tdl_type(native_type: type | GenericAlias) -> TdlType: if primitive_tdl_type is not None: return primitive_tdl_type - if native_type in (int, float, str, complex, bytes): - msg = f"{native_type} is not a valid TDL type." - raise TypeError(msg) - if isinstance(native_type, GenericAlias): origin = get_origin(native_type) if origin is list: From e916cb9ba39c05cbfe6124cf20232e20233942e4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 23 Aug 2025 13:10:59 -0400 Subject: [PATCH 91/91] Use type dict to convert primitive type --- .../src/spider_py/type/tdl_convert.py | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/python/spider-py/src/spider_py/type/tdl_convert.py b/python/spider-py/src/spider_py/type/tdl_convert.py index c6fa40e5..3caad86e 100644 --- a/python/spider-py/src/spider_py/type/tdl_convert.py +++ b/python/spider-py/src/spider_py/type/tdl_convert.py @@ -20,6 +20,16 @@ from spider_py.type.type import Double, Float, Int8, Int16, Int32, Int64 from spider_py.type.utils import get_class_name +TypeDict = { + Int8: Int8Type(), + Int16: Int16Type(), + Int32: Int32Type(), + Int64: Int64Type(), + Float: FloatType(), + Double: DoubleType(), + bool: BoolType(), +} + def _to_primitive_tdl_type(native_type: type | GenericAlias) -> TdlType | None: """ @@ -30,27 +40,14 @@ def _to_primitive_tdl_type(native_type: type | GenericAlias) -> TdlType | None: - None if `native_type` is not a primitive Python type. :raises TypeError: If `native_type` is a primitive Python type not supported by TDL. """ - tdl_type: TdlType | None = None - if native_type is Int8: - tdl_type = Int8Type() - elif native_type is Int16: - tdl_type = Int16Type() - elif native_type is Int32: - tdl_type = Int32Type() - elif native_type is Int64: - tdl_type = Int64Type() - elif native_type is Float: - tdl_type = FloatType() - elif native_type is Double: - tdl_type = DoubleType() - elif native_type is bool: - tdl_type = BoolType() + if isinstance(native_type, type) and native_type in TypeDict: + return TypeDict[native_type] if native_type in (int, float, str, complex, bytes): msg = f"{native_type} is not a TDL type. Please use the corresponding TDL primitive type." raise TypeError(msg) - return tdl_type + return None def to_tdl_type(native_type: type | GenericAlias) -> TdlType: