Merge remote-tracking branch 'upstream/main' into dov/password-auth-scratch

Author: DAlperin

Cargo.lock

@@ -1,26 +1,29 @@
 [workspace]
 members = [
+    "misc/bazel/cargo-gazelle",
     "src/adapter",
     "src/adapter-types",
     "src/alloc",
     "src/alloc-default",
     "src/arrow-util",
     "src/audit-log",
+    "src/auth",
     "src/avro",
-    "src/aws-util",
     "src/aws-secrets-controller",
+    "src/aws-util",
     "src/balancerd",
     "src/build-info",
     "src/build-tools",
-    "src/ccsr",
     "src/catalog",
     "src/catalog-debug",
     "src/catalog-protos",
+    "src/ccsr",
     "src/cloud-api",
     "src/cloud-provider",
+    "src/cloud-resources",
     "src/cluster",
-    "src/clusterd",
     "src/cluster-client",
+    "src/clusterd",
     "src/compute",
     "src/compute-client",
     "src/compute-types",
@@ -32,6 +35,8 @@ members = [
     "src/dyncfgs",
     "src/environmentd",
     "src/expr",
+    "src/expr-derive",
+    "src/expr-derive-impl",
     "src/expr-parser",
     "src/expr-test-util",
     "src/fivetran-destination",
@@ -44,13 +49,14 @@ members = [
     "src/license-keys",
     "src/lowertest",
     "src/lowertest-derive",
+    "src/lsp-server",
+    "src/materialized",
     "src/materialized",
     "src/metabase",
+    "src/metrics",
     "src/mysql-util",
     "src/mz",
-    "src/lsp-server",
-    "src/materialized",
-    "src/metrics",
+    "src/mz-debug",
     "src/npm",
     "src/orchestrator",
     "src/orchestrator-kubernetes",
@@ -64,7 +70,6 @@ members = [
     "src/persist-cli",
     "src/persist-client",
     "src/persist-proc",
-    "src/txn-wal",
     "src/persist-types",
     "src/pgcopy",
     "src/pgrepr",
@@ -78,23 +83,23 @@ members = [
     "src/prof",
     "src/prof-http",
     "src/proto",
+    "src/regexp",
     "src/repr",
     "src/repr-test-util",
     "src/rocksdb",
     "src/rocksdb-types",
     "src/s3-datagen",
     "src/secrets",
     "src/segment",
-    "src/self-managed-debug",
     "src/server-core",
     "src/service",
-    "src/ssh-util",
     "src/sql",
     "src/sql-lexer",
     "src/sql-parser",
     "src/sql-pretty",
     "src/sql-server-util",
     "src/sqllogictest",
+    "src/ssh-util",
     "src/storage",
     "src/storage-client",
     "src/storage-controller",
@@ -106,14 +111,11 @@ members = [
     "src/tls-util",
     "src/tracing",
     "src/transform",
-    "src/cloud-resources",
+    "src/txn-wal",
     "src/walkabout",
     "src/workspace-hack",
     "test/metabase/smoketest",
     "test/test-util",
-    "src/regexp",
-    "misc/bazel/cargo-gazelle",
-    "src/auth",
 ]
 
 # We make all members of the workspace default members, *except* for a few
@@ -156,6 +158,8 @@ default-members = [
     "src/dyncfgs",
     "src/environmentd",
     "src/expr",
+    "src/expr-derive",
+    "src/expr-derive-impl",
     "src/expr-parser",
     "src/expr-test-util",
     "src/frontegg-auth",
@@ -173,6 +177,7 @@ default-members = [
     "src/metrics",
     "src/mysql-util",
     "src/mz",
+    "src/mz-debug",
     "src/npm",
     "src/orchestrator",
     "src/orchestrator-kubernetes",
@@ -207,7 +212,6 @@ default-members = [
     "src/s3-datagen",
     "src/secrets",
     "src/segment",
-    "src/self-managed-debug",
     "src/server-core",
     "src/service",
     "src/sql",
@@ -240,6 +244,8 @@ exclude = [
     # All WASM crates are split into their own workspace to avoid needles cache
     #  invalidations for the core Mz crates.
     "misc/wasm/*",
+    # Ignore any Rust dependencies that python packages might pull in.
+    "misc/python/venv/*",
 ]
 
 # Use Cargo's new feature resolver, which can handle target-specific features.
@@ -271,6 +277,10 @@ differential-dataflow = { opt-level = 3 }
 mz-compute = { opt-level = 3 }
 mz-transform = { opt-level = 3 }
 
+# Recommended by insta
+insta = { opt-level = 3 }
+similar = { opt-level = 3 }
+
 [profile.release]
 # Compile time seems similar to "lto = false", runtime ~10% faster
 lto = "thin"
@@ -313,11 +323,8 @@ serde-value = { git = "https://github.com/MaterializeInc/serde-value.git" }
 # upstream.
 tracing-opentelemetry = { git = "https://github.com/MaterializeInc/tracing-opentelemetry.git" }
 
-# Waiting on https://github.com/launchdarkly/rust-server-sdk/pull/20 to make
-# it into a release.
-# Also bumps lru to 0.12.0
-# Needs a more complex update to 2.0.0, with a custom TLS connector.
-launchdarkly-server-sdk = { git = "https://github.com/MaterializeInc/rust-server-sdk", rev = "87c0f67aa51fa936d3a6a49605e8e3f565d15281" }
+# Waiting for resolution of https://github.com/launchdarkly/rust-server-sdk/issues/116
+launchdarkly-server-sdk = { git = "https://github.com/MaterializeInc/rust-server-sdk", rev = "23666d095c39adc53a151969c1aa1e1ec484f3e6" }
 
 # Waiting on https://github.com/edenhill/librdkafka/pull/4051.
 rdkafka = { git = "https://github.com/MaterializeInc/rust-rdkafka.git" }
@@ -327,6 +334,11 @@ rdkafka-sys = { git = "https://github.com/MaterializeInc/rust-rdkafka.git" }
 reqwest-middleware = { git = "https://github.com/MaterializeInc/reqwest-middleware.git" }
 reqwest-retry = { git = "https://github.com/MaterializeInc/reqwest-middleware.git" }
 
+# Need to upstream a few PRs related to test builders.
+#
+# Note: All changes in our fork of tiberius should be pushed to the `mz_changes` branch.
+tiberius = { git = "https://github.com/MaterializeInc/tiberius", rev = "06423ac498a78ba2a1df8fd595208e6bdf0d5939" }
+
 # BEGIN LINT CONFIG
 # DO NOT EDIT. Automatically generated by bin/gen-lints.
 [workspace.lints.rust]

Cargo.toml

@@ -13,7 +13,7 @@ Business Source License 1.1
 
 Licensor:                  Materialize, Inc.
 
-Licensed Work:             Materialize Version 20250325
+Licensed Work:             Materialize Version 20250408
                            The Licensed Work is © 2025 Materialize, Inc.
 
 Additional Use Grant:      Within a single installation of the Licensed Work,
@@ -39,7 +39,7 @@ Additional Use Grant:      Within a single installation of the Licensed Work,
                            whose definitions are controlled by such third
                            parties.
 
-Change Date:               March 25, 2029
+Change Date:               April 08, 2029
 
 Change License:            Apache License, Version 2.0
 

LICENSE

@@ -31,9 +31,9 @@ included by calling a `*_dependencies()` macro.
 #
 # Note: In an ideal world the two rule sets would be combined into one.
 
-BAZEL_SKYLIB_VERSION = "1.6.1"
+BAZEL_SKYLIB_VERSION = "1.7.1"
 
-BAZEL_SKYLIB_INTEGRITY = "sha256-nziIakBUjG6WwQa3UvJCEw7hGqoGila6flb0UR8z5PI="
+BAZEL_SKYLIB_INTEGRITY = "sha256-vCg8381SalLDIBJ5zaS8KYZS76iYsQtNsIN9xRZSdW8="
 
 maybe(
     http_archive,
@@ -65,15 +65,22 @@ aspect_bazel_lib_dependencies()
 # Register bazel-lib toolchains
 aspect_bazel_lib_register_toolchains()
 
+# C Repositories
+#
+# Loads all of the C dependencies that we rely on.
+load("//misc/bazel/c_deps:repositories.bzl", "c_repositories")
+
+c_repositories()
+
 # `rules_cc`
 #
 # Rules for building C/C++ projects. These are slowly being upstreamed into the
 # Bazel source tree, but some projects (e.g. protobuf) still depend on this
 # rule set.
 
-RULES_CC_VERSION = "0.0.9"
+RULES_CC_VERSION = "0.1.1"
 
-RULES_CC_INTEGRITY = "sha256-IDeHW5pEVtzkp50RKorohbvEqtlo5lh9ym5k86CQDN8="
+RULES_CC_INTEGRITY = "sha256-cS13hosxUt1hjE1k+q3e/MWWX5D13m5t0dXdzQvoLUI="
 
 maybe(
     http_archive,
@@ -300,29 +307,22 @@ load("//misc/bazel/c_deps:extra_setup.bzl", "protoc_setup")
 
 protoc_setup()
 
-# C Repositories
-#
-# Loads all of the C dependencies that we rely on.
-load("//misc/bazel/c_deps:repositories.bzl", "c_repositories")
-
-c_repositories()
-
 # `rules_rust`
 #
 # Rules for building Rust crates, and several convienence macros for building all transitive
 # dependencies.
 
-RULES_RUST_VERSION = "0.54.2"
+RULES_RUST_VERSION = "0.59.3"
 
-RULES_RUST_INTEGRITY = "sha256-6hXbBNIbd6EvuBc/B5RMmAEhxLy5cs8EGuHpjqwjcz4="
+RULES_RUST_INTEGRITY = "sha256-pPPz9Yewxoqs6ZhcQAaI8AeFCy/S/ipQTEkTbZ3syz4="
 
 maybe(
     http_archive,
     name = "rules_rust",
     integrity = RULES_RUST_INTEGRITY,
-    strip_prefix = "rules_rust-v{0}".format(RULES_RUST_VERSION),
+    strip_prefix = "rules_rust-mz-{0}".format(RULES_RUST_VERSION),
     urls = [
-        "https://github.com/MaterializeInc/rules_rust/releases/download/v{0}/rules_rust-v{0}.tar.zst".format(RULES_RUST_VERSION),
+        "https://github.com/MaterializeInc/rules_rust/releases/download/mz-{0}/rules_rust-mz-{0}.tar.zst".format(RULES_RUST_VERSION),
     ],
 )
 
@@ -419,7 +419,17 @@ rust_toolchains(
 # Rules and Toolchains for running [`bindgen`](https://github.com/rust-lang/rust-bindgen)
 # a tool for generating Rust FFI bindings to C.
 
-load("@rules_rust//bindgen:repositories.bzl", "rust_bindgen_dependencies")
+maybe(
+    http_archive,
+    name = "rules_rust_bindgen",
+    integrity = RULES_RUST_INTEGRITY,
+    strip_prefix = "rules_rust-mz-{0}/extensions/bindgen".format(RULES_RUST_VERSION),
+    urls = [
+        "https://github.com/MaterializeInc/rules_rust/releases/download/mz-{0}/rules_rust-mz-{0}.tar.zst".format(RULES_RUST_VERSION),
+    ],
+)
+
+load("@rules_rust_bindgen//:repositories.bzl", "rust_bindgen_dependencies")
 
 rust_bindgen_dependencies()
 
@@ -461,6 +471,7 @@ crates_repository(
             # Note: The below targets are from the additive build file.
             additive_build_file = "@//misc/bazel/c_deps:rust-sys/BUILD.rocksdb.bazel",
             compile_data = [":out_dir"],
+            compile_data_glob_excludes = ["rocksdb/**"],
             gen_build_script = False,
             rustc_env = {
                 "OUT_DIR": "$(execpath :out_dir)",
@@ -558,20 +569,20 @@ crates_repository(
         "rustc-demangle": [crate.annotation(rustc_flags = ["-Copt-level=3"])],
         "timely": [crate.annotation(rustc_flags = ["-Copt-level=3"])],
         "differential-dataflow": [crate.annotation(rustc_flags = ["-Copt-level=3"])],
+        "insta": [crate.annotation(rustc_flags = ["-Copt-level=3"])],
+        "similar": [crate.annotation(rustc_flags = ["-Copt-level=3"])],
     },
     cargo_config = "//:.cargo/config.toml",
     cargo_lockfile = "//:Cargo.lock",
     generator_sha256s = {
-        "aarch64-apple-darwin": "8204746334a17823bd6a54ce2c3821b0bdca96576700d568e2ca2bd8224dc0ea",
-        "x86_64-apple-darwin": "2ee14b230d32c05415852b7a388b76e700c87c506459e5b31ced19d6c131b6d0",
-        "aarch64-unknown-linux-gnu": "3792feb084bd43b9a7a9cd75be86ee9910b46db59360d6b29c9cca2f8889a0aa",
-        "x86_64-unknown-linux-gnu": "2b9d07f34694f63f0cc704989ad6ec148ff8d126579832f4f4d88edea75875b2",
+        "aarch64-apple-darwin": "c38c9c0efc11fcf9c32b9e0f4f4849df7c823f207c7f5ba5f6ab1e0e2167693d",
+        "aarch64-unknown-linux-gnu": "5bdc9a10ec5f17f5140a81ce7cb0c0ce6e82d4d862d3ce3a301ea23f72f20630",
+        "x86_64-unknown-linux-gnu": "abcd8212d64ea4c0f5e856af663c05ebeb2800a02c251f6eb62061f4e8ca1735",
     },
     generator_urls = {
-        "aarch64-apple-darwin": "https://github.com/MaterializeInc/rules_rust/releases/download/v{0}/cargo-bazel-aarch64-apple-darwin".format(RULES_RUST_VERSION),
-        "x86_64-apple-darwin": "https://github.com/MaterializeInc/rules_rust/releases/download/v{0}/cargo-bazel-x86_64-apple-darwin".format(RULES_RUST_VERSION),
-        "aarch64-unknown-linux-gnu": "https://github.com/MaterializeInc/rules_rust/releases/download/v{0}/cargo-bazel-aarch64-unknown-linux-gnu".format(RULES_RUST_VERSION),
-        "x86_64-unknown-linux-gnu": "https://github.com/MaterializeInc/rules_rust/releases/download/v{0}/cargo-bazel-x86_64-unknown-linux-gnu".format(RULES_RUST_VERSION),
+        "aarch64-apple-darwin": "https://github.com/MaterializeInc/rules_rust/releases/download/mz-{0}/cargo-bazel-aarch64-apple-darwin".format(RULES_RUST_VERSION),
+        "aarch64-unknown-linux-gnu": "https://github.com/MaterializeInc/rules_rust/releases/download/mz-{0}/cargo-bazel-aarch64-unknown-linux-gnu".format(RULES_RUST_VERSION),
+        "x86_64-unknown-linux-gnu": "https://github.com/MaterializeInc/rules_rust/releases/download/mz-{0}/cargo-bazel-x86_64-unknown-linux-gnu".format(RULES_RUST_VERSION),
     },
     # When `isolated` is true, Bazel will create a new `$CARGO_HOME`, i.e. it
     # won't use `~/.cargo`, when re-pinning. This is nice but not totally
@@ -611,6 +622,8 @@ crates_repository(
         "//:src/dyncfg-launchdarkly/Cargo.toml",
         "//:src/dyncfgs/Cargo.toml",
         "//:src/environmentd/Cargo.toml",
+        "//:src/expr-derive/Cargo.toml",
+        "//:src/expr-derive-impl/Cargo.toml",
         "//:src/expr-parser/Cargo.toml",
         "//:src/expr-test-util/Cargo.toml",
         "//:src/expr/Cargo.toml",
@@ -630,6 +643,7 @@ crates_repository(
         "//:src/metrics/Cargo.toml",
         "//:src/mysql-util/Cargo.toml",
         "//:src/mz/Cargo.toml",
+        "//:src/mz-debug/Cargo.toml",
         "//:src/npm/Cargo.toml",
         "//:src/orchestrator-kubernetes/Cargo.toml",
         "//:src/orchestrator-process/Cargo.toml",
@@ -664,7 +678,6 @@ crates_repository(
         "//:src/s3-datagen/Cargo.toml",
         "//:src/secrets/Cargo.toml",
         "//:src/segment/Cargo.toml",
-        "//:src/self-managed-debug/Cargo.toml",
         "//:src/server-core/Cargo.toml",
         "//:src/service/Cargo.toml",
         "//:src/sql-lexer/Cargo.toml",

WORKSPACE

@@ -38,7 +38,7 @@ version=${1#v}
 
 sed -i.bak \
     "s/^version = .*/version = \"$version\"/" \
-    src/{clusterd,environmentd,materialized,persist-client,testdrive,catalog-debug,balancerd,orchestratord,self-managed-debug}/Cargo.toml
+    src/{clusterd,environmentd,materialized,persist-client,testdrive,catalog-debug,balancerd,orchestratord,mz-debug}/Cargo.toml
 
 if ! [[ "$version" = *-dev* ]]; then
     sed -i.bak \
@@ -58,7 +58,7 @@ else
     done
 fi
 
-rm -f src/{clusterd,environmentd,materialized,persist-client,testdrive,catalog-debug,balancerd,orchestratord,self-managed-debug}/Cargo.toml.bak LICENSE.bak
+rm -f src/{clusterd,environmentd,materialized,persist-client,testdrive,catalog-debug,balancerd,orchestratord,mz-debug}/Cargo.toml.bak LICENSE.bak
 
 cargo update --workspace
 

bin/bump-version

@@ -197,6 +197,7 @@ case "$cmd" in
             --env LAUNCHDARKLY_API_TOKEN
             --env LAUNCHDARKLY_SDK_KEY
             --env NIGHTLY_CANARY_APP_PASSWORD
+            --env MZ_CI_LICENSE_KEY
             --env MZ_CLI_APP_PASSWORD
             --env MZ_SOFT_ASSERTIONS
             --env NO_COLOR

bin/ci-builder

@@ -4,4 +4,4 @@
 # add to this list without consulting with @testing!
 
 pip==25.0.1
-setuptools==77.0.3
+setuptools==78.1.0