Make the shard manager's persistence compare-and-swap based across local and distributed backends

.agents/skills/db-migration-scripts/SKILL.md

@@ -30,7 +30,7 @@ root's query layer is shared:
 | Migration root | Query layer | Convention |
 |---|---|---|
 | `golem-registry-service/db/migration/` | one impl expanded per backend by `#[trait_gen(PostgresPool -> PostgresPool, SqlitePool)]` | converge by default |
-| `golem-shard-manager/db/migration/` | same `trait_gen` pattern (`src/quota/quota_repo.rs`, `src/sharding/persistence.rs`) | converge by default |
+| `golem-shard-manager/db/migration/` | same `trait_gen` pattern (`src/quota/quota_repo.rs`, `src/sharding/persistence/db.rs`) | converge by default |
 | `golem-worker-executor/db/migration/{indexed,keyvalue,scheduler}/` | separate `postgres.rs` / `sqlite.rs` under `src/storage/<subsystem>/` (alongside `multi_sqlite.rs`, `redis.rs`, `memory.rs`) | diverge where the engine calls for it |
 
 ### Shared-query roots
@@ -109,8 +109,8 @@ Examples of the relevant integration coverage:
 
 - Registry repository tests initialize PostgreSQL and SQLite from
   `golem-registry-service/db/migration/`.
-- `golem-shard-manager`'s `persistence` tests run each test through both `sqlite` and `postgres`
-  matrix dimensions.
+- `golem-shard-manager`'s `persistence` tests run each test through the `sqlite`, `postgres` and
+  `etcd` matrix dimensions.
 - Worker-executor indexed/key-value storage tests include SQLite and PostgreSQL dimensions; select
   the storage test that exercises the changed schema.
 

.github/actions/setup-rust/action.yml

@@ -39,6 +39,13 @@ runs:
         for target in ${{ inputs.rust-targets }}; do
           rustup target add "$target"
         done
+    - name: Install protoc
+      # `etcd-client`'s build script compiles its protos with tonic-prost-build, which shells out
+      # to `protoc`. The workspace's own protos are compiled with protox (pure Rust) and need no
+      # binary, so this is the only thing that requires it.
+      uses: arduino/setup-protoc@v3
+      with:
+        repo-token: ${{ github.token }}
     - name: Setup Rust cache
       if: inputs.use-cache == 'true'
       uses: Swatinem/rust-cache@v2

AGENTS.md

@@ -25,6 +25,9 @@ explicitly revised, **do not perform backward-compatibility work**.
 - **cargo-make**: Latest version (`cargo install --force cargo-make`)
 - **cargo-test-r**: Match the version CI installs, pinned in
   `.github/actions/restore-binaries/action.yml` (`cargo install --force --locked cargo-test-r@<VERSION>`)
+- **protoc**: Required to build `golem-shard-manager`, and therefore the `golem` binary. The
+  `etcd-client` crate's build script compiles its protos with `tonic-prost-build`, which shells out
+  to the protobuf compiler; the workspace's own protos use `protox` and need no binary
 - **redis-server**: Required by worker-executor, service-integration, and CLI integration tests that
   use spawned test dependencies; not required by every unit test
 - **docker**: Required by tests that use containerized dependencies

Cargo.toml

@@ -107,6 +107,7 @@ dir-diff = "0.3.3"
 dirs = "6.0.0"
 dotenvy = "0.15.7"
 drop-stream = "0.3.2"
+etcd-client = "0.19.0"
 evicting_cache_map = "0.4.0"
 expectrl = "0.8.0"
 fancy-regex = "0.14.0"

Cross.toml

@@ -1,7 +1,9 @@
 [target.aarch64-unknown-linux-gnu]
 pre-build = [
     "dpkg --add-architecture $CROSS_DEB_ARCH",
-    "apt-get update && apt-get --assume-yes install pkg-config libssl-dev:$CROSS_DEB_ARCH libssl-dev unzip curl",
+    # protobuf-compiler is a build-time tool for the host, not a target library: etcd-client's
+    # build script shells out to protoc. The runner's protoc is outside this container.
+    "apt-get update && apt-get --assume-yes install pkg-config libssl-dev:$CROSS_DEB_ARCH libssl-dev unzip curl protobuf-compiler",
 ]
 env.passthrough = [
     "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu",

cli/golem/src/launch.rs

@@ -314,7 +314,7 @@ fn shard_manager_config(
             port: 0,
             ..Default::default()
         },
-        db: DbConfig::Sqlite(DbSqliteConfig {
+        persistence: golem_shard_manager::config::PersistenceConfig::Sqlite(DbSqliteConfig {
             database: args
                 .data_dir
                 .join("shard_manager.db")

docker-examples/published-postgres/compose.yaml

@@ -92,14 +92,14 @@ services:
       GOLEM__REGISTRY_SERVICE__HOST: golem-registry-service
       GOLEM__REGISTRY_SERVICE__PORT: ${REGISTRY_SERVICE_GRPC_PORT}
 
-      GOLEM__DB__TYPE: Postgres
-      GOLEM__DB__CONFIG__DATABASE: golem_db
-      GOLEM__DB__CONFIG__SCHEMA: golem_shard_manager
-      GOLEM__DB__CONFIG__MAX_CONNECTIONS: 10
-      GOLEM__DB__CONFIG__HOST: postgres
-      GOLEM__DB__CONFIG__PORT: 5432
-      GOLEM__DB__CONFIG__USERNAME: golem_user
-      GOLEM__DB__CONFIG__PASSWORD: golem_password
+      GOLEM__PERSISTENCE__TYPE: Postgres
+      GOLEM__PERSISTENCE__CONFIG__DATABASE: golem_db
+      GOLEM__PERSISTENCE__CONFIG__SCHEMA: golem_shard_manager
+      GOLEM__PERSISTENCE__CONFIG__MAX_CONNECTIONS: 10
+      GOLEM__PERSISTENCE__CONFIG__HOST: postgres
+      GOLEM__PERSISTENCE__CONFIG__PORT: 5432
+      GOLEM__PERSISTENCE__CONFIG__USERNAME: golem_user
+      GOLEM__PERSISTENCE__CONFIG__PASSWORD: golem_password
     depends_on:
       postgres:
         condition: service_healthy

golem-shard-manager/Cargo.toml

@@ -32,6 +32,7 @@ chrono = { workspace = true }
 async-trait = { workspace = true }
 conditional-trait-gen = { workspace = true }
 desert_rust = { workspace = true }
+etcd-client = { workspace = true }
 futures = { workspace = true }
 http = { workspace = true }
 humantime-serde = { workspace = true }

golem-shard-manager/config/shard-manager.sample.env

@@ -5,15 +5,15 @@ GOLEM__NUMBER_OF_SHARDS=1024
 GOLEM__REBALANCE_THRESHOLD=0.1
 GOLEM__RUNTIME_METRICS_SAMPLING_INTERVAL="5s"
 GOLEM__SHARD_LEASE_DURATION="1m"
-GOLEM__DB__TYPE="Sqlite"
-GOLEM__DB__CONFIG__DATABASE="golem_shard_manager.db"
-GOLEM__DB__CONFIG__FOREIGN_KEYS=false
-GOLEM__DB__CONFIG__MAX_CONNECTIONS=10
 GOLEM__GRPC__PORT=9092
 GOLEM__GRPC__TLS__TYPE="Disabled"
 GOLEM__HEALTH_CHECK__DELAY="10s"
 GOLEM__HEALTH_CHECK__SILENT=false
 GOLEM__HEALTH_CHECK__MODE__TYPE="Grpc"
+GOLEM__PERSISTENCE__TYPE="Sqlite"
+GOLEM__PERSISTENCE__CONFIG__DATABASE="golem_shard_manager.db"
+GOLEM__PERSISTENCE__CONFIG__FOREIGN_KEYS=false
+GOLEM__PERSISTENCE__CONFIG__MAX_CONNECTIONS=10
 GOLEM__QUOTA__DEFINITION_STALENESS_TTL="5m"
 GOLEM__QUOTA__LEASE_DURATION="1m"
 GOLEM__QUOTA__MIN_EXECUTORS=2
@@ -100,16 +100,111 @@ GOLEM__NUMBER_OF_SHARDS=1024
 GOLEM__REBALANCE_THRESHOLD=0.1
 GOLEM__RUNTIME_METRICS_SAMPLING_INTERVAL="5s"
 GOLEM__SHARD_LEASE_DURATION="1m"
-GOLEM__DB__TYPE="Sqlite"
-GOLEM__DB__CONFIG__DATABASE="golem_shard_manager.db"
-GOLEM__DB__CONFIG__FOREIGN_KEYS=false
-GOLEM__DB__CONFIG__MAX_CONNECTIONS=10
 GOLEM__GRPC__PORT=9092
 GOLEM__GRPC__TLS__TYPE="Disabled"
 GOLEM__HEALTH_CHECK__DELAY="1s"
 GOLEM__HEALTH_CHECK__SILENT=false
 GOLEM__HEALTH_CHECK__MODE__TYPE="K8s"
 GOLEM__HEALTH_CHECK__MODE__CONFIG__NAMESPACE="namespace"
+GOLEM__PERSISTENCE__TYPE="Sqlite"
+GOLEM__PERSISTENCE__CONFIG__DATABASE="golem_shard_manager.db"
+GOLEM__PERSISTENCE__CONFIG__FOREIGN_KEYS=false
+GOLEM__PERSISTENCE__CONFIG__MAX_CONNECTIONS=10
+GOLEM__QUOTA__DEFINITION_STALENESS_TTL="5m"
+GOLEM__QUOTA__LEASE_DURATION="1m"
+GOLEM__QUOTA__MIN_EXECUTORS=2
+GOLEM__REGISTRY_SERVICE__CONNECT_TIMEOUT="10s"
+GOLEM__REGISTRY_SERVICE__HOST="localhost"
+GOLEM__REGISTRY_SERVICE__MAX_MESSAGE_SIZE=52428800
+GOLEM__REGISTRY_SERVICE__PORT=8080
+#GOLEM__REGISTRY_SERVICE__REQUEST_TIMEOUT=
+GOLEM__REGISTRY_SERVICE__INVALIDATION_EVENT_SUBSCRIBER__INITIAL_BACKOFF="100ms"
+GOLEM__REGISTRY_SERVICE__INVALIDATION_EVENT_SUBSCRIBER__MAX_BACKOFF="30s"
+GOLEM__REGISTRY_SERVICE__RETRIES_ON_UNAVAILABLE__MAX_ATTEMPTS=5
+GOLEM__REGISTRY_SERVICE__RETRIES_ON_UNAVAILABLE__MAX_DELAY="2s"
+GOLEM__REGISTRY_SERVICE__RETRIES_ON_UNAVAILABLE__MAX_JITTER_FACTOR=0.15
+GOLEM__REGISTRY_SERVICE__RETRIES_ON_UNAVAILABLE__MIN_DELAY="100ms"
+GOLEM__REGISTRY_SERVICE__RETRIES_ON_UNAVAILABLE__MULTIPLIER=2.0
+GOLEM__REGISTRY_SERVICE__TLS__TYPE="Disabled"
+GOLEM__RESOURCE_DEFINITION_FETCHER__CACHE_EVICTION_PERIOD="1m"
+GOLEM__RESOURCE_DEFINITION_FETCHER__CACHE_MAX_CAPACITY=1024
+GOLEM__RESOURCE_DEFINITION_FETCHER__CACHE_TTL="5m"
+GOLEM__TRACING__CONSOLE=false
+GOLEM__TRACING__DTOR_FRIENDLY=false
+#GOLEM__TRACING__FILE_DIR=
+GOLEM__TRACING__FILE_NAME="shard-manager.log"
+GOLEM__TRACING__FILE_TRUNCATE=true
+GOLEM__TRACING__FILE__ANSI=false
+GOLEM__TRACING__FILE__COMPACT=false
+GOLEM__TRACING__FILE__ENABLED=false
+GOLEM__TRACING__FILE__JSON=true
+GOLEM__TRACING__FILE__JSON_FLATTEN=true
+GOLEM__TRACING__FILE__JSON_FLATTEN_SPAN=true
+GOLEM__TRACING__FILE__JSON_SOURCE_LOCATION=false
+GOLEM__TRACING__FILE__PRETTY=false
+GOLEM__TRACING__FILE__SPAN_EVENTS_ACTIVE=false
+GOLEM__TRACING__FILE__SPAN_EVENTS_FULL=false
+GOLEM__TRACING__FILE__WITHOUT_TIME=false
+GOLEM__TRACING__OTLP__ENABLED=false
+GOLEM__TRACING__OTLP__HOST="localhost"
+GOLEM__TRACING__OTLP__PORT=4318
+GOLEM__TRACING__OTLP__SERVICE_NAME="golem"
+GOLEM__TRACING__STDERR__ANSI=false
+GOLEM__TRACING__STDERR__COMPACT=false
+GOLEM__TRACING__STDERR__ENABLED=false
+GOLEM__TRACING__STDERR__JSON=false
+GOLEM__TRACING__STDERR__JSON_FLATTEN=false
+GOLEM__TRACING__STDERR__JSON_FLATTEN_SPAN=false
+GOLEM__TRACING__STDERR__JSON_SOURCE_LOCATION=false
+GOLEM__TRACING__STDERR__PRETTY=false
+GOLEM__TRACING__STDERR__SPAN_EVENTS_ACTIVE=false
+GOLEM__TRACING__STDERR__SPAN_EVENTS_FULL=false
+GOLEM__TRACING__STDERR__WITHOUT_TIME=false
+GOLEM__TRACING__STDOUT__ANSI=true
+GOLEM__TRACING__STDOUT__COMPACT=false
+GOLEM__TRACING__STDOUT__ENABLED=true
+GOLEM__TRACING__STDOUT__JSON=false
+GOLEM__TRACING__STDOUT__JSON_FLATTEN=true
+GOLEM__TRACING__STDOUT__JSON_FLATTEN_SPAN=true
+GOLEM__TRACING__STDOUT__JSON_SOURCE_LOCATION=false
+GOLEM__TRACING__STDOUT__PRETTY=false
+GOLEM__TRACING__STDOUT__SPAN_EVENTS_ACTIVE=false
+GOLEM__TRACING__STDOUT__SPAN_EVENTS_FULL=false
+GOLEM__TRACING__STDOUT__WITHOUT_TIME=false
+GOLEM__WORKER_EXECUTORS__ASSIGN_SHARDS_TIMEOUT="5s"
+GOLEM__WORKER_EXECUTORS__CONNECT_TIMEOUT="10s"
+GOLEM__WORKER_EXECUTORS__HEALTH_CHECK_TIMEOUT="2s"
+GOLEM__WORKER_EXECUTORS__MAX_MESSAGE_SIZE=33554432
+#GOLEM__WORKER_EXECUTORS__REQUEST_TIMEOUT=
+GOLEM__WORKER_EXECUTORS__REVOKE_SHARDS_TIMEOUT="5s"
+GOLEM__WORKER_EXECUTORS__RETRIES__MAX_ATTEMPTS=5
+GOLEM__WORKER_EXECUTORS__RETRIES__MAX_DELAY="2s"
+GOLEM__WORKER_EXECUTORS__RETRIES__MAX_JITTER_FACTOR=0.15
+GOLEM__WORKER_EXECUTORS__RETRIES__MIN_DELAY="100ms"
+GOLEM__WORKER_EXECUTORS__RETRIES__MULTIPLIER=2.0
+GOLEM__WORKER_EXECUTORS__RETRIES_ON_UNAVAILABLE__MAX_ATTEMPTS=5
+GOLEM__WORKER_EXECUTORS__RETRIES_ON_UNAVAILABLE__MAX_DELAY="2s"
+GOLEM__WORKER_EXECUTORS__RETRIES_ON_UNAVAILABLE__MAX_JITTER_FACTOR=0.15
+GOLEM__WORKER_EXECUTORS__RETRIES_ON_UNAVAILABLE__MIN_DELAY="100ms"
+GOLEM__WORKER_EXECUTORS__RETRIES_ON_UNAVAILABLE__MULTIPLIER=2.0
+GOLEM__WORKER_EXECUTORS__TLS__TYPE="Disabled"
+
+### Generated from example config: with etcd persistence (distributed mode)
+
+GOLEM__HTTP_PORT=8081
+GOLEM__NUMBER_OF_SHARDS=1024
+GOLEM__REBALANCE_THRESHOLD=0.1
+GOLEM__RUNTIME_METRICS_SAMPLING_INTERVAL="5s"
+GOLEM__SHARD_LEASE_DURATION="1m"
+GOLEM__GRPC__PORT=9092
+GOLEM__GRPC__TLS__TYPE="Disabled"
+GOLEM__HEALTH_CHECK__DELAY="10s"
+GOLEM__HEALTH_CHECK__SILENT=false
+GOLEM__HEALTH_CHECK__MODE__TYPE="Grpc"
+GOLEM__PERSISTENCE__TYPE="Etcd"
+GOLEM__PERSISTENCE__CONFIG__CONNECT_TIMEOUT="10s"
+GOLEM__PERSISTENCE__CONFIG__ENDPOINTS=["http://localhost:2379"]
+GOLEM__PERSISTENCE__CONFIG__REQUEST_TIMEOUT="5s"
 GOLEM__QUOTA__DEFINITION_STALENESS_TTL="5m"
 GOLEM__QUOTA__LEASE_DURATION="1m"
 GOLEM__QUOTA__MIN_EXECUTORS=2