Unify filesystem storage accounting

.github/workflows/ci.yaml

@@ -191,6 +191,58 @@ jobs:
           artifact-name: worker-executor-tests-${{ matrix.group.name }}-report
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
+  managed-xfs-tests:
+    name: managed-xfs-tests
+    runs-on: blacksmith-8vcpu-ubuntu-2404
+    needs:
+      - build-and-store
+      - merge-test-components
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 1
+      - uses: ./.github/actions/setup-rust
+        with:
+          use-cache: 'false'
+          install-cargo-binstall: 'true'
+      - uses: ./.github/actions/restore-binaries
+        with:
+          run-id: ${{ github.run_id }}
+          fail-on-cache-miss: 'true'
+      - uses: ./.github/actions/restore-test-components
+        with:
+          run-id: ${{ github.run_id }}
+      - name: Set up Lima
+        id: lima
+        env:
+          GH_TOKEN: ${{ github.token }}
+          LIMA_VERSION: v2.2.0
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qqy --no-install-recommends ovmf qemu-system-x86 qemu-utils
+          test -c /dev/kvm
+          sudo chown "$(whoami)" /dev/kvm
+          archive="lima-${LIMA_VERSION#v}-Linux-x86_64.tar.gz"
+          curl -fOSL "https://github.com/lima-vm/lima/releases/download/${LIMA_VERSION}/${archive}"
+          gh attestation verify --owner lima-vm "${archive}"
+          sudo tar -C /usr/local -xzf "${archive}"
+          rm "${archive}"
+          echo "version=${LIMA_VERSION}" >> "${GITHUB_OUTPUT}"
+      - name: Cache Lima images
+        uses: actions/cache@v5
+        with:
+          path: ~/.cache/lima
+          key: lima-${{ runner.os }}-${{ steps.lima.outputs.version }}
+      - name: Run managed XFS suite
+        env:
+          GOLEM_MANAGED_XFS_REUSE_TEST_BINARIES: 1
+          GOLEM_MANAGED_XFS_TARGET_DIR: ${{ github.workspace }}/target
+          GOLEM_MANAGED_XFS_CARGO_TEST_R: ${{ github.workspace }}/target/cargo-test-r
+        run: |
+          cp "$(command -v cargo-test-r)" "${GOLEM_MANAGED_XFS_CARGO_TEST_R}"
+          integration-tests/scripts/managed-filesystem/run-lima.sh
+
   build-cli-test-bins:
     env:
       CARGO_BUILD_JOBS: 20
@@ -1026,6 +1078,7 @@ jobs:
       - unit-tests-and-checks
       - golem-schema-guest
       - worker-tests
+      - managed-xfs-tests
       - it
       - it-cli
     if: "startsWith(github.ref, 'refs/tags/v')"

Cargo.toml

@@ -77,6 +77,7 @@ bitflags = "2.6.0"
 blake3 = { version = "1.8.2", features = ["rayon"] }
 bytes = "1.10.1"
 camino = "1.1.10"
+cap-fs-ext = "3.4.5" # keep in sync with wasmtime
 cap-std = "3.4.5" # keep in sync with wasmtime
 cap-time-ext = "3.4.5"
 cargo_metadata = "0.21.0"
@@ -155,6 +156,8 @@ kube-derive = "0.98.0"
 lazy_static = "1.5.0"
 leb128 = "0.2.5"
 lenient_bool = "0.1.1"
+libc = "0.2.186"
+linux-raw-sys = { version = "0.12.1", features = ["general", "ioctl"] }
 log = "0.4.26"
 mac_address = "1.1.8"
 mappable-rc = "0.1.1"
@@ -219,6 +222,7 @@ rsa = "0.9.7"
 
 rust_decimal = "1.39.0"
 rustc-hash = "2.1.1"
+rustix = { version = "1.1.4", features = ["fs"] }
 rustls = { version = "0.23.23", features = ["ring"] }
 rustls-pemfile = "2.2.0"
 sanitize-filename = "0.6.0"
@@ -309,7 +313,7 @@ wat = "1.248.0"
 wax = "0.6.0"
 webbrowser = "1.0.4"
 webpki-roots = { version = "0.26.7" }
-windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_System_Console"] }
+windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Globalization", "Win32_System_Console"] }
 which = "8.0.0"
 wit-bindgen = { git = "https://github.com/golemcloud/wit-bindgen", branch = "golem-outline-lift-v0.58.0", version = "=0.59.0" }
 wit-bindgen-rust = "0.58.0"

cli/golem-cli/command-output-schema/command-output.schema.json

@@ -4464,7 +4464,6 @@
         { "$ref": "#/definitions/PublicOplogEntrySuccessfulUpdate" },
         { "$ref": "#/definitions/PublicOplogEntryFailedUpdate" },
         { "$ref": "#/definitions/PublicOplogEntryGrowMemory" },
-        { "$ref": "#/definitions/PublicOplogEntryFilesystemStorageUsageUpdate" },
         { "$ref": "#/definitions/PublicOplogEntryCreateResource" },
         { "$ref": "#/definitions/PublicOplogEntryDropResource" },
         { "$ref": "#/definitions/PublicOplogEntryLog" },
@@ -4686,12 +4685,6 @@
       "properties": { "type": { "const": "GrowMemory" }, "timestamp": { "type": "string" }, "delta": { "type": "integer", "minimum": 0 } },
       "additionalProperties": false
     },
-    "PublicOplogEntryFilesystemStorageUsageUpdate": {
-      "type": "object",
-      "required": ["type", "timestamp", "delta"],
-      "properties": { "type": { "const": "FilesystemStorageUsageUpdate" }, "timestamp": { "type": "string" }, "delta": { "type": "integer" } },
-      "additionalProperties": false
-    },
     "PublicOplogEntryCreateResource": {
       "type": "object",
       "required": ["type", "timestamp", "id", "name", "owner"],

cli/golem-cli/src/model/agent/oplog.rs

@@ -378,17 +378,6 @@ impl TextOutput for PublicOplogEntry {
                     format_id(&format_binary_size(&params.delta)),
                 ));
             }
-            PublicOplogEntry::FilesystemStorageUsageUpdate(params) => {
-                logln(format_message_highlight("STORAGE USAGE UPDATE"));
-                logln(format!(
-                    "{pad}at:                {}",
-                    format_id(&params.timestamp)
-                ));
-                logln(format!(
-                    "{pad}delta:             {}",
-                    format_id(&params.delta.to_string()),
-                ));
-            }
             PublicOplogEntry::CreateResource(params) => {
                 logln(format_message_highlight("CREATE RESOURCE"));
                 logln(format!(

cli/golem-cli/src/model/cli_output/tests.rs

@@ -1978,10 +1978,6 @@ fn sample_public_oplog_entries() -> Vec<golem_common::model::oplog::PublicOplogE
             timestamp: timestamp(),
             delta: 64,
         }),
-        PublicOplogEntry::FilesystemStorageUsageUpdate(FilesystemStorageUsageUpdateParams {
-            timestamp: timestamp(),
-            delta: -5,
-        }),
         PublicOplogEntry::CreateResource(CreateResourceParams {
             timestamp: timestamp(),
             id: AgentResourceId(1),

cli/golem-cli/tests/app/account.rs

@@ -144,6 +144,7 @@ async fn account_usage_reports_sparse_allocated_memory(_tracing: &Tracing) {
     const BILLING_REPORT_TIMEOUT: Duration = Duration::from_secs(120);
 
     let mut ctx = TestContext::new();
+    ctx.add_env_var("GOLEM__RESOURCE_USAGE_METERING__MEMORY", "true");
     ctx.start_server().await;
 
     let app_name = "memory-billing";

cli/golem-cli/wit/deps/golem-1.x/golem-oplog.wit

@@ -421,11 +421,6 @@ interface oplog {
         delta: u64
     }
 
-    record filesystem-storage-usage-update-parameters {
-        timestamp: datetime,
-        delta: s64
-    }
-
     type agent-resource-id = u64;
 
     record create-resource-parameters {
@@ -581,8 +576,6 @@ interface oplog {
         exceeded-table-limit,
         exceeded-http-call-limit,
         exceeded-rpc-call-limit,
-        node-out-of-filesystem-storage,
-        agent-exceeded-filesystem-storage-limit,
         agent-terminated-by-quota(agent-terminated-by-quota-error),
         ephemeral-sleep-too-long(ephemeral-sleep-too-long),
         ephemeral-fuel-exhausted(ephemeral-fuel-exhausted),
@@ -812,8 +805,6 @@ interface oplog {
         failed-update(failed-update-parameters),
         /// Increased total linear memory size
         grow-memory(grow-memory-parameters),
-        /// Updated filesystem usage by a signed delta
-        filesystem-storage-usage-update(filesystem-storage-usage-update-parameters),
         /// Created a resource instance
         create-resource(raw-create-resource-parameters),
         /// Dropped a resource instance
@@ -934,8 +925,6 @@ interface oplog {
         failed-update(failed-update-parameters),
         /// Increased total linear memory size
         grow-memory(grow-memory-parameters),
-        /// Updated filesystem usage by a signed delta
-        filesystem-storage-usage-update(filesystem-storage-usage-update-parameters),
         /// Created a resource instance
         create-resource(create-resource-parameters),
         /// Dropped a resource instance

cli/golem/src/command_handler.rs

@@ -21,6 +21,7 @@ use golem_cli::error::NonSuccessfulExit;
 use golem_cli::fs;
 use golem_cli::log::{LogColorize, log_warn_action};
 use golem_cli::model::app::ResolvedLocalServer;
+use golem_worker_executor::services::golem_config::ResourceUsageMeteringConfig;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use tracing::debug;
@@ -81,6 +82,7 @@ impl CommandHandlerHooks for ServerCommandHandler {
             ports_file: args.ports_file.clone(),
             data_dir: data_dir.clone(),
             agent_filesystem_root: args.agent_filesystem_root.clone(),
+            resource_usage_metering: resource_usage_metering_from_env()?,
         })
         .await
         .map_err(|err| map_local_server_startup_error(err, &data_dir))?;
@@ -117,7 +119,35 @@ fn launch_args_from_run_args_and_manifest(
     args: &RunArgs,
     ctx: &Context,
 ) -> anyhow::Result<LaunchArgs> {
-    launch_args_from_run_args_and_local_server(args, ctx.manifest_local_server())
+    launch_args_from_run_args_and_local_server(
+        args,
+        ctx.manifest_local_server(),
+        resource_usage_metering_from_env()?,
+    )
+}
+
+fn resource_usage_metering_from_env() -> anyhow::Result<ResourceUsageMeteringConfig> {
+    Ok(ResourceUsageMeteringConfig {
+        compute: metering_dimension_from_env("GOLEM__RESOURCE_USAGE_METERING__COMPUTE")?,
+        memory: metering_dimension_from_env("GOLEM__RESOURCE_USAGE_METERING__MEMORY")?,
+        filesystem: metering_dimension_from_env("GOLEM__RESOURCE_USAGE_METERING__FILESYSTEM")?,
+    })
+}
+
+fn metering_dimension_from_env(name: &str) -> anyhow::Result<bool> {
+    match std::env::var(name) {
+        Ok(value) => parse_metering_dimension(name, &value),
+        Err(std::env::VarError::NotPresent) => Ok(false),
+        Err(std::env::VarError::NotUnicode(_)) => {
+            bail!("Failed to parse {name}: non-Unicode value")
+        }
+    }
+}
+
+fn parse_metering_dimension(name: &str, value: &str) -> anyhow::Result<bool> {
+    value
+        .parse()
+        .with_context(|| format!("Failed to parse {name}: {value}"))
 }
 
 fn data_dir_from_local_server(
@@ -132,6 +162,7 @@ fn data_dir_from_local_server(
 fn launch_args_from_run_args_and_local_server(
     args: &RunArgs,
     local_server: Option<&ResolvedLocalServer>,
+    resource_usage_metering: ResourceUsageMeteringConfig,
 ) -> anyhow::Result<LaunchArgs> {
     Ok(LaunchArgs {
         router_addr: args
@@ -163,6 +194,7 @@ fn launch_args_from_run_args_and_local_server(
             .agent_filesystem_root
             .clone()
             .or_else(|| local_server.and_then(|manifest| manifest.agent_filesystem_root.clone())),
+        resource_usage_metering,
     })
 }
 
@@ -218,6 +250,13 @@ mod tests {
         ResolvedLocalServer::from_raw_with_base_dir(&value, Path::new("/tmp/test-app"))
     }
 
+    #[test]
+    fn metering_dimension_values_are_validated() {
+        assert!(parse_metering_dimension("METERING", "true").unwrap());
+        assert!(!parse_metering_dimension("METERING", "false").unwrap());
+        assert!(parse_metering_dimension("METERING", "invalid").is_err());
+    }
+
     #[test]
     fn manifest_local_server_values_are_used_when_cli_args_are_absent() {
         let manifest = local_server(LocalServer {
@@ -230,8 +269,12 @@ mod tests {
             agent_filesystem_root: Some(PathBuf::from("/tmp/test-app/.golem/agents")),
         });
 
-        let args = launch_args_from_run_args_and_local_server(&RunArgs::default(), Some(&manifest))
-            .unwrap();
+        let args = launch_args_from_run_args_and_local_server(
+            &RunArgs::default(),
+            Some(&manifest),
+            ResourceUsageMeteringConfig::default(),
+        )
+        .unwrap();
 
         assert_eq!(args.router_addr, "127.0.0.1");
         assert_eq!(args.router_port, 9882);
@@ -270,7 +313,12 @@ mod tests {
             agent_filesystem_root: Some(PathBuf::from("cli-agents")),
         };
 
-        let args = launch_args_from_run_args_and_local_server(&run_args, Some(&manifest)).unwrap();
+        let args = launch_args_from_run_args_and_local_server(
+            &run_args,
+            Some(&manifest),
+            ResourceUsageMeteringConfig::default(),
+        )
+        .unwrap();
 
         assert_eq!(args.router_addr, "0.0.0.0");
         assert_eq!(args.router_port, 10000);

cli/golem/src/launch.rs

@@ -41,8 +41,8 @@ use golem_worker_executor::services::golem_config::{
     AgentTypesServiceConfig, AgentWebhooksServiceConfig, EnvironmentStateServiceConfig,
     FilesystemStorageConfig, GolemConfig as WorkerExecutorConfig, IndexedStorageConfig,
     IndexedStorageKVStoreMultiSqliteConfig, KeyValueStorageConfig,
-    KeyValueStorageMultiSqliteConfig, ResourceLimitsConfig, SchedulerStorageConfig,
-    WorkerServiceGrpcConfig,
+    KeyValueStorageMultiSqliteConfig, ResourceLimitsConfig, ResourceUsageMeteringConfig,
+    SchedulerStorageConfig, WorkerServiceGrpcConfig,
 };
 use golem_worker_service::WorkerService;
 use golem_worker_service::config::{
@@ -69,6 +69,7 @@ pub struct LaunchArgs {
     pub ports_file: Option<PathBuf>,
     pub data_dir: PathBuf,
     pub agent_filesystem_root: Option<PathBuf>,
+    pub resource_usage_metering: ResourceUsageMeteringConfig,
 }
 
 impl LaunchArgs {
@@ -398,6 +399,7 @@ fn worker_executor_config(
             ..Default::default()
         },
         resource_limits: ResourceLimitsConfig::default(),
+        resource_usage_metering: args.resource_usage_metering,
         agent_types_service: AgentTypesServiceConfig::Grpc(
             golem_worker_executor::services::golem_config::AgentTypesServiceGrpcConfig {
                 ..Default::default()

docs/src/content/next/how-to-guides/common/golem-debug-agent-history.mdx

@@ -58,7 +58,6 @@ Each text entry is printed with its index (e.g. `#00042:`) followed by a labeled
 | `INTERRUPTED` / `EXITED` | Agent interrupted or exited |
 | `NOP` | No-operation marker |
 | `JUMP` | Oplog jump — shows from/to indices |
-| `STORAGE USAGE UPDATE` | Filesystem storage usage change |
 
 ### Examples
 

docs/src/content/next/operate/_meta.js

@@ -1,5 +1,6 @@
 export default {
   persistence: "Persistence",
+  resource_metering: "Resource Metering",
   metrics: "Metrics",
   logs: "Logs",
   durable_streams: "Durable Streams",