Add Filesystem datastore support

.github/workflows/galexie.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-24.04
     strategy:
       matrix:
-        storage_type: [ GCS, S3 ]
+        storage_type: [ GCS, S3, Filesystem ]
     env:
       CAPTIVE_CORE_DEBIAN_PKG_VERSION: 25.0.0-2911.e9748b05a.noble
       GALEXIE_INTEGRATION_TESTS_ENABLED: "true"

config/config.example.toml

@@ -7,7 +7,7 @@ admin_port = 6061
 
 # Datastore Configuration
 [datastore_config]
-# Specifies the type of datastore. Currently, Google Cloud Storage (GCS) and s3-compatible storage (S3) are supported.
+# Specifies the type of datastore. Currently, Google Cloud Storage (GCS), s3-compatible storage (S3), and Filesystem are supported.
 type = "GCS"
 
 [datastore_config.params]
@@ -25,6 +25,13 @@ destination_bucket_path = "your-bucket-name/<optional_subpath1>/<optional_subpat
 # The below example is for Cloudflare R2, but you can replace it with your S3-compatible storage endpoint.
 #endpoint_url = "https://00000000000000000000000000000000.cloudflarestorage.com"
 
+# params required for Filesystem storage
+# Note: Filesystem storage is not recommended for production use. It is
+# intended for development and testing purposes only.
+# This implementation does not support storing metadata.
+# The local filesystem path for storing data.
+#destination_path = "/path/to/local/storage"
+
 [datastore_config.schema]
 # Configuration for data organization
 ledgers_per_file = 1      # Number of ledgers stored in each file.

go.mod

@@ -35,7 +35,7 @@ require (
 	github.com/docker/docker v28.0.0+incompatible
 	github.com/docker/go-connections v0.5.0
 	github.com/fsouza/fake-gcs-server v1.49.2
-	github.com/stellar/go-stellar-sdk v0.0.0-20251210093830-27970a00c2d7
+	github.com/stellar/go-stellar-sdk v0.0.0-20260108045717-08ad73d9ed0e
 )
 
 require (

go.sum

@@ -380,8 +380,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.17.0 h1:I5txKw7MJasPL/BrfkbA0Jyo/oELqVmux4pR/UxOMfI=
 github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0+yVI=
-github.com/stellar/go-stellar-sdk v0.0.0-20251210093830-27970a00c2d7 h1:vzKfpTTcwQNojsDG+qClvt1oh7P1mLxdmhAnFlvP8No=
-github.com/stellar/go-stellar-sdk v0.0.0-20251210093830-27970a00c2d7/go.mod h1:fZPcxQZw1I0zZ+X76uFcVPqmQCaYbWc87lDFW/kQJaY=
+github.com/stellar/go-stellar-sdk v0.0.0-20260108045717-08ad73d9ed0e h1:05/g4c3cuD1NOvJHMkIHUlKAoKtsQ1WxzlbyozK3hfY=
+github.com/stellar/go-stellar-sdk v0.0.0-20260108045717-08ad73d9ed0e/go.mod h1:4osSHRDfb/D7zE2iNjmPdg2Gmbx0M/JLhTM4YcwPW5s=
 github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2 h1:OzCVd0SV5qE3ZcDeSFCmOWLZfEWZ3Oe8KtmSOYKEVWE=
 github.com/stellar/go-xdr v0.0.0-20231122183749-b53fb00bcac2/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

test/integration_test.go

@@ -74,6 +74,18 @@
 	suite.Run(t, galexieS3Suite)
 }
 
+// TestGalexieFilesystemTestSuite runs tests with Filesystem backend
+func TestGalexieFilesystemTestSuite(t *testing.T) {
+	if os.Getenv("GALEXIE_INTEGRATION_TESTS_ENABLED") != "true" {
+		t.Skip("skipping integration test: GALEXIE_INTEGRATION_TESTS_ENABLED not true")
+	}
+
+	galexieFilesystemSuite := &GalexieTestSuite{
+		storageType: "Filesystem",
+	}
+	suite.Run(t, galexieFilesystemSuite)
+}
+
 type GalexieTestSuite struct {
 	suite.Suite
 	tempConfigFile        string
@@ -86,7 +98,11 @@
 	gcsServer             *fakestorage.Server
 	finishedSetup         bool
 	config                galexie.Config
-	storageType           string // "GCS" or "S3"
+	storageType           string // "GCS", "S3", or "Filesystem"
+}
+
+func (s *GalexieTestSuite) filesystemDataPath() string {
+	return filepath.Join(s.testTempDir, "filesystem-data")
 }
 
 func (s *GalexieTestSuite) TestScanAndFill() {
@@ -376,6 +392,12 @@
 		filepath.Join(s.testTempDir, "captive-core"))
 	galexieConfigTemplate.Set("datastore_config.type", s.storageType)
 
+	// Set storage-specific params
+	if s.storageType == "Filesystem" {
+		galexieConfigTemplate.Set("datastore_config.params.destination_path",
+			s.filesystemDataPath())
+	}
+
 	// Apply any per-config overrides
 	if mutate != nil {
 		mutate(galexieConfigTemplate)
@@ -479,6 +501,8 @@
 		s.setupGCS(t)
 	} else if s.storageType == "S3" {
 		s.setupS3(t)
+	} else if s.storageType == "Filesystem" {
+		s.setupFilesystem(t)
 	}
 }
 
@@ -490,6 +514,11 @@
 		s.T().Logf("Stopping the localstack container %v", s.localStackContainerID)
 		s.stopAndLogContainer(s.localStackContainerID, "localstack")
 		s.localStackContainerID = ""
+	} else if s.storageType == "Filesystem" {
+		// Clean up filesystem data between tests
+		if err := os.RemoveAll(s.filesystemDataPath()); err != nil {
+			s.T().Logf("Failed to clean up filesystem data path: %v", err)
+		}
 	}
 }
 
@@ -779,6 +808,14 @@
 	t.Setenv("AWS_SECRET_ACCESS_KEY", "ACCESS_KEY")
 }
 
+func (s *GalexieTestSuite) setupFilesystem(t *testing.T) {
+	path := s.filesystemDataPath()
+	// Clean any existing data from previous tests
+	require.NoError(t, os.RemoveAll(path))
+	require.NoError(t, os.MkdirAll(path, 0755))
+	t.Logf("Filesystem datastore path: %s", path)
+}
+
 func (s *GalexieTestSuite) TearDownSuite() {
 	t := s.T()