Fixed OOM during seeding and added progress logging during seed stage

Author: daniel-percona

README.md

@@ -117,44 +117,45 @@ Flags:
 
 Environment Variables (Overrides):
  [Connection]
-  PLGM_URI                     Connection URI
-  PLGM_USERNAME                Database User
-  PLGM_PASSWORD                Database Password (Recommended: Use Prompt)
-  PLGM_DIRECT_CONNECTION       Force direct connection (true/false)
-  PLGM_REPLICASET_NAME             Replica Set name
-  PLGM_READ_PREFERENCE         nearest
+  PLGM_URI                            Connection URI
+  PLGM_USERNAME                       Database User
+  PLGM_PASSWORD                       Database Password (Recommended: Use Prompt)
+  PLGM_DIRECT_CONNECTION              Force direct connection (true/false)
+  PLGM_REPLICASET_NAME                Replica Set name
+  PLGM_READ_PREFERENCE                nearest
 
  [Workload Core]
-  PLGM_DEFAULT_WORKLOAD        Use built-in workload (true/false)
-  PLGM_COLLECTIONS_PATH        Path to collection JSON
-  PLGM_QUERIES_PATH            Path to query JSON
-  PLGM_DURATION                Test duration (e.g. 60s, 5m)
-  PLGM_CONCURRENCY             Number of active workers
-  PLGM_DOCUMENTS_COUNT         Initial seed document count
-  PLGM_DROP_COLLECTIONS        Drop collections on start (true/false)
-  PLGM_SKIP_SEED               Do not seed initial data on start (true/false)
-  PLGM_DEBUG_MODE              Enable verbose logic logs (true/false)
-  PLGM_USE_TRANSACTIONS        Enable transactional workloads (true/false)
-  PLGM_MAX_TRANSACTION_OPS     Maximum number of operations to group into a single transaction block
+  PLGM_DEFAULT_WORKLOAD               Use built-in workload (true/false)
+  PLGM_COLLECTIONS_PATH               Path to collection JSON
+  PLGM_QUERIES_PATH                   Path to query JSON
+  PLGM_DURATION                       Test duration (e.g. 60s, 5m)
+  PLGM_CONCURRENCY                    Number of active workers
+  PLGM_DOCUMENTS_COUNT                Initial seed document count
+  PLGM_DROP_COLLECTIONS               Drop collections on start (true/false)
+  PLGM_SKIP_SEED                      Do not seed initial data on start (true/false)
+  PLGM_SEED_BATCH_SIZE                Number of documents to insert per batch during SEED phase
+  PLGM_DEBUG_MODE                     Enable verbose logic logs (true/false)
+  PLGM_USE_TRANSACTIONS               Enable transactional workloads (true/false)
+  PLGM_MAX_TRANSACTION_OPS            Maximum number of operations to group into a single transaction block
 
  [Operation Ratios] (Must sum to ~100)
-  PLGM_FIND_PERCENT            % of ops that are FIND
-  PLGM_UPDATE_PERCENT          % of ops that are UPDATE
-  PLGM_INSERT_PERCENT          % of ops that are INSERT
-  PLGM_DELETE_PERCENT          % of ops that are DELETE
-  PLGM_AGGREGATE_PERCENT       % of ops that are AGGREGATE
-  PLGM_TRANSACTION_PERCENT     % of ops that are TRANSACTIONAL
-  PLGM_BULK_INSERT_PERCENT     % of ops that are BULK INSERTS
+  PLGM_FIND_PERCENT                   % of ops that are FIND
+  PLGM_UPDATE_PERCENT                 % of ops that are UPDATE
+  PLGM_INSERT_PERCENT                 % of ops that are INSERT
+  PLGM_DELETE_PERCENT                 % of ops that are DELETE
+  PLGM_AGGREGATE_PERCENT              % of ops that are AGGREGATE
+  PLGM_TRANSACTION_PERCENT            % of ops that are TRANSACTIONAL
+  PLGM_BULK_INSERT_PERCENT            % of ops that are BULK INSERTS
 
  [Performance Optimization]
-  PLGM_FIND_BATCH_SIZE         Docs returned per cursor batch
-  PLGM_INSERT_BATCH_SIZE       Number of docs in batch bulk insert
-  PLGM_FIND_LIMIT              Max docs per Find query
-  PLGM_INSERT_CACHE_SIZE       Generator buffer size
-  PLGM_OP_TIMEOUT_MS           Soft timeout per DB op (ms)
-  PLGM_RETRY_ATTEMPTS          Retry attempts for failures
-  PLGM_RETRY_BACKOFF_MS        Wait time between retries (ms)
-  PLGM_STATUS_REFRESH_RATE_SEC Status report interval (sec)
+  PLGM_FIND_BATCH_SIZE                Docs returned per cursor batch
+  PLGM_INSERT_BATCH_SIZE              Number of docs in batch bulk insert
+  PLGM_FIND_LIMIT                     Max docs per Find query
+  PLGM_INSERT_CACHE_SIZE              Generator buffer size
+  PLGM_OP_TIMEOUT_MS                  Soft timeout per DB op (ms)
+  PLGM_RETRY_ATTEMPTS                 Retry attempts for failures
+  PLGM_RETRY_BACKOFF_MS               Wait time between retries (ms)
+  PLGM_STATUS_REFRESH_RATE_SEC        Status report interval (sec)
   GOMAXPROCS                          Go Runtime CPU limit
 ```
 
@@ -246,6 +247,7 @@ You can override any setting in `config.yaml` using environment variables. This
 | `documents_count` | `PLGM_DOCUMENTS_COUNT` | Number of documents to seed initially | `10000` |
 | `drop_collections` | `PLGM_DROP_COLLECTIONS` | Drop collections before starting (`true`/`false`) | `true` |
 | `skip_seed` | `PLGM_SKIP_SEED` | Do not seed initial data on start (`true`/`false`) | `true` |
+| `seed_batch_size` | `PLGM_SEED_BATCH_SIZE` | Number of documents to insert per batch during SEED phase | `1000` |
 | `debug_mode` | `PLGM_DEBUG_MODE` | Enable verbose debug logging (`true`/`false`) | `false` |
 | `use_transactions` | `PLGM_USE_TRANSACTIONS` | Enable Transactional Workloads (`true`/`false`) | `false` |
 | `max_transaction_ops` | `PLGM_MAX_TRANSACTION_OPS` | Maximum number of operations to group into a single transaction block | `5` |
@@ -482,6 +484,9 @@ These settings affect the efficiency of individual database operations and memor
     * *Default:* `10`
 * **`insert_batch_size`**: The number of documents to be inserted by bulk inserts.
     * *Default:* `10`   
+* **`seed_batch_size`**: The number of documents grouped into a single InsertMany call during the initial data seeding phase.
+    * *Tip:* Keeps memory usage stable when seeding millions of documents. A value of 1000 is recommended for performance.
+    * *Default:* `1000`
 * **`find_limit`**: The hard limit on documents returned for `find` operations.
     * *Default:* `5`
 * **`insert_cache_size`**: The buffer size for the document generator channel.

cmd/plgm/main.go

@@ -57,6 +57,7 @@ func main() {
 		fmt.Fprintf(os.Stderr, "  %-35s %s\n", "PLGM_DOCUMENTS_COUNT", "Initial seed document count")
 		fmt.Fprintf(os.Stderr, "  %-35s %s\n", "PLGM_DROP_COLLECTIONS", "Drop collections on start (true/false)")
 		fmt.Fprintf(os.Stderr, "  %-35s %s\n", "PLGM_SKIP_SEED", "Do not seed initial data on start (true/false)")
+		fmt.Fprintf(os.Stderr, "  %-35s %s\n", "PLGM_SEED_BATCH_SIZE", "Number of documents to insert per batch during SEED phase")
 		fmt.Fprintf(os.Stderr, "  %-35s %s\n", "PLGM_DEBUG_MODE", "Enable verbose logic logs (true/false)")
 		fmt.Fprintf(os.Stderr, "  %-35s %s\n", "PLGM_USE_TRANSACTIONS", "Enable transactional workloads (true/false)")
 		fmt.Fprintf(os.Stderr, "  %-35s %s\n", "PLGM_MAX_TRANSACTION_OPS", "Maximum number of operations to group into a single transaction block")

config.yaml

@@ -62,7 +62,7 @@ drop_collections: false
 skip_seed: true
 
 # Number of documents to insert during the initialization "Seed" phase.
-documents_count: 1000
+documents_count: 100000
 
 # Number of concurrent "Active Workers" to spawn.
 # Higher = more load, provided the client CPU/Network can handle it.
@@ -105,7 +105,8 @@ use_findone_for_limit_one: true # Optimization: use FindOne() if limit is 1.
 
 # --- Write Operations ---
 insert_cache_size: 1000         # Size of the internal buffer for generated documents.
-insert_batch_size: 10           # Number of documents per InsertMany call
+insert_batch_size: 10           # Number of documents per InsertMany call during WORKLOAD phase.
+seed_batch_size: 1000           # Number of documents per InsertMany call during SEED phase.
 
 # --- Monitoring ---
 status_refresh_rate_sec: 1      # How often (seconds) to print the "AVG Ops/Sec" log line.

internal/config/config.go

@@ -30,6 +30,7 @@ type AppConfig struct {
 	TransactionPercent int    `yaml:"transaction_percent"`
 	BulkInsertPercent  int    `yaml:"bulk_insert_percent"`
 	InsertBatchSize    int    `yaml:"insert_batch_size"`
+	SeedBatchSize      int    `yaml:"seed_batch_size"`
 	UseTransactions    bool   `yaml:"use_transactions"`
 	MaxTransactionOps  int    `yaml:"max_transaction_ops"`
 	DebugMode          bool   `yaml:"debug_mode"`
@@ -91,6 +92,10 @@ func applyDefaults(cfg *AppConfig) {
 	if cfg.InsertBatchSize <= 0 {
 		cfg.InsertBatchSize = 10
 	}
+	// Default to 1000 for fast seeding
+	if cfg.SeedBatchSize <= 0 {
+		cfg.SeedBatchSize = 1000
+	}
 	if cfg.FindLimit <= 0 {
 		cfg.FindLimit = 5
 	}
@@ -284,6 +289,11 @@ func applyEnvOverrides(cfg *AppConfig) map[string]bool {
 			cfg.InsertBatchSize = n
 		}
 	}
+	if v := os.Getenv("PLGM_SEED_BATCH_SIZE"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			cfg.SeedBatchSize = n
+		}
+	}
 
 	return overrides
 }

internal/logger/logger.go

@@ -54,3 +54,11 @@ func Header(msg string) {
 func Section(msg string) {
 	fmt.Printf("\n  %s %s\n", GreenString(">"), msg)
 }
+
+// Debug prints a formatted debug message with the [DEBUG] tag in Purple.
+func Debug(format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	// Indent multiline messages to align with the text
+	msg = strings.ReplaceAll(msg, "\n", "\n          ")
+	fmt.Printf("  %s %s%s\n", Purple+"[DEBUG]"+Reset, msg, Reset)
+}

internal/mongo/setup.go

@@ -16,20 +16,65 @@ import (
 
 func InsertRandomDocuments(ctx context.Context, db *mongo.Database, col config.CollectionDefinition, count int, cfg *config.AppConfig) error {
 	logger.Info("Seeding %d documents into '%s.%s'...", count, col.DatabaseName, col.Name)
-	docs := make([]interface{}, count)
+
+	// 1. Configure Batch Size
+	batchSize := cfg.SeedBatchSize
+	if batchSize <= 0 {
+		batchSize = 1000
+	}
+
+	// 2. Configure Progress Reporting
+	// Calculate 10% interval
+	modu := int(float32(count) * 0.1)
+	if modu < 1 {
+		modu = 1
+	}
+	nextLogTarget := modu
+
+	logger.Debug("Inserting documents in batches of %d", batchSize)
+	logger.Debug("Progress reporting every %d documents", modu)
+
+	targetDB := db.Client().Database(col.DatabaseName)
+	collection := targetDB.Collection(col.Name)
+
+	// Pre-allocate batch slice
+	batch := make([]interface{}, 0, batchSize)
+	totalInserted := 0
 
 	for i := 0; i < count; i++ {
-		docs[i] = workloads.GenerateDocument(col, cfg)
+		// Generate document
+		batch = append(batch, workloads.GenerateDocument(col, cfg))
+
+		// If batch is full, InsertMany
+		if len(batch) >= batchSize {
+			if _, err := collection.InsertMany(ctx, batch); err != nil {
+				return fmt.Errorf("insert batch into %s.%s: %w", col.DatabaseName, col.Name, err)
+			}
+
+			totalInserted += len(batch)
+			batch = batch[:0] // Reset batch, keep capacity
+
+			// Check if we crossed the 10% threshold
+			if totalInserted >= nextLogTarget {
+				logger.Info("-- Inserted %d documents...", totalInserted)
+				// Advance target to next 10% marker
+				for totalInserted >= nextLogTarget {
+					nextLogTarget += modu
+				}
+			}
+		}
 	}
 
-	if len(docs) > 0 {
-		// Use database from definition
-		targetDB := db.Client().Database(col.DatabaseName)
-		_, err := targetDB.Collection(col.Name).InsertMany(ctx, docs)
-		if err != nil {
-			return fmt.Errorf("insert documents into %s.%s: %w", col.DatabaseName, col.Name, err)
+	// Insert any remaining documents
+	if len(batch) > 0 {
+		if _, err := collection.InsertMany(ctx, batch); err != nil {
+			return fmt.Errorf("insert remaining documents into %s.%s: %w", col.DatabaseName, col.Name, err)
 		}
+		totalInserted += len(batch)
+		logger.Info("-- Inserted %d documents (Final)...", totalInserted)
 	}
+
+	logger.Debug("Document generation and seeding complete")
 	return nil
 }