Email simulator script

This commit adds two scripts intended to simulate currently ongoing
discussions.

Author: dutow

Makefile

@@ -1,6 +1,6 @@
 COMPOSE ?= docker compose -f docker-compose.dev.yml
 
-.PHONY: dev dev-detach down shell console test imap logs db-migrate db-reset psql
+.PHONY: dev dev-detach down shell console test imap logs db-migrate db-reset psql sim-email-once sim-email-stream
 
 dev: ## Start dev stack (foreground)
 	$(COMPOSE) up --build
@@ -34,3 +34,9 @@ imap: ## Start stack with IMAP worker profile
 
 logs: ## Follow web logs
 	$(COMPOSE) logs -f web
+
+sim-email-once: ## Send a single simulated email (env: SENT_OFFSET_SECONDS, EXISTING_ALIAS_PROB, EXISTING_TOPIC_PROB)
+	$(COMPOSE) exec web ruby script/simulate_email_once.rb
+
+sim-email-stream: ## Start a continuous simulated email stream (env: MIN_INTERVAL_SECONDS, MAX_INTERVAL_SECONDS, EXISTING_ALIAS_PROB, EXISTING_TOPIC_PROB)
+	$(COMPOSE) exec web ruby script/simulate_email_stream.rb

README.md

@@ -31,7 +31,19 @@ Makefile shortcuts:
 * `make db-migrate` / `make db-reset`
 * `make psql`
 
-Optional IMAP worker (off by default):
+### Incoming email simulator
+
+There are two helper scripts `script/simulate_email_once.rb` and `simulate_email_stream.rb` that simulate incoming emails.
+The scripts can be configured by a few environment variables, for details see the source of the scripts.
+
+Makefile shortcuts:
+* `make sim-email-once`
+* `make sim-email-stream`
+
+### IMAP worker
+
+The "production" IMAP worker which pulls actual mailing list messages from an IMAP label can be also run locally.
+
 ```bash
 docker compose -f docker-compose.dev.yml --profile imap up --build
 ```

lib/dev/email_simulator.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "faker"
+
+module Dev
+  class EmailSimulator
+    DEFAULT_EXISTING_ALIAS_PROB = 0.9
+    DEFAULT_EXISTING_TOPIC_PROB = 0.9
+
+    def initialize(existing_alias_prob: DEFAULT_EXISTING_ALIAS_PROB, existing_topic_prob: DEFAULT_EXISTING_TOPIC_PROB)
+      @existing_alias_prob = existing_alias_prob
+      @existing_topic_prob = existing_topic_prob
+    end
+
+    def generate_mail(sent_at: Time.current)
+      from_alias = pick_alias
+      topic, reply_to = pick_topic
+
+      mail = Mail.new
+      mail.date = sent_at
+      mail.message_id = "<sim-#{SecureRandom.hex}@hackorum.dev>"
+      mail.from = "#{from_alias.name} <#{from_alias.email}>"
+      mail.to = to_addresses(reply_to, topic)
+      mail.subject = subject_for(topic, reply_to)
+
+      if reply_to&.message_id.present?
+        mail.in_reply_to = reply_to.message_id
+        mail.references = reply_to.message_id
+      end
+
+      mail.body = body_for(reply_to)
+      mail
+    end
+
+    def ingest!(mail)
+      EmailIngestor.new.ingest_raw(mail.to_s)
+    end
+
+    def generate_and_ingest!(sent_at: Time.current)
+      ingest!(generate_mail(sent_at: sent_at))
+    end
+
+    private
+
+    def pick_alias
+      use_existing = rand < @existing_alias_prob
+      existing = Alias.order("RANDOM()").first if use_existing
+      return existing if existing
+
+      Alias.create!(
+        name: Faker::Name.name,
+        email: Faker::Internet.email,
+        primary_alias: false,
+        verified_at: Time.current
+      )
+    end
+
+    def pick_topic
+      use_existing = rand < @existing_topic_prob
+      existing_topic = Topic.joins(:messages).order("RANDOM()").first if use_existing
+      return [existing_topic, existing_topic&.messages&.order("RANDOM()")&.first] if existing_topic
+
+      [nil, nil]
+    end
+
+    def subject_for(topic, reply_to)
+      return "Re: #{topic.title}" if topic && reply_to
+      return topic.title if topic
+      Faker::Hacker.say_something_smart.capitalize
+    end
+
+    def body_for(reply_to)
+      paragraphs = Faker::Lorem.paragraphs(number: rand(2..5))
+      body = paragraphs.join("\n\n")
+      if reply_to
+        quoted = reply_to.body.to_s.lines.first(4).map { |l| "> #{l.chomp}" }.join("\n")
+        body = "#{quoted}\n\n#{body}"
+      end
+      body
+    end
+
+    def to_addresses(reply_to, topic)
+      recipients = []
+      if reply_to
+        recipients << reply_to.sender.email
+      elsif topic
+        recipients << topic.creator.email
+      else
+        recipients << Faker::Internet.email
+      end
+      recipients
+    end
+  end
+end

script/simulate_email_once.rb

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../config/environment"
+require_relative "../lib/dev/email_simulator"
+
+offset_seconds = ENV.fetch("SENT_OFFSET_SECONDS", "5").to_i
+existing_alias_prob = ENV.fetch("EXISTING_ALIAS_PROB", "0.9").to_f
+existing_topic_prob = ENV.fetch("EXISTING_TOPIC_PROB", "0.9").to_f
+
+simulator = Dev::EmailSimulator.new(
+  existing_alias_prob: existing_alias_prob,
+  existing_topic_prob: existing_topic_prob
+)
+
+sent_at = Time.current - offset_seconds
+msg = simulator.generate_mail(sent_at: sent_at)
+simulator.ingest!(msg)
+
+puts "Injected simulated email at #{sent_at} with subject: #{msg.subject}"

script/simulate_email_stream.rb

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../config/environment"
+require_relative "../lib/dev/email_simulator"
+
+min_interval = ENV.fetch("MIN_INTERVAL_SECONDS", "30").to_i
+max_interval = ENV.fetch("MAX_INTERVAL_SECONDS", "90").to_i
+existing_alias_prob = ENV.fetch("EXISTING_ALIAS_PROB", "0.9").to_f
+existing_topic_prob = ENV.fetch("EXISTING_TOPIC_PROB", "0.9").to_f
+
+raise ArgumentError, "MIN_INTERVAL_SECONDS must be > 0" if min_interval <= 0
+raise ArgumentError, "MAX_INTERVAL_SECONDS must be >= MIN_INTERVAL_SECONDS" if max_interval < min_interval
+
+simulator = Dev::EmailSimulator.new(
+  existing_alias_prob: existing_alias_prob,
+  existing_topic_prob: existing_topic_prob
+)
+
+puts "Starting email stream simulator (interval #{min_interval}-#{max_interval}s, existing alias prob #{existing_alias_prob}, existing topic prob #{existing_topic_prob})"
+
+loop do
+  sent_at = Time.current - rand(1..10)
+  mail = simulator.generate_mail(sent_at: sent_at)
+  simulator.ingest!(mail)
+  puts "[#{Time.current}] Injected simulated email with subject: #{mail.subject}"
+
+  sleep rand(min_interval..max_interval)
+end