Added FileScrivener, ImbuingTransmuter, converted acks to use reference position instead of text, added polymorphic serialization attributes to entries.

.gitignore

@@ -58,5 +58,12 @@ _ReSharper.*/
 *.rej
 scratch.txt
 
+# Local scratch notes
+/scratch.md
+
+# FileScrivener toy outputs — keep scope tight
+# Only ignore the toy's artifacts; do not blanket-ignore any /data folders
+/src/toys/Coven.Toys.FileScrivenerConsole/data/
+
 # Codex CLI generated state
 **/.codex/

.vscode/launch.json

@@ -67,6 +67,17 @@
       "justMyCode": true,
       "preLaunchTask": "build"
     },
+    {
+      "name": "Launch: Coven.Toys.FileScrivenerConsole",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/toys/Coven.Toys.FileScrivenerConsole/bin/Debug/net10.0/Coven.Toys.FileScrivenerConsole.dll",
+      "cwd": "${workspaceFolder}/src/toys/Coven.Toys.FileScrivenerConsole",
+      "console": "integratedTerminal",
+      "stopAtEntry": false,
+      "justMyCode": true,
+      "preLaunchTask": "build"
+    },
     {
       "name": "Attach: .NET Core",
       "type": "coreclr",

INDEX.md

@@ -25,6 +25,8 @@ Project overview: see [README](/README.md).
 - Agents (branch): [/src/Coven.Agents](/src/Coven.Agents/) ([README](/src/Coven.Agents/README.md))
 - Agents OpenAI (leaf): [/src/Coven.Agents.OpenAI](/src/Coven.Agents.OpenAI/) ([README](/src/Coven.Agents.OpenAI/README.md))
 
+- Scriveners: [/src/Coven.Scriveners.FileScrivener](/src/Coven.Scriveners.FileScrivener/) ([README](/src/Coven.Scriveners.FileScrivener/README.md))
+
 - Tests: [/src/Coven.Core.Tests](/src/Coven.Core.Tests/), [/src/Coven.Daemonology.Tests](/src/Coven.Daemonology.Tests/)
 
 ## Samples
@@ -36,3 +38,4 @@ Project overview: see [README](/README.md).
 - [/src/toys/Coven.Toys.ConsoleOpenAIStreaming](/src/toys/Coven.Toys.ConsoleOpenAIStreaming/)
 - [/src/toys/Coven.Toys.DiscordChat](/src/toys/Coven.Toys.DiscordChat/)
 - [/src/toys/Coven.Toys.DiscordStreaming](/src/toys/Coven.Toys.DiscordStreaming/)
+- [/src/toys/Coven.Toys.FileScrivenerConsole](/src/toys/Coven.Toys.FileScrivenerConsole/)

architecture/Journaling-and-Scriveners.md

@@ -35,6 +35,12 @@ Scriveners are append‑only journals that record typed entries. They decouple p
 - Avoid long‑running synchronous work in consumers; prefer daemons that tail asynchronously.
 - Treat journals as the source of truth for cross‑component communication.
 
+## Persistence
+- In‑memory by default: `InMemoryScrivener<T>` (fast, replayable within process).
+- File‑backed option: `Coven.Scriveners.FileScrivener` provides a background flusher that appends NDJSON snapshots to disk while your app continues to use the in‑memory journal.
+  - See package README: `/src/Coven.Scriveners.FileScrivener/README.md`.
+  - Try the toy: `/src/toys/Coven.Toys.FileScrivenerConsole/`.
+
 ## Related
 - Windowing/Shattering: see “Windowing and Shattering”.
 - Daemon lifecycle: see `src/Coven.Daemonology/README.md`.

architecture/README.md

@@ -43,6 +43,7 @@ Concrete examples help ground the vocabulary above. The Sample 01 app wires Disc
 - Journals (Scriveners): append‑only logs connecting router↔branches↔leaves.
   - Chat journal: `IScrivener<ChatEntry>` (Discord adapter implements its own scrivener; router reads/writes entries).
   - Agent journal: `IScrivener<AgentEntry>` (OpenAI adapter scrivener; router reads/writes entries).
+  - Persistence: use `Coven.Scriveners.FileScrivener` to append NDJSON snapshots to disk while keeping in‑process journals in memory.
 
 - Directionality in the router logic:
   - Afferent (inbound): Discord adapter writes `ChatAfferent` when a user posts in the channel; router reads and forwards as an `AgentPrompt`.

src/Coven.Agents.OpenAI/OpenAIAgentSession.cs

@@ -7,20 +7,26 @@
 
 namespace Coven.Agents.OpenAI;
 
+/// <summary>
+/// Coordinates an OpenAI agent session bridging the OpenAI and Agent journals.
+/// Uses imbuing transmuters to carry the source journal position as a reagent for position-based ACKs.
+/// </summary>
 internal sealed class OpenAIAgentSession(
     IOpenAIGatewayConnection gateway,
     IScrivener<OpenAIEntry> openAIJournal,
     IScrivener<AgentEntry> agentJournal,
     IShatterPolicy<OpenAIEntry> shatterPolicy,
-    IBiDirectionalTransmuter<OpenAIEntry, AgentEntry> transmuter,
+    IImbuingTransmuter<OpenAIEntry, long, AgentEntry> afferentTransmuter,
+    IImbuingTransmuter<AgentEntry, long, OpenAIEntry> efferentTransmuter,
     ILogger<OpenAIAgentSession> logger,
     CancellationToken sessionToken) : IAsyncDisposable
 {
     private readonly IOpenAIGatewayConnection _gateway = gateway ?? throw new ArgumentNullException(nameof(gateway));
     private readonly IScrivener<OpenAIEntry> _openAIJournal = openAIJournal ?? throw new ArgumentNullException(nameof(openAIJournal));
     private readonly IScrivener<AgentEntry> _agentJournal = agentJournal ?? throw new ArgumentNullException(nameof(agentJournal));
     private readonly IShatterPolicy<OpenAIEntry> _shatterPolicy = shatterPolicy ?? throw new ArgumentNullException(nameof(shatterPolicy));
-    private readonly IBiDirectionalTransmuter<OpenAIEntry, AgentEntry> _transmuter = transmuter ?? throw new ArgumentNullException(nameof(transmuter));
+    private readonly IImbuingTransmuter<OpenAIEntry, long, AgentEntry> _afferentTransmuter = afferentTransmuter ?? throw new ArgumentNullException(nameof(afferentTransmuter));
+    private readonly IImbuingTransmuter<AgentEntry, long, OpenAIEntry> _efferentTransmuter = efferentTransmuter ?? throw new ArgumentNullException(nameof(efferentTransmuter));
     private readonly ILogger _logger = logger ?? throw new ArgumentNullException(nameof(logger));
     private readonly CancellationToken _sessionToken = sessionToken;
 
@@ -55,7 +61,7 @@ public async Task StartAsync()
                             if (openAIEntry is OpenAIAfferentThoughtChunk)
                             {
                                 produced = true;
-                                AgentEntry agentChunk = await _transmuter.TransmuteAfferent(openAIEntry, ct).ConfigureAwait(false);
+                                AgentEntry agentChunk = await _afferentTransmuter.Transmute(openAIEntry, position, ct).ConfigureAwait(false);
                                 long pos = await _agentJournal.WriteAsync(agentChunk, ct).ConfigureAwait(false);
                                 OpenAILog.OpenAIToAgentsAppended(_logger, agentChunk.GetType().Name, pos);
                             }
@@ -67,7 +73,7 @@ public async Task StartAsync()
                         }
                     }
 
-                    AgentEntry agent = await _transmuter.TransmuteAfferent(entry, ct).ConfigureAwait(false);
+                    AgentEntry agent = await _afferentTransmuter.Transmute(entry, position, ct).ConfigureAwait(false);
                     OpenAILog.OpenAIToAgentsTransmuted(_logger, entry.GetType().Name, agent.GetType().Name);
                     long agentPos = await _agentJournal.WriteAsync(agent, ct).ConfigureAwait(false);
                     OpenAILog.OpenAIToAgentsAppended(_logger, agent.GetType().Name, agentPos);
@@ -98,7 +104,7 @@ public async Task StartAsync()
                     }
 
                     OpenAILog.AgentsToOpenAIObserved(_logger, entry.GetType().Name, position);
-                    OpenAIEntry openAI = await _transmuter.TransmuteEfferent(entry, ct).ConfigureAwait(false);
+                    OpenAIEntry openAI = await _efferentTransmuter.Transmute(entry, position, ct).ConfigureAwait(false);
                     OpenAILog.AgentsToOpenAITransmuted(_logger, entry.GetType().Name, openAI.GetType().Name);
                     long aiPos = await _openAIJournal.WriteAsync(openAI, ct).ConfigureAwait(false);
                     OpenAILog.AgentsToOpenAIAppended(_logger, openAI.GetType().Name, aiPos);

src/Coven.Agents.OpenAI/OpenAIAgentSessionFactory.cs

@@ -7,21 +7,26 @@
 
 namespace Coven.Agents.OpenAI;
 
+/// <summary>
+/// Constructs <see cref="OpenAIAgentSession"/> instances with imbuing transmuters for position-based acknowledgements.
+/// </summary>
 internal sealed class OpenAIAgentSessionFactory(
     IOpenAIGatewayConnection gatewayConnection,
     IScrivener<OpenAIEntry> openAIJournal,
     IScrivener<AgentEntry> agentJournal,
-    IBiDirectionalTransmuter<OpenAIEntry, AgentEntry> transmuter,
+    IImbuingTransmuter<OpenAIEntry, long, AgentEntry> afferentTransmuter,
+    IImbuingTransmuter<AgentEntry, long, OpenAIEntry> efferentTransmuter,
     IShatterPolicy<OpenAIEntry> shatterPolicy,
     ILogger<OpenAIAgentSession> logger)
 {
     private readonly IOpenAIGatewayConnection _gatewayConnection = gatewayConnection ?? throw new ArgumentNullException(nameof(gatewayConnection));
     private readonly IScrivener<OpenAIEntry> _openAIJournal = openAIJournal ?? throw new ArgumentNullException(nameof(openAIJournal));
     private readonly IScrivener<AgentEntry> _agentJournal = agentJournal ?? throw new ArgumentNullException(nameof(agentJournal));
-    private readonly IBiDirectionalTransmuter<OpenAIEntry, AgentEntry> _transmuter = transmuter ?? throw new ArgumentNullException(nameof(transmuter));
+    private readonly IImbuingTransmuter<OpenAIEntry, long, AgentEntry> _afferentTransmuter = afferentTransmuter ?? throw new ArgumentNullException(nameof(afferentTransmuter));
+    private readonly IImbuingTransmuter<AgentEntry, long, OpenAIEntry> _efferentTransmuter = efferentTransmuter ?? throw new ArgumentNullException(nameof(efferentTransmuter));
     private readonly IShatterPolicy<OpenAIEntry> _shatterPolicy = shatterPolicy ?? throw new ArgumentNullException(nameof(shatterPolicy));
     private readonly ILogger<OpenAIAgentSession> _logger = logger ?? throw new ArgumentNullException(nameof(logger));
 
     public OpenAIAgentSession Create(CancellationToken sessionToken)
-        => new(_gatewayConnection, _openAIJournal, _agentJournal, _shatterPolicy, _transmuter, _logger, sessionToken);
+        => new(_gatewayConnection, _openAIJournal, _agentJournal, _shatterPolicy, _afferentTransmuter, _efferentTransmuter, _logger, sessionToken);
 }

src/Coven.Agents.OpenAI/OpenAIAgentsServiceCollectionExtensions.cs

@@ -14,7 +14,7 @@ namespace Coven.Agents.OpenAI;
 
 /// <summary>
 /// Dependency Injection helpers for wiring the OpenAI agent integration.
-/// Registers journals, gateway connection, transmuters, windowing daemons, and the official OpenAI client.
+/// Registers journals, gateway connection, imbuing transmuters (position-based ACKs), windowing daemons, and the official OpenAI client.
 /// </summary>
 public static class OpenAIAgentsServiceCollectionExtensions
 {
@@ -87,8 +87,9 @@ public static IServiceCollection AddOpenAIAgents(this IServiceCollection service
             services.TryAddScoped<IOpenAIGatewayConnection, OpenAIRequestGatewayConnection>();
         }
 
-        // Transmuter and daemon
-        services.AddScoped<IBiDirectionalTransmuter<OpenAIEntry, AgentEntry>, OpenAITransmuter>();
+        // Transmuters and daemon
+        services.AddScoped<IImbuingTransmuter<OpenAIEntry, long, AgentEntry>, OpenAITransmuter>();
+        services.AddScoped<IImbuingTransmuter<AgentEntry, long, OpenAIEntry>, OpenAITransmuter>();
         services.TryAddScoped<ITransmuter<OpenAIEntry, ResponseItem?>, OpenAIEntryToResponseItemTransmuter>();
         services.TryAddScoped<IOpenAITranscriptBuilder, DefaultOpenAITranscriptBuilder>();
         // Session-local shattering for OpenAI chunks: split on paragraph boundary

src/Coven.Agents.OpenAI/OpenAIEntry.cs

@@ -1,12 +1,22 @@
 // SPDX-License-Identifier: BUSL-1.1
 
+using System.Text.Json.Serialization;
 using Coven.Core;
 
 namespace Coven.Agents.OpenAI;
 
 /// <summary>
 /// Base entry type for OpenAI agent journals (requests, responses, thoughts, chunks, acknowledgements).
 /// </summary>
+[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")]
+[JsonDerivedType(typeof(OpenAIEfferent), nameof(OpenAIEfferent))]
+[JsonDerivedType(typeof(OpenAIAfferent), nameof(OpenAIAfferent))]
+[JsonDerivedType(typeof(OpenAIAfferentChunk), nameof(OpenAIAfferentChunk))]
+[JsonDerivedType(typeof(OpenAIAfferentThoughtChunk), nameof(OpenAIAfferentThoughtChunk))]
+[JsonDerivedType(typeof(OpenAIThought), nameof(OpenAIThought))]
+[JsonDerivedType(typeof(OpenAIAck), nameof(OpenAIAck))]
+[JsonDerivedType(typeof(OpenAIEfferentThoughtChunk), nameof(OpenAIEfferentThoughtChunk))]
+[JsonDerivedType(typeof(OpenAIStreamCompleted), nameof(OpenAIStreamCompleted))]
 public abstract record OpenAIEntry(
     string Sender
 ) : Entry;
@@ -57,7 +67,7 @@ string Model
 /// <summary>OpenAI acknowledgement used for synchronization.</summary>
 public sealed record OpenAIAck(
     string Sender,
-    string Text
+    long Position
 ) : OpenAIEntry(Sender);
 
 // Streaming thought chunks (efferent): agent streams thoughts out

src/Coven.Agents.OpenAI/OpenAITransmuter.cs

@@ -5,47 +5,63 @@
 namespace Coven.Agents.OpenAI;
 
 /// <summary>
-/// Maps between OpenAI-specific entries and generic Agent entries.
+/// Maps between OpenAI-specific entries and generic Agent entries using position-imbued ACKs.
 /// Afferent: OpenAI → Agent; Efferent: Agent → OpenAI.
 /// </summary>
-internal sealed class OpenAITransmuter : IBiDirectionalTransmuter<OpenAIEntry, AgentEntry>
+internal sealed class OpenAITransmuter
+    : IImbuingTransmuter<OpenAIEntry, long, AgentEntry>,
+      IImbuingTransmuter<AgentEntry, long, OpenAIEntry>
 {
-    public Task<AgentEntry> TransmuteAfferent(OpenAIEntry Input, CancellationToken cancellationToken)
+    /// <summary>
+    /// Transmutes OpenAI-afferent entries to Agent entries.
+    /// </summary>
+    /// <param name="Input">The source OpenAI entry.</param>
+    /// <param name="Reagent">The source journal position used for position-based acknowledgements.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>The mapped Agent entry.</returns>
+    public Task<AgentEntry> Transmute(OpenAIEntry Input, long Reagent, CancellationToken cancellationToken = default)
     {
         cancellationToken.ThrowIfCancellationRequested();
 
         return Input switch
         {
             OpenAIAfferent incoming => Task.FromResult<AgentEntry>(new AgentResponse(incoming.Sender, incoming.Text)),
-            // Today, Afferent chunks include thoughts, tomorrow who knows.
             OpenAIAfferentChunk chunk => Task.FromResult<AgentEntry>(new AgentAfferentChunk(chunk.Sender, chunk.Text)),
-            // Streaming thought chunks from OpenAI surface as afferent thought drafts
             OpenAIAfferentThoughtChunk tChunk => Task.FromResult<AgentEntry>(new AgentAfferentThoughtChunk(tChunk.Sender, tChunk.Text)),
-            // A full OpenAIThought should surface as a fixed AgentThought
             OpenAIThought thought => Task.FromResult<AgentEntry>(new AgentThought(thought.Sender, thought.Text)),
             OpenAIStreamCompleted done => Task.FromResult<AgentEntry>(new AgentStreamCompleted(done.Sender)),
-            OpenAIEfferent outgoing => Task.FromResult<AgentEntry>(new AgentAck(outgoing.Sender)),
+            // For efferent records observed on the OpenAI journal or explicit OpenAI acks, emit an AgentAck with the source position
+            OpenAIEfferent outgoing => Task.FromResult<AgentEntry>(new AgentAck(outgoing.Sender, Reagent)),
+            OpenAIAck => Task.FromResult<AgentEntry>(new AgentAck(Input.Sender, Reagent)),
             _ => throw new ArgumentOutOfRangeException(nameof(Input))
         };
     }
 
-    public Task<OpenAIEntry> TransmuteEfferent(AgentEntry Output, CancellationToken cancellationToken)
+    /// <summary>
+    /// Transmutes Agent-efferent entries to OpenAI entries.
+    /// </summary>
+    /// <param name="Input">The source Agent entry.</param>
+    /// <param name="Reagent">The source journal position used for position-based acknowledgements.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>The mapped OpenAI entry.</returns>
+    public Task<OpenAIEntry> Transmute(AgentEntry Input, long Reagent, CancellationToken cancellationToken = default)
     {
         cancellationToken.ThrowIfCancellationRequested();
 
-        return Output switch
+        return Input switch
         {
             AgentPrompt prompt => Task.FromResult<OpenAIEntry>(new OpenAIEfferent(prompt.Sender, prompt.Text)),
-            AgentResponse response => Task.FromResult<OpenAIEntry>(new OpenAIAck(response.Sender, response.Text)),
-            AgentThought thought => Task.FromResult<OpenAIEntry>(new OpenAIAck(thought.Sender, thought.Text)),
-            AgentEfferentChunk efferentChunk => Task.FromResult<OpenAIEntry>(new OpenAIAck(efferentChunk.Sender, efferentChunk.Text)),
-            AgentAfferentChunk afferentChunk => Task.FromResult<OpenAIEntry>(new OpenAIAck(afferentChunk.Sender, afferentChunk.Text)),
-            // Streaming efferent thought drafts map to OpenAI efferent thought chunk (not forwarded by gateway today)
+
+            // All other Agent entries (including drafts and fixed) acknowledge with the position being processed
+            AgentResponse response => Task.FromResult<OpenAIEntry>(new OpenAIAck(response.Sender, Reagent)),
+            AgentThought thought => Task.FromResult<OpenAIEntry>(new OpenAIAck(thought.Sender, Reagent)),
+            AgentEfferentChunk efferentChunk => Task.FromResult<OpenAIEntry>(new OpenAIAck(efferentChunk.Sender, Reagent)),
+            AgentAfferentChunk afferentChunk => Task.FromResult<OpenAIEntry>(new OpenAIAck(afferentChunk.Sender, Reagent)),
             AgentEfferentThoughtChunk etChunk => Task.FromResult<OpenAIEntry>(new OpenAIEfferentThoughtChunk(etChunk.Sender, etChunk.Text)),
-            // Afferent thought drafts are not sent outward; ack for completeness
-            AgentAfferentThoughtChunk atChunk => Task.FromResult<OpenAIEntry>(new OpenAIAck(atChunk.Sender, atChunk.Text)),
-            AgentStreamCompleted done => Task.FromResult<OpenAIEntry>(new OpenAIAck(done.Sender, string.Empty)),
-            _ => throw new ArgumentOutOfRangeException(nameof(Output))
+            AgentAfferentThoughtChunk atChunk => Task.FromResult<OpenAIEntry>(new OpenAIAck(atChunk.Sender, Reagent)),
+            AgentStreamCompleted done => Task.FromResult<OpenAIEntry>(new OpenAIAck(done.Sender, Reagent)),
+            AgentAck ack => Task.FromResult<OpenAIEntry>(new OpenAIAck(ack.Sender, Reagent)),
+            _ => throw new ArgumentOutOfRangeException(nameof(Input))
         };
     }
 }

src/Coven.Agents/AgentEntry.cs

@@ -1,12 +1,23 @@
 // SPDX-License-Identifier: BUSL-1.1
 
+using System.Text.Json.Serialization;
 using Coven.Core;
 
 namespace Coven.Agents;
 
 /// <summary>
 /// Base entry type for agent journals (prompts, responses, thoughts, acks, and streaming chunks).
 /// </summary>
+[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")]
+[JsonDerivedType(typeof(AgentPrompt), nameof(AgentPrompt))]
+[JsonDerivedType(typeof(AgentResponse), nameof(AgentResponse))]
+[JsonDerivedType(typeof(AgentThought), nameof(AgentThought))]
+[JsonDerivedType(typeof(AgentAck), nameof(AgentAck))]
+[JsonDerivedType(typeof(AgentEfferentChunk), nameof(AgentEfferentChunk))]
+[JsonDerivedType(typeof(AgentAfferentChunk), nameof(AgentAfferentChunk))]
+[JsonDerivedType(typeof(AgentEfferentThoughtChunk), nameof(AgentEfferentThoughtChunk))]
+[JsonDerivedType(typeof(AgentAfferentThoughtChunk), nameof(AgentAfferentThoughtChunk))]
+[JsonDerivedType(typeof(AgentStreamCompleted), nameof(AgentStreamCompleted))]
 public abstract record AgentEntry(string Sender) : Entry;
 
 /// <summary>
@@ -23,8 +34,8 @@ public sealed record AgentResponse(string Sender, string Text) : AgentEntry(Send
 /// <summary>Represents an agent's introspective thought (not typically user-visible).</summary>
 public sealed record AgentThought(string Sender, string Text) : AgentEntry(Sender);
 
-/// <summary>Represents an acknowledgement for internal synchronization.</summary>
-public sealed record AgentAck(string Sender) : AgentEntry(Sender);
+/// <summary>Represents an acknowledgement for internal synchronization. Carries the position being acknowledged.</summary>
+public sealed record AgentAck(string Sender, long Position) : AgentEntry(Sender);
 
 // Streaming additions
 /// <summary>Outgoing (efferent) response chunk prior to finalization.</summary>