Merge pull request #6 from vizhub-core/streaming-non-code-lines

curran · web-flow · commit 7331c804bbc4 · 2025-04-04T12:15:37.000-04:00
Add streaming parsing of non-code lines
diff --git a/README.md b/README.md
@@ -129,10 +129,13 @@ Converts an array of file objects into a Markdown string using the Bold Format.
 
 A streaming parser that processes Markdown content chunk by chunk, emitting callbacks when file names or code blocks are detected. This is particularly useful for real-time code editing views where users can see LLMs edit specific files as they generate content.
 
-```typescript
-import { StreamingMarkdownParser, StreamingParserCallbacks } from "llm-code-format";
+````typescript
+import {
+  StreamingMarkdownParser,
+  StreamingParserCallbacks,
+} from "llm-code-format";
 
-// Define callbacks for file name changes and code lines
+// Define callbacks for file name changes, code lines, and non-code lines
 const callbacks: StreamingParserCallbacks = {
   onFileNameChange: (fileName, format) => {
     console.log(`File changed to: ${fileName} (${format})`);
@@ -141,7 +144,11 @@ const callbacks: StreamingParserCallbacks = {
   onCodeLine: (line) => {
     console.log(`Code line: ${line}`);
     // Append the line to the current file's content in the UI
-  }
+  },
+  onNonCodeLine: (line) => {
+    console.log(`Comment/text: ${line}`);
+    // Process non-code, non-header lines (e.g., for displaying comments)
+  },
 };
 
 // Create a new parser instance with the callbacks
@@ -156,7 +163,7 @@ parser.processChunk("```\n");
 
 // Flush any remaining content when the stream ends
 parser.flushRemaining();
-```
+````
 
 #### Methods
 
@@ -170,9 +177,14 @@ parser.flushRemaining();
 type StreamingParserCallbacks = {
   onFileNameChange: (fileName: string, format: string) => void;
   onCodeLine: (line: string) => void;
+  onNonCodeLine?: (line: string) => void;
 };
 ```
 
+- **onFileNameChange**: Called when a file header is detected outside a code fence.
+- **onCodeLine**: Called for each line emitted from inside a code fence.
+- **onNonCodeLine**: Called for each line outside code fences that is not a file header.
+
 Currently, the streaming parser only supports the "Bold Format" (`**filename.js**`), but is designed to be extensible for supporting additional formats in the future.
 
 ## License
diff --git a/src/streamingParser.test.ts b/src/streamingParser.test.ts
@@ -10,6 +10,7 @@ describe("StreamingMarkdownParser", () => {
     format: string;
   }>;
   let codeLines: string[];
+  let nonCodeLines: string[];
   let parser: StreamingMarkdownParser;
 
   const callbacks: StreamingParserCallbacks = {
@@ -19,11 +20,15 @@ describe("StreamingMarkdownParser", () => {
     onCodeLine: (line) => {
       codeLines.push(line);
     },
+    onNonCodeLine: (line) => {
+      nonCodeLines.push(line);
+    },
   };
 
   beforeEach(() => {
     fileNameChanges = [];
     codeLines = [];
+    nonCodeLines = [];
     parser = new StreamingMarkdownParser(callbacks);
   });
 
@@ -86,13 +91,14 @@ describe("StreamingMarkdownParser", () => {
     ]);
   });
 
-  it("should not trigger callbacks for irrelevant lines outside code fences", () => {
+  it("should not trigger file name or code callbacks for irrelevant lines outside code fences", () => {
     // Provide a line that doesn't match any header or code fence
     parser.processChunk("This is an irrelevant line\n");
     parser.flushRemaining();
 
     expect(fileNameChanges).toEqual([]);
     expect(codeLines).toEqual([]);
+    expect(nonCodeLines).toEqual(["This is an irrelevant line"]);
   });
 
   it("should handle nested chunks with header and code block boundaries", () => {
@@ -117,6 +123,7 @@ describe("StreamingMarkdownParser", () => {
     ]);
 
     expect(codeLines).toEqual(["<html>", "</html>", "body { color: blue; }"]);
+    expect(nonCodeLines).toEqual(["Some irrelevant line"]);
   });
 
   it("should handle bold format with extra text", () => {
@@ -130,4 +137,27 @@ describe("StreamingMarkdownParser", () => {
     ]);
     expect(codeLines).toEqual(["<html>", "</html>"]);
   });
+  it("should capture all non-code, non-header lines", () => {
+    const input =
+      "This is a regular text line\n" +
+      "**index.html**\n" +
+      "This is a comment about the file\n" +
+      "```\n<html>\n</html>\n```\n" +
+      "Another comment line\n" +
+      "And one more line\n";
+
+    parser.processChunk(input);
+    parser.flushRemaining();
+
+    expect(fileNameChanges).toEqual([
+      { name: "index.html", format: "Bold Format" },
+    ]);
+    expect(codeLines).toEqual(["<html>", "</html>"]);
+    expect(nonCodeLines).toEqual([
+      "This is a regular text line",
+      "This is a comment about the file",
+      "Another comment line",
+      "And one more line",
+    ]);
+  });
 });
diff --git a/src/streamingParser.ts b/src/streamingParser.ts
@@ -11,6 +11,12 @@ export type StreamingParserCallbacks = {
    * @param line - A line of code from the code block.
    */
   onCodeLine: (line: string) => void;
+
+  /**
+   * Called for each line outside code fences that is not a file header.
+   * @param line - A line of text that is not code or a file header.
+   */
+  onNonCodeLine?: (line: string) => void;
 };
 
 export class StreamingMarkdownParser {
@@ -96,7 +102,19 @@ export class StreamingMarkdownParser {
           break; // Stop after the first matching header is found.
         }
       }
-      // Non-header lines outside code fences are ignored
+      // Non-header lines outside code fences
+      let isHeader = false;
+      for (const { regex } of this.headerPatterns) {
+        if (regex.test(line)) {
+          isHeader = true;
+          break;
+        }
+      }
+
+      // If it's not a header and the callback exists, call it
+      if (!isHeader && this.callbacks.onNonCodeLine) {
+        this.callbacks.onNonCodeLine(line);
+      }
     }
   }
 }
