Truncate oversized git diffs instead of failing (#1499)

juliusmarminge · web-flow · commit 9e60597174dc · 2026-03-28T20:50:23.000-07:00
diff --git a/apps/server/src/checkpointing/Layers/CheckpointStore.test.ts b/apps/server/src/checkpointing/Layers/CheckpointStore.test.ts
@@ -0,0 +1,122 @@
+import path from "node:path";
+
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { it } from "@effect/vitest";
+import { Effect, FileSystem, Layer, PlatformError, Scope } from "effect";
+import { describe, expect } from "vitest";
+
+import { checkpointRefForThreadTurn } from "../Utils.ts";
+import { CheckpointStoreLive } from "./CheckpointStore.ts";
+import { CheckpointStore } from "../Services/CheckpointStore.ts";
+import { GitCoreLive } from "../../git/Layers/GitCore.ts";
+import { GitCore } from "../../git/Services/GitCore.ts";
+import { GitCommandError } from "../../git/Errors.ts";
+import { ServerConfig } from "../../config.ts";
+import { ThreadId } from "@t3tools/contracts";
+
+const ServerConfigLayer = ServerConfig.layerTest(process.cwd(), {
+  prefix: "t3-checkpoint-store-test-",
+});
+const GitCoreTestLayer = GitCoreLive.pipe(
+  Layer.provide(ServerConfigLayer),
+  Layer.provide(NodeServices.layer),
+);
+const CheckpointStoreTestLayer = CheckpointStoreLive.pipe(
+  Layer.provide(GitCoreTestLayer),
+  Layer.provide(NodeServices.layer),
+);
+const TestLayer = Layer.mergeAll(NodeServices.layer, GitCoreTestLayer, CheckpointStoreTestLayer);
+
+function makeTmpDir(
+  prefix = "checkpoint-store-test-",
+): Effect.Effect<string, PlatformError.PlatformError, FileSystem.FileSystem | Scope.Scope> {
+  return Effect.gen(function* () {
+    const fileSystem = yield* FileSystem.FileSystem;
+    return yield* fileSystem.makeTempDirectoryScoped({ prefix });
+  });
+}
+
+function writeTextFile(
+  filePath: string,
+  contents: string,
+): Effect.Effect<void, PlatformError.PlatformError, FileSystem.FileSystem> {
+  return Effect.gen(function* () {
+    const fileSystem = yield* FileSystem.FileSystem;
+    yield* fileSystem.writeFileString(filePath, contents);
+  });
+}
+
+function git(
+  cwd: string,
+  args: ReadonlyArray<string>,
+): Effect.Effect<string, GitCommandError, GitCore> {
+  return Effect.gen(function* () {
+    const gitCore = yield* GitCore;
+    const result = yield* gitCore.execute({
+      operation: "CheckpointStore.test.git",
+      cwd,
+      args,
+      timeoutMs: 10_000,
+    });
+    return result.stdout.trim();
+  });
+}
+
+function initRepoWithCommit(
+  cwd: string,
+): Effect.Effect<
+  void,
+  GitCommandError | PlatformError.PlatformError,
+  GitCore | FileSystem.FileSystem
+> {
+  return Effect.gen(function* () {
+    const core = yield* GitCore;
+    yield* core.initRepo({ cwd });
+    yield* git(cwd, ["config", "user.email", "test@test.com"]);
+    yield* git(cwd, ["config", "user.name", "Test"]);
+    yield* writeTextFile(path.join(cwd, "README.md"), "# test\n");
+    yield* git(cwd, ["add", "."]);
+    yield* git(cwd, ["commit", "-m", "initial commit"]);
+  });
+}
+
+function buildLargeText(lineCount = 20_000): string {
+  return Array.from({ length: lineCount }, (_, index) => `line ${String(index).padStart(5, "0")}`)
+    .join("\n")
+    .concat("\n");
+}
+
+it.layer(TestLayer)("CheckpointStoreLive", (it) => {
+  describe("diffCheckpoints", () => {
+    it.effect("returns full oversized checkpoint diffs without truncation", () =>
+      Effect.gen(function* () {
+        const tmp = yield* makeTmpDir();
+        yield* initRepoWithCommit(tmp);
+        const checkpointStore = yield* CheckpointStore;
+        const threadId = ThreadId.makeUnsafe("thread-checkpoint-store");
+        const fromCheckpointRef = checkpointRefForThreadTurn(threadId, 0);
+        const toCheckpointRef = checkpointRefForThreadTurn(threadId, 1);
+
+        yield* checkpointStore.captureCheckpoint({
+          cwd: tmp,
+          checkpointRef: fromCheckpointRef,
+        });
+        yield* writeTextFile(path.join(tmp, "README.md"), buildLargeText());
+        yield* checkpointStore.captureCheckpoint({
+          cwd: tmp,
+          checkpointRef: toCheckpointRef,
+        });
+
+        const diff = yield* checkpointStore.diffCheckpoints({
+          cwd: tmp,
+          fromCheckpointRef,
+          toCheckpointRef,
+        });
+
+        expect(diff).toContain("diff --git");
+        expect(diff).not.toContain("[truncated]");
+        expect(diff).toContain("+line 19999");
+      }),
+    );
+  });
+});
diff --git a/apps/server/src/git/Layers/GitCore.test.ts b/apps/server/src/git/Layers/GitCore.test.ts
@@ -132,6 +132,12 @@ function commitWithDate(
   });
 }
 
+function buildLargeText(lineCount = 20_000): string {
+  return Array.from({ length: lineCount }, (_, index) => `line ${String(index).padStart(5, "0")}`)
+    .join("\n")
+    .concat("\n");
+}
+
 // ── Tests ──
 
 it.layer(TestLayer)("git integration", (it) => {
@@ -1670,6 +1676,40 @@ it.layer(TestLayer)("git integration", (it) => {
       }),
     );
 
+    it.effect("prepareCommitContext truncates oversized staged patches instead of failing", () =>
+      Effect.gen(function* () {
+        const tmp = yield* makeTmpDir();
+        yield* initRepoWithCommit(tmp);
+        const core = yield* GitCore;
+
+        yield* writeTextFile(path.join(tmp, "README.md"), buildLargeText());
+
+        const context = yield* core.prepareCommitContext(tmp);
+        expect(context).not.toBeNull();
+        expect(context!.stagedSummary).toContain("README.md");
+        expect(context!.stagedPatch).toContain("[truncated]");
+      }),
+    );
+
+    it.effect("readRangeContext truncates oversized diff patches instead of failing", () =>
+      Effect.gen(function* () {
+        const tmp = yield* makeTmpDir();
+        const { initialBranch } = yield* initRepoWithCommit(tmp);
+        const core = yield* GitCore;
+
+        yield* core.createBranch({ cwd: tmp, branch: "feature/large-range-context" });
+        yield* core.checkoutBranch({ cwd: tmp, branch: "feature/large-range-context" });
+        yield* writeTextFile(path.join(tmp, "large.txt"), buildLargeText());
+        yield* git(tmp, ["add", "large.txt"]);
+        yield* git(tmp, ["commit", "-m", "Add large range context"]);
+
+        const rangeContext = yield* core.readRangeContext(tmp, initialBranch);
+        expect(rangeContext.commitSummary).toContain("Add large range context");
+        expect(rangeContext.diffSummary).toContain("large.txt");
+        expect(rangeContext.diffPatch).toContain("[truncated]");
+      }),
+    );
+
     it.effect("pushes with upstream setup and then skips when up to date", () =>
       Effect.gen(function* () {
         const tmp = yield* makeTmpDir();
diff --git a/apps/server/src/git/Layers/GitCore.ts b/apps/server/src/git/Layers/GitCore.ts
@@ -32,6 +32,11 @@ import { decodeJsonResult } from "@t3tools/shared/schemaJson";
 
 const DEFAULT_TIMEOUT_MS = 30_000;
 const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000;
+const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]";
+const PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES = 49_000;
+const RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES = 19_000;
+const RANGE_DIFF_SUMMARY_MAX_OUTPUT_BYTES = 19_000;
+const RANGE_DIFF_PATCH_MAX_OUTPUT_BYTES = 59_000;
 const STATUS_UPSTREAM_REFRESH_INTERVAL = Duration.seconds(15);
 const STATUS_UPSTREAM_REFRESH_TIMEOUT = Duration.seconds(5);
 const STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY = 2_048;
@@ -53,6 +58,8 @@ interface ExecuteGitOptions {
   timeoutMs?: number | undefined;
   allowNonZeroExit?: boolean | undefined;
   fallbackErrorMessage?: string | undefined;
+  maxOutputBytes?: number | undefined;
+  truncateOutputAtMaxBytes?: boolean | undefined;
   progress?: ExecuteGitProgress | undefined;
 }
 
@@ -439,12 +446,14 @@ const collectOutput = Effect.fn(function* <E>(
   input: Pick<ExecuteGitInput, "operation" | "cwd" | "args">,
   stream: Stream.Stream<Uint8Array, E>,
   maxOutputBytes: number,
+  truncateOutputAtMaxBytes: boolean,
   onLine: ((line: string) => Effect.Effect<void, never>) | undefined,
 ): Effect.fn.Return<string, GitCommandError> {
   const decoder = new TextDecoder();
   let bytes = 0;
   let text = "";
   let lineBuffer = "";
+  let truncated = false;
 
   const emitCompleteLines = (flush: boolean) =>
     Effect.gen(function* () {
@@ -469,27 +478,38 @@ const collectOutput = Effect.fn(function* <E>(
 
   yield* Stream.runForEach(stream, (chunk) =>
     Effect.gen(function* () {
-      bytes += chunk.byteLength;
-      if (bytes > maxOutputBytes) {
+      if (truncateOutputAtMaxBytes && truncated) {
+        return;
+      }
+      const nextBytes = bytes + chunk.byteLength;
+      if (!truncateOutputAtMaxBytes && nextBytes > maxOutputBytes) {
         return yield* new GitCommandError({
           operation: input.operation,
           command: quoteGitCommand(input.args),
           cwd: input.cwd,
           detail: `${quoteGitCommand(input.args)} output exceeded ${maxOutputBytes} bytes and was truncated.`,
         });
       }
-      const decoded = decoder.decode(chunk, { stream: true });
+
+      const chunkToDecode =
+        truncateOutputAtMaxBytes && nextBytes > maxOutputBytes
+          ? chunk.subarray(0, Math.max(0, maxOutputBytes - bytes))
+          : chunk;
+      bytes += chunkToDecode.byteLength;
+      truncated = truncateOutputAtMaxBytes && nextBytes > maxOutputBytes;
+
+      const decoded = decoder.decode(chunkToDecode, { stream: !truncated });
       text += decoded;
       lineBuffer += decoded;
       yield* emitCompleteLines(false);
     }),
   ).pipe(Effect.mapError(toGitCommandError(input, "output stream failed.")));
 
-  const remainder = decoder.decode();
+  const remainder = truncated ? "" : decoder.decode();
   text += remainder;
   lineBuffer += remainder;
   yield* emitCompleteLines(true);
-  return text;
+  return truncated ? `${text}${OUTPUT_TRUNCATED_MARKER}` : text;
 });
 
 export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"] }) =>
@@ -511,6 +531,7 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
         } as const;
         const timeoutMs = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;
         const maxOutputBytes = input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
+        const truncateOutputAtMaxBytes = input.truncateOutputAtMaxBytes ?? false;
 
         const commandEffect = Effect.gen(function* () {
           const trace2Monitor = yield* createTrace2Monitor(commandInput, input.progress).pipe(
@@ -537,12 +558,14 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
                 commandInput,
                 child.stdout,
                 maxOutputBytes,
+                truncateOutputAtMaxBytes,
                 input.progress?.onStdoutLine,
               ),
               collectOutput(
                 commandInput,
                 child.stderr,
                 maxOutputBytes,
+                truncateOutputAtMaxBytes,
                 input.progress?.onStderrLine,
               ),
               child.exitCode.pipe(
@@ -603,6 +626,10 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
         args,
         allowNonZeroExit: true,
         ...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
+        ...(options.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}),
+        ...(options.truncateOutputAtMaxBytes !== undefined
+          ? { truncateOutputAtMaxBytes: options.truncateOutputAtMaxBytes }
+          : {}),
         ...(options.progress ? { progress: options.progress } : {}),
       }).pipe(
         Effect.flatMap((result) => {
@@ -647,6 +674,14 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
         Effect.map((result) => result.stdout),
       );
 
+    const runGitStdoutWithOptions = (
+      operation: string,
+      cwd: string,
+      args: readonly string[],
+      options: ExecuteGitOptions = {},
+    ): Effect.Effect<string, GitCommandError> =>
+      executeGit(operation, cwd, args, options).pipe(Effect.map((result) => result.stdout));
+
     const branchExists = (cwd: string, branch: string): Effect.Effect<boolean, GitCommandError> =>
       executeGit(
         "GitCore.branchExists",
@@ -1162,12 +1197,15 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
           return null;
         }
 
-        const stagedPatch = yield* runGitStdout("GitCore.prepareCommitContext.stagedPatch", cwd, [
-          "diff",
-          "--cached",
-          "--patch",
-          "--minimal",
-        ]);
+        const stagedPatch = yield* runGitStdoutWithOptions(
+          "GitCore.prepareCommitContext.stagedPatch",
+          cwd,
+          ["diff", "--cached", "--patch", "--minimal"],
+          {
+            maxOutputBytes: PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES,
+            truncateOutputAtMaxBytes: true,
+          },
+        );
 
         return {
           stagedSummary,
@@ -1363,14 +1401,33 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
         const range = `${baseBranch}..HEAD`;
         const [commitSummary, diffSummary, diffPatch] = yield* Effect.all(
           [
-            runGitStdout("GitCore.readRangeContext.log", cwd, ["log", "--oneline", range]),
-            runGitStdout("GitCore.readRangeContext.diffStat", cwd, ["diff", "--stat", range]),
-            runGitStdout("GitCore.readRangeContext.diffPatch", cwd, [
-              "diff",
-              "--patch",
-              "--minimal",
-              range,
-            ]),
+            runGitStdoutWithOptions(
+              "GitCore.readRangeContext.log",
+              cwd,
+              ["log", "--oneline", range],
+              {
+                maxOutputBytes: RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES,
+                truncateOutputAtMaxBytes: true,
+              },
+            ),
+            runGitStdoutWithOptions(
+              "GitCore.readRangeContext.diffStat",
+              cwd,
+              ["diff", "--stat", range],
+              {
+                maxOutputBytes: RANGE_DIFF_SUMMARY_MAX_OUTPUT_BYTES,
+                truncateOutputAtMaxBytes: true,
+              },
+            ),
+            runGitStdoutWithOptions(
+              "GitCore.readRangeContext.diffPatch",
+              cwd,
+              ["diff", "--patch", "--minimal", range],
+              {
+                maxOutputBytes: RANGE_DIFF_PATCH_MAX_OUTPUT_BYTES,
+                truncateOutputAtMaxBytes: true,
+              },
+            ),
           ],
           { concurrency: "unbounded" },
         );
diff --git a/apps/server/src/git/Services/GitCore.ts b/apps/server/src/git/Services/GitCore.ts
@@ -32,6 +32,7 @@ export interface ExecuteGitInput {
   readonly allowNonZeroExit?: boolean;
   readonly timeoutMs?: number;
   readonly maxOutputBytes?: number;
+  readonly truncateOutputAtMaxBytes?: boolean;
   readonly progress?: ExecuteGitProgress;
 }
 

Original file line number	Diff line number	Diff line change
`@@ -32,6 +32,7 @@ export interface ExecuteGitInput {`
`32`	`32`	`readonly allowNonZeroExit?: boolean;`
`33`	`33`	`readonly timeoutMs?: number;`
`34`	`34`	`readonly maxOutputBytes?: number;`
	`35`	`+ readonly truncateOutputAtMaxBytes?: boolean;`
`35`	`36`	`readonly progress?: ExecuteGitProgress;`
`36`	`37`	`}`
`37`	`38`