Merge remote-tracking branch 'upstream/main' into feat/provider-usage-limits

Author: Aditya190803

apps/marketing/src/pages/download.astro

@@ -1,5 +1,6 @@
 ---
 import Layout from "../layouts/Layout.astro";
+import { RELEASES_URL } from "../lib/releases";
 ---
 
 <Layout title="Download — T3 Code" description="Download T3 Code for macOS, Windows, or Linux.">
@@ -17,11 +18,11 @@ import Layout from "../layouts/Layout.astro";
         <h2 class="platform-name">macOS</h2>
       </div>
       <div class="download-cards">
-        <a class="download-card" data-asset="arm64.dmg" href="#">
+        <a class="download-card" data-asset="arm64.dmg" href={RELEASES_URL}>
           <span class="card-arch">Apple Silicon (arm64)</span>
           <span class="card-format">.dmg</span>
         </a>
-        <a class="download-card" data-asset="x64.dmg" href="#">
+        <a class="download-card" data-asset="x64.dmg" href={RELEASES_URL}>
           <span class="card-arch">Intel (x64)</span>
           <span class="card-format">.dmg</span>
         </a>
@@ -35,7 +36,7 @@ import Layout from "../layouts/Layout.astro";
         <h2 class="platform-name">Windows</h2>
       </div>
       <div class="download-cards">
-        <a class="download-card" data-asset="x64.exe" href="#">
+        <a class="download-card" data-asset="x64.exe" href={RELEASES_URL}>
           <span class="card-arch">Windows 10, 11</span>
           <span class="card-format">.exe</span>
         </a>
@@ -49,7 +50,7 @@ import Layout from "../layouts/Layout.astro";
         <h2 class="platform-name">Linux</h2>
       </div>
       <div class="download-cards">
-        <a class="download-card" data-asset="x86_64.AppImage" href="#">
+        <a class="download-card" data-asset="x86_64.AppImage" href={RELEASES_URL}>
           <span class="card-arch">x86_64</span>
           <span class="card-format">AppImage</span>
         </a>

apps/server/integration/OrchestrationEngineHarness.integration.ts

@@ -49,7 +49,7 @@ import { CheckpointReactorLive } from "../src/orchestration/Layers/CheckpointRea
 import { OrchestrationEngineLive } from "../src/orchestration/Layers/OrchestrationEngine.ts";
 import { OrchestrationProjectionPipelineLive } from "../src/orchestration/Layers/ProjectionPipeline.ts";
 import { OrchestrationProjectionSnapshotQueryLive } from "../src/orchestration/Layers/ProjectionSnapshotQuery.ts";
-import { RuntimeReceiptBusLive } from "../src/orchestration/Layers/RuntimeReceiptBus.ts";
+import { RuntimeReceiptBusTest } from "../src/orchestration/Layers/RuntimeReceiptBus.ts";
 import { OrchestrationReactorLive } from "../src/orchestration/Layers/OrchestrationReactor.ts";
 import { ProviderCommandReactorLive } from "../src/orchestration/Layers/ProviderCommandReactor.ts";
 import { ProviderRuntimeIngestionLive } from "../src/orchestration/Layers/ProviderRuntimeIngestion.ts";
@@ -301,7 +301,7 @@ export const makeOrchestrationIntegrationHarness = (
       ProjectionPendingApprovalRepositoryLive,
       checkpointStoreLayer,
       providerLayer.pipe(Layer.provide(usageLimitsRepositoryLayer)),
-      RuntimeReceiptBusLive,
+      RuntimeReceiptBusTest,
     );
     const serverSettingsLayer = ServerSettingsService.layerTest();
     const runtimeIngestionLayer = ProviderRuntimeIngestionLive.pipe(
@@ -380,7 +380,7 @@ export const makeOrchestrationIntegrationHarness = (
       runtime.runPromise(reactor.start().pipe(Scope.provide(scope))),
     ).pipe(Effect.orDie);
     const receiptHistory = yield* Ref.make<ReadonlyArray<OrchestrationRuntimeReceipt>>([]);
-    yield* Stream.runForEach(runtimeReceiptBus.stream, (receipt) =>
+    yield* Stream.runForEach(runtimeReceiptBus.streamEventsForTest, (receipt) =>
       Ref.update(receiptHistory, (history) => [...history, receipt]).pipe(Effect.asVoid),
     ).pipe(Effect.forkIn(scope));
     yield* Effect.sleep(10);

apps/server/integration/providerService.integration.test.ts

@@ -91,6 +91,7 @@ const collectEventsDuring = <A, E, R>(
       Effect.forkScoped,
     );
 
+    yield* Effect.sleep("50 millis");
     yield* action;
 
     return yield* Effect.forEach(
@@ -109,7 +110,6 @@ const runTurn = (input: {
 }) =>
   Effect.gen(function* () {
     yield* input.harness.queueTurnResponse(input.threadId, input.response);
-
     return yield* collectEventsDuring(
       input.provider.streamEvents,
       input.response.events.length,
@@ -121,7 +121,7 @@ const runTurn = (input: {
     );
   });
 
-it.effect("replays typed runtime fixture events", () =>
+it.live("replays typed runtime fixture events", () =>
   Effect.gen(function* () {
     const fixture = yield* makeIntegrationFixture;
 
@@ -154,7 +154,7 @@ it.effect("replays typed runtime fixture events", () =>
   }).pipe(Effect.provide(NodeServices.layer)),
 );
 
-it.effect("replays file-changing fixture turn events", () =>
+it.live("replays file-changing fixture turn events", () =>
   Effect.gen(function* () {
     const fixture = yield* makeIntegrationFixture;
     const { join } = yield* Path.Path;
@@ -193,7 +193,7 @@ it.effect("replays file-changing fixture turn events", () =>
   }).pipe(Effect.provide(NodeServices.layer)),
 );
 
-it.effect("runs multi-turn tool/approval flow", () =>
+it.live("runs multi-turn tool/approval flow", () =>
   Effect.gen(function* () {
     const fixture = yield* makeIntegrationFixture;
     const { join } = yield* Path.Path;
@@ -247,7 +247,7 @@ it.effect("runs multi-turn tool/approval flow", () =>
   }).pipe(Effect.provide(NodeServices.layer)),
 );
 
-it.effect("rolls back provider conversation state only", () =>
+it.live("rolls back provider conversation state only", () =>
   Effect.gen(function* () {
     const fixture = yield* makeIntegrationFixture;
     const { join } = yield* Path.Path;

apps/server/src/http.ts

@@ -1,6 +1,15 @@
 import Mime from "@effect/platform-node/Mime";
-import { Effect, FileSystem, Option, Path } from "effect";
-import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
+import { Data, Effect, FileSystem, Layer, Option, Path } from "effect";
+import { cast } from "effect/Function";
+import {
+  HttpBody,
+  HttpClient,
+  HttpClientResponse,
+  HttpRouter,
+  HttpServerResponse,
+  HttpServerRequest,
+} from "effect/unstable/http";
+import { OtlpTracer } from "effect/unstable/observability";
 
 import {
   ATTACHMENTS_ROUTE_PREFIX,
@@ -9,10 +18,74 @@ import {
 } from "./attachmentPaths";
 import { resolveAttachmentPathById } from "./attachmentStore";
 import { ServerConfig } from "./config";
+import { decodeOtlpTraceRecords } from "./observability/TraceRecord.ts";
+import { BrowserTraceCollector } from "./observability/Services/BrowserTraceCollector.ts";
 import { ProjectFaviconResolver } from "./project/Services/ProjectFaviconResolver";
 
 const PROJECT_FAVICON_CACHE_CONTROL = "public, max-age=3600";
 const FALLBACK_PROJECT_FAVICON_SVG = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="24" height="24" fill="none" stroke="#6b728080" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" data-fallback="project-favicon"><path d="M20 20a2 2 0 0 0 2-2V8a2 2 0 0 0-2-2h-8l-2-2H4a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2Z"/></svg>`;
+const OTLP_TRACES_PROXY_PATH = "/api/observability/v1/traces";
+
+class DecodeOtlpTraceRecordsError extends Data.TaggedError("DecodeOtlpTraceRecordsError")<{
+  readonly cause: unknown;
+  readonly bodyJson: OtlpTracer.TraceData;
+}> {}
+
+export const otlpTracesProxyRouteLayer = HttpRouter.add(
+  "POST",
+  OTLP_TRACES_PROXY_PATH,
+  Effect.gen(function* () {
+    const request = yield* HttpServerRequest.HttpServerRequest;
+    const config = yield* ServerConfig;
+    const otlpTracesUrl = config.otlpTracesUrl;
+    const browserTraceCollector = yield* BrowserTraceCollector;
+    const httpClient = yield* HttpClient.HttpClient;
+    const bodyJson = cast<unknown, OtlpTracer.TraceData>(yield* request.json);
+
+    yield* Effect.try({
+      try: () => decodeOtlpTraceRecords(bodyJson),
+      catch: (cause) => new DecodeOtlpTraceRecordsError({ cause, bodyJson }),
+    }).pipe(
+      Effect.flatMap((records) => browserTraceCollector.record(records)),
+      Effect.catch((cause) =>
+        Effect.logWarning("Failed to decode browser OTLP traces", {
+          cause,
+          bodyJson,
+        }),
+      ),
+    );
+
+    if (otlpTracesUrl === undefined) {
+      return HttpServerResponse.empty({ status: 204 });
+    }
+
+    return yield* httpClient
+      .post(otlpTracesUrl, {
+        body: HttpBody.jsonUnsafe(bodyJson),
+      })
+      .pipe(
+        Effect.flatMap(HttpClientResponse.filterStatusOk),
+        Effect.as(HttpServerResponse.empty({ status: 204 })),
+        Effect.tapError((cause) =>
+          Effect.logWarning("Failed to export browser OTLP traces", {
+            cause,
+            otlpTracesUrl,
+          }),
+        ),
+        Effect.catch(() =>
+          Effect.succeed(HttpServerResponse.text("Trace export failed.", { status: 502 })),
+        ),
+      );
+  }),
+).pipe(
+  Layer.provide(
+    HttpRouter.cors({
+      allowedMethods: ["POST", "OPTIONS"],
+      allowedHeaders: ["content-type"],
+      maxAge: 600,
+    }),
+  ),
+);
 
 export const attachmentsRouteLayer = HttpRouter.add(
   "GET",

apps/server/src/observability/Layers/Observability.ts

@@ -4,6 +4,8 @@ import { OtlpMetrics, OtlpSerialization, OtlpTracer } from "effect/unstable/obse
 import { ServerConfig } from "../../config.ts";
 import { ServerLoggerLive } from "../../serverLogger.ts";
 import { makeLocalFileTracer } from "../LocalFileTracer.ts";
+import { BrowserTraceCollector } from "../Services/BrowserTraceCollector.ts";
+import { makeTraceSink } from "../TraceSink.ts";
 
 const otlpSerializationLayer = OtlpSerialization.layerJson;
 
@@ -16,9 +18,14 @@ export const ObservabilityLive = Layer.unwrap(
       Layer.succeed(References.TracerTimingEnabled, config.traceTimingEnabled),
     );
 
-    const tracerLayer = Layer.effect(
-      Tracer.Tracer,
+    const tracerLayer = Layer.unwrap(
       Effect.gen(function* () {
+        const sink = yield* makeTraceSink({
+          filePath: config.serverTracePath,
+          maxBytes: config.traceMaxBytes,
+          maxFiles: config.traceMaxFiles,
+          batchWindowMs: config.traceBatchWindowMs,
+        });
         const delegate =
           config.otlpTracesUrl === undefined
             ? undefined
@@ -34,13 +41,26 @@ export const ObservabilityLive = Layer.unwrap(
                 },
               });
 
-        return yield* makeLocalFileTracer({
+        const tracer = yield* makeLocalFileTracer({
           filePath: config.serverTracePath,
           maxBytes: config.traceMaxBytes,
           maxFiles: config.traceMaxFiles,
           batchWindowMs: config.traceBatchWindowMs,
+          sink,
           ...(delegate ? { delegate } : {}),
         });
+
+        return Layer.mergeAll(
+          Layer.succeed(Tracer.Tracer, tracer),
+          Layer.succeed(BrowserTraceCollector, {
+            record: (records) =>
+              Effect.sync(() => {
+                for (const record of records) {
+                  sink.push(record);
+                }
+              }),
+          }),
+        );
       }),
     ).pipe(Layer.provideMerge(otlpSerializationLayer));
 

apps/server/src/observability/LocalFileTracer.test.ts

@@ -5,7 +5,7 @@ import path from "node:path";
 import { assert, describe, it } from "@effect/vitest";
 import { Effect, Layer, Logger, References, Tracer } from "effect";
 
-import type { TraceRecord } from "./TraceRecord.ts";
+import type { EffectTraceRecord } from "./TraceRecord.ts";
 import { makeLocalFileTracer } from "./LocalFileTracer.ts";
 
 const makeTestLayer = (tracePath: string) =>
@@ -23,13 +23,13 @@ const makeTestLayer = (tracePath: string) =>
     Layer.succeed(References.MinimumLogLevel, "Info"),
   );
 
-const readTraceRecords = (tracePath: string): Array<TraceRecord> =>
+const readTraceRecords = (tracePath: string): Array<EffectTraceRecord> =>
   fs
     .readFileSync(tracePath, "utf8")
     .trim()
     .split("\n")
     .filter((line) => line.length > 0)
-    .map((line) => JSON.parse(line) as TraceRecord);
+    .map((line) => JSON.parse(line) as EffectTraceRecord);
 
 describe("LocalFileTracer", () => {
   it.effect("writes nested spans to disk and captures log messages as span events", () =>

apps/server/src/observability/LocalFileTracer.ts

@@ -1,15 +1,16 @@
 import type * as Exit from "effect/Exit";
 import { Effect, Option, Tracer } from "effect";
 
-import { spanToTraceRecord } from "./TraceRecord.ts";
-import { makeTraceSink } from "./TraceSink.ts";
+import { EffectTraceRecord, spanToTraceRecord } from "./TraceRecord.ts";
+import { makeTraceSink, type TraceSink } from "./TraceSink.ts";
 
 export interface LocalFileTracerOptions {
   readonly filePath: string;
   readonly maxBytes: number;
   readonly maxFiles: number;
   readonly batchWindowMs: number;
   readonly delegate?: Tracer.Tracer;
+  readonly sink?: TraceSink;
 }
 
 class LocalFileSpan implements Tracer.Span {
@@ -30,7 +31,7 @@ class LocalFileSpan implements Tracer.Span {
   constructor(
     options: Parameters<Tracer.Tracer["span"]>[0],
     private readonly delegate: Tracer.Span,
-    private readonly push: (record: ReturnType<typeof spanToTraceRecord>) => void,
+    private readonly push: (record: EffectTraceRecord) => void,
   ) {
     this.name = delegate.name;
     this.spanId = delegate.spanId;
@@ -82,12 +83,14 @@ class LocalFileSpan implements Tracer.Span {
 export const makeLocalFileTracer = Effect.fn("makeLocalFileTracer")(function* (
   options: LocalFileTracerOptions,
 ) {
-  const sink = yield* makeTraceSink({
-    filePath: options.filePath,
-    maxBytes: options.maxBytes,
-    maxFiles: options.maxFiles,
-    batchWindowMs: options.batchWindowMs,
-  });
+  const sink =
+    options.sink ??
+    (yield* makeTraceSink({
+      filePath: options.filePath,
+      maxBytes: options.maxBytes,
+      maxFiles: options.maxFiles,
+      batchWindowMs: options.batchWindowMs,
+    }));
 
   const delegate =
     options.delegate ??

apps/server/src/observability/Services/BrowserTraceCollector.ts

@@ -0,0 +1,13 @@
+import { ServiceMap } from "effect";
+import type { Effect } from "effect";
+
+import type { TraceRecord } from "../TraceRecord.ts";
+
+export interface BrowserTraceCollectorShape {
+  readonly record: (records: ReadonlyArray<TraceRecord>) => Effect.Effect<void>;
+}
+
+export class BrowserTraceCollector extends ServiceMap.Service<
+  BrowserTraceCollector,
+  BrowserTraceCollectorShape
+>()("t3/observability/Services/BrowserTraceCollector") {}