Skip to content

Commit 6886c51

Browse files
authored
fix: fixed file upload error & tag query causing slow load times (#134)
1 parent 8508599 commit 6886c51

21 files changed

Lines changed: 519 additions & 268 deletions

File tree

apps/server/package.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@
1313
"build": "bun run check && bun build src/index.ts --target bun --minify --outdir dist",
1414
"start": "bun dist/index.js",
1515
"cf:dev": "sudo wrangler dev",
16-
"cf:deploy:preview": "wrangler deploy --env preview",
17-
"cf:deploy:staging": "wrangler deploy --env staging",
18-
"cf:deploy:production": "wrangler deploy --env production",
16+
"cf:deploy:preview": "wrangler deploy --env preview --keep-vars",
17+
"cf:deploy:staging": "wrangler deploy --env staging --keep-vars",
18+
"cf:deploy:production": "wrangler deploy --env production --keep-vars",
1919
"sync-wrangler-secrets": "bun run scripts/sync-wrangler-secrets.ts",
2020
"docker:build": "bun run build && docker build -t nimbus-server-manual .",
2121
"docker:run": "source .env && docker run --name nimbus-server-manual --env-file .env -p $SERVER_PORT:$SERVER_PORT nimbus-server-manual:latest",

apps/server/src/index.ts

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { DRIVE_PROVIDER_HEADERS } from "@nimbus/shared";
22
import { contextStorage } from "hono/context-storage";
33
import { createPublicRouter } from "./hono";
44
import { ContextManager } from "./context";
5+
import { timeout } from "hono/timeout";
56
import { cors } from "hono/cors";
67
import routes from "./routes";
78

@@ -23,22 +24,26 @@ const app = createPublicRouter()
2324
maxAge: 43200, // 12 hours
2425
})
2526
)
26-
.use("*", async (c, next) => {
27-
const env = c.var.env;
28-
const { db, redisClient, auth } = await c.var.contextManager.createContext();
29-
c.set("db", db);
30-
c.set("redisClient", redisClient);
31-
c.set("auth", auth);
32-
try {
33-
await next();
34-
} finally {
35-
// WARNING: make sure to add WRANGLER_DEV to .dev.vars for wrangler dev
36-
// for local dev, always keep context open UNLESS wrangler dev, close context
37-
if (env.IS_EDGE_RUNTIME && (env.NODE_ENV === "production" || env.WRANGLER_DEV)) {
38-
await c.var.contextManager.close();
27+
.use(
28+
"*",
29+
async (c, next) => {
30+
const env = c.var.env;
31+
const { db, redisClient, auth } = await c.var.contextManager.createContext();
32+
c.set("db", db);
33+
c.set("redisClient", redisClient);
34+
c.set("auth", auth);
35+
try {
36+
await next();
37+
} finally {
38+
// WARNING: make sure to add WRANGLER_DEV to .dev.vars for wrangler dev
39+
// for local dev, always keep context open UNLESS wrangler dev, close context
40+
if (env.IS_EDGE_RUNTIME && (env.NODE_ENV === "production" || env.WRANGLER_DEV)) {
41+
await c.var.contextManager.close();
42+
}
3943
}
40-
}
41-
})
44+
},
45+
timeout(5000)
46+
)
4247
.get("/kamehame", c => c.text("HAAAAAAAAAAAAAA"))
4348
.route("/api", routes);
4449

apps/server/src/providers/google/google-drive-provider.ts

Lines changed: 79 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,11 @@ import { Readable } from "node:stream";
1919
export class GoogleDriveProvider implements Provider {
2020
private drive: drive_v3.Drive;
2121
private accessToken: string;
22+
private isEdgeRuntime: boolean;
2223

23-
constructor(accessToken: string) {
24+
constructor(accessToken: string, isEdgeRuntime: boolean = false) {
2425
this.accessToken = accessToken;
26+
this.isEdgeRuntime = isEdgeRuntime;
2527
const oauth2Client = new OAuth2Client();
2628
oauth2Client.setCredentials({ access_token: accessToken });
2729
this.drive = new drive_v3.Drive({
@@ -57,17 +59,82 @@ export class GoogleDriveProvider implements Provider {
5759
// fields: this.getFileFields(),
5860
});
5961
} else {
60-
// For files with content
61-
const media = {
62-
mimeType,
63-
body: this.normalizeContent(content),
64-
};
65-
66-
response = await this.drive.files.create({
67-
requestBody: fileMetadata,
68-
media,
69-
// fields: this.getFileFields(),
70-
});
62+
// ! Note: when attempting to use the sdk, cloudflare envrionments error on upload. Using the api via fetch directly allows for uploads on cloudflare environments
63+
// ! Also, this code was made with AI
64+
if (this.isEdgeRuntime) {
65+
const initRes = await fetch("https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable", {
66+
method: "POST",
67+
headers: {
68+
Authorization: `Bearer ${this.accessToken}`,
69+
"Content-Type": "application/json; charset=UTF-8",
70+
},
71+
body: JSON.stringify(fileMetadata),
72+
});
73+
74+
if (!initRes.ok) {
75+
const details = await initRes.text().catch(() => initRes.statusText);
76+
throw new Error(`Failed to initiate resumable upload. Status: ${initRes.status}. ${details}`);
77+
}
78+
79+
const uploadUrl = initRes.headers.get("Location");
80+
if (!uploadUrl) {
81+
throw new Error("Resumable upload URL was not returned by Google Drive API");
82+
}
83+
84+
let contentBuffer: Uint8Array;
85+
if (typeof Buffer !== "undefined" && Buffer.isBuffer(content)) {
86+
contentBuffer = new Uint8Array(content as Buffer);
87+
} else if (content instanceof Uint8Array) {
88+
contentBuffer = content;
89+
} else {
90+
const chunks: Uint8Array[] = [];
91+
for await (const chunk of content as NodeJS.ReadableStream) {
92+
chunks.push(typeof chunk === "string" ? new TextEncoder().encode(chunk) : new Uint8Array(chunk));
93+
}
94+
const total = chunks.reduce((n, c) => n + c.byteLength, 0);
95+
contentBuffer = new Uint8Array(total);
96+
let offset = 0;
97+
for (const c of chunks) {
98+
contentBuffer.set(c, offset);
99+
offset += c.byteLength;
100+
}
101+
}
102+
103+
// Create a concrete ArrayBuffer to avoid SharedArrayBuffer typing issues
104+
const arrayBufferForBody = (() => {
105+
const ab = new ArrayBuffer(contentBuffer.byteLength);
106+
new Uint8Array(ab).set(contentBuffer);
107+
return ab;
108+
})();
109+
110+
const uploadRes = await fetch(uploadUrl, {
111+
method: "PUT",
112+
headers: {
113+
"Content-Type": mimeType,
114+
"Content-Length": String(contentBuffer.byteLength),
115+
},
116+
body: arrayBufferForBody,
117+
});
118+
119+
if (!uploadRes.ok) {
120+
const details = await uploadRes.text().catch(() => uploadRes.statusText);
121+
throw new Error(`Resumable upload failed. Status: ${uploadRes.status}. ${details}`);
122+
}
123+
124+
const created: drive_v3.Schema$File = await uploadRes.json();
125+
return created ? this.mapToFile(created) : null;
126+
} else {
127+
const media = {
128+
mimeType,
129+
body: this.normalizeContent(content),
130+
};
131+
132+
response = await this.drive.files.create({
133+
requestBody: fileMetadata,
134+
media,
135+
// fields: this.getFileFields(),
136+
});
137+
}
71138
}
72139

73140
return response.data ? this.mapToFile(response.data) : null;

apps/server/src/routes/drives/index.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ const drivesRouter = createDriveProviderRouter()
4343

4444
// Check if file is already pinned for this account
4545
const firstResult = await c.var.db.query.pinnedFile.findFirst({
46-
where: (table, { eq }) => eq(table.userId, user.id),
46+
where: (table, { eq }) => eq(table.fileId, fileId),
4747
});
4848

4949
if (firstResult) {

apps/server/src/routes/files/file-service.ts

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ import type {
55
GetFileByIdSchema,
66
GetFilesSchema,
77
UpdateFileSchema,
8+
Tag,
89
} from "@nimbus/shared";
910
import { getDriveProviderContext } from "../../hono";
1011
import { TagService } from "../tags/tag-service";
11-
import type { Readable } from "node:stream";
1212

1313
interface CreateFileOptions {
1414
name: string;
@@ -43,19 +43,20 @@ export class FileService {
4343
return null;
4444
}
4545

46-
// Add tags to files, handling any tag retrieval failures
47-
const filesWithTags = await Promise.all(
48-
res.items.map(async item => {
49-
if (!item.id) return { ...item, tags: [] };
50-
try {
51-
const tags = await this.tagService.getFileTags(item.id, user.id);
52-
return { ...item, tags };
53-
} catch (error) {
54-
console.error(`Failed to get tags for file ${item.id}:`, error);
55-
return { ...item, tags: [] };
56-
}
57-
})
58-
);
46+
// Batch load tags for files to avoid N parallel queries
47+
const fileIds = res.items.map(i => i.id).filter((id): id is string => Boolean(id));
48+
let tagsByFileId: Record<string, Tag[]> = {};
49+
try {
50+
// Lazy import type to avoid circular import issues at top
51+
tagsByFileId = await this.tagService.getTagsByFileIds(fileIds, user.id);
52+
} catch (error) {
53+
console.error("Failed to batch get tags for files:", error);
54+
}
55+
56+
const filesWithTags = res.items.map(item => {
57+
const tags = item.id ? (tagsByFileId[item.id] ?? []) : [];
58+
return { ...item, tags };
59+
});
5960

6061
return filesWithTags as File[];
6162
}
@@ -83,7 +84,7 @@ export class FileService {
8384
return drive.delete(options.fileId);
8485
}
8586

86-
async createFile(options: CreateFileOptions, fileStream?: Readable) {
87+
async createFile(options: CreateFileOptions, fileStream?: Buffer<ArrayBuffer>) {
8788
const drive = this.c.var.provider;
8889
return drive.create(options, fileStream);
8990
}

apps/server/src/routes/files/index.ts

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -91,22 +91,18 @@ const filesRouter = createDriveProviderRouter()
9191
});
9292
}
9393

94-
// Convert File to Readable stream for upload
9594
const arrayBuffer = await file.arrayBuffer();
9695
const fileBuffer = Buffer.from(arrayBuffer);
97-
const readableStream = new Readable();
98-
readableStream.push(fileBuffer);
99-
readableStream.push(null); // Signal end of stream
10096

10197
// Upload with timeout
10298
const UPLOAD_TIMEOUT = 5 * 60 * 1000;
10399
const uploadPromise = fileService.createFile(
104100
{
105101
name: file.name,
106-
mimeType: file.type,
102+
mimeType: file.type || "application/octet-stream",
107103
parentId,
108104
},
109-
readableStream
105+
fileBuffer
110106
);
111107

112108
const uploadedFile = await Promise.race([

apps/server/src/routes/index.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ const driveProviderRouter = createDriveProviderRouter()
8383
}
8484

8585
if (parsedProviderName.data === "google") {
86-
provider = new GoogleDriveProvider(accessToken);
86+
provider = new GoogleDriveProvider(accessToken, c.var.env.IS_EDGE_RUNTIME);
8787
} else if (parsedProviderName.data === "microsoft") {
8888
provider = new OneDriveProvider(accessToken);
8989
} else if (parsedProviderName.data === "box") {

apps/server/src/routes/tags/tag-service.ts

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,48 @@ export class TagService {
261261
}));
262262
}
263263

264+
// Get tags for multiple files in one batched query
265+
async getTagsByFileIds(fileIds: string[], userId: string): Promise<Record<string, Tag[]>> {
266+
if (fileIds.length === 0) return {};
267+
268+
// Fetch all file->tag associations for these files
269+
const associations = await this.c.var.db.query.fileTag.findMany({
270+
where: (table, { and, inArray, eq }) => and(inArray(table.fileId, fileIds), eq(table.userId, userId)),
271+
});
272+
273+
if (associations.length === 0) return {};
274+
275+
// Fetch all tags referenced by these associations
276+
const uniqueTagIds = Array.from(new Set(associations.map(a => a.tagId)));
277+
const tags = await this.c.var.db.query.tag.findMany({
278+
where: (table, { and, inArray, eq }) => and(inArray(table.id, uniqueTagIds), eq(table.userId, userId)),
279+
});
280+
281+
// Map tagId -> Tag
282+
const tagById = new Map<string, Tag>(
283+
tags.map(t => [
284+
t.id,
285+
{
286+
...t,
287+
parentId: t.parentId || undefined,
288+
createdAt: t.createdAt.toISOString(),
289+
updatedAt: t.updatedAt.toISOString(),
290+
} as Tag,
291+
])
292+
);
293+
294+
// Build fileId -> Tag[] map
295+
const result: Record<string, Tag[]> = {};
296+
for (const assoc of associations) {
297+
const t = tagById.get(assoc.tagId);
298+
if (!t) continue;
299+
const list = result[assoc.fileId] ?? (result[assoc.fileId] = []);
300+
list.push(t);
301+
}
302+
303+
return result;
304+
}
305+
264306
// Get all child tag IDs recursively
265307
private async getAllChildTagIds(parentId: string, userId: string): Promise<string[]> {
266308
const childTags = await this.c.var.db.query.tag.findMany({

apps/web/package.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@
1010
"start": "bun run .next/build-start-folder/apps/web/server.js",
1111
"cf:build": "opennextjs-cloudflare build",
1212
"cf:preview": "opennextjs-cloudflare preview",
13-
"cf:deploy:preview": "opennextjs-cloudflare deploy --env preview",
14-
"cf:deploy:staging": "opennextjs-cloudflare deploy --env staging",
15-
"cf:deploy:production": "opennextjs-cloudflare deploy --env production",
13+
"cf:deploy:preview": "opennextjs-cloudflare deploy --env preview --keep-vars",
14+
"cf:deploy:staging": "opennextjs-cloudflare deploy --env staging --keep-vars",
15+
"cf:deploy:production": "opennextjs-cloudflare deploy --env production --keep-vars",
1616
"cf:typegen": "wrangler types --env-interface CloudflareEnv cloudflare-env.d.ts",
1717
"docker:build": "bun run build && docker build -t nimbus-web-manual .",
1818
"docker:run": "source .env && docker run --name nimbus-web-manual --env-file .env -p $WEB_PORT:$WEB_PORT nimbus-web-manual:latest",

apps/web/src/app/(protected)/(dashboard)/dashboard/[providerSlug]/[accountId]/page.tsx

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
"use client";
22

3+
import SmallScreenError from "@/components/dashboard/screen-size-error";
34
import { FileTable } from "@/components/dashboard/file-browser";
45
import { useGetFiles } from "@/hooks/useFileOperations";
56
import { Header } from "@/components/dashboard/header";
7+
import { Suspense, useEffect, useState } from "react";
68
import { useSearchParams } from "next/navigation";
7-
import { Suspense } from "react";
89

910
export default function DrivePage() {
1011
const searchParams = useSearchParams();
@@ -18,6 +19,22 @@ export default function DrivePage() {
1819
returnedValues: ["id", "name", "mimeType", "size", "modifiedTime", "webContentLink", "webViewLink"],
1920
});
2021

22+
// This prevents screens that are too small from using the app
23+
const [isSmallScreen, setIsSmallScreen] = useState(false);
24+
25+
useEffect(() => {
26+
function handleResize() {
27+
setIsSmallScreen(window.innerWidth < 700);
28+
}
29+
handleResize();
30+
window.addEventListener("resize", handleResize);
31+
return () => window.removeEventListener("resize", handleResize);
32+
}, []);
33+
34+
if (isSmallScreen) {
35+
return <SmallScreenError />;
36+
}
37+
2138
return (
2239
<>
2340
<Suspense fallback={null}>

0 commit comments

Comments
 (0)