From da3debb08eb07dea474a3f4e9c3a9bedb7e696a6 Mon Sep 17 00:00:00 2001 From: Cole Bemis Date: Fri, 17 Nov 2023 18:25:39 -0800 Subject: [PATCH] Refactor file loading logic and add support for Git LFS --- src/components/file-preview.tsx | 43 ++++++++++++++++++------- src/global-state.ts | 2 +- src/utils/fs.ts | 18 +++++++++++ src/utils/git-lfs.ts | 57 +++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 src/utils/git-lfs.ts diff --git a/src/components/file-preview.tsx b/src/components/file-preview.tsx index f056f5b4..a820608e 100644 --- a/src/components/file-preview.tsx +++ b/src/components/file-preview.tsx @@ -1,8 +1,9 @@ import { useAtomValue } from "jotai" import React from "react" import { LoadingIcon16 } from "../components/icons" -import { githubRepoAtom, githubUserAtom } from "../global-state" -import { readRawFile } from "../utils/github-fs" +import { ROOT_DIR, githubRepoAtom, githubUserAtom } from "../global-state" +import { readFile } from "../utils/fs" +import { isTrackedWithGitLfs, resolveGitLfsPointer } from "../utils/git-lfs" export const fileCache = new Map() @@ -20,29 +21,49 @@ export function FilePreview({ path, alt = "" }: FilePreviewProps) { const [isLoading, setIsLoading] = React.useState(!cachedFile) React.useEffect(() => { - if (!file) { - loadFile() - } + // If file is already cached, don't fetch it again + if (file) return + + // Use ignore flag to avoid race conditions + // Reference: https://react.dev/reference/react/useEffect#fetching-data-with-effects + let ignore = false async function loadFile() { if (!githubUser || !githubRepo) return + console.log(githubUser, githubRepo) + try { setIsLoading(true) - const file = await readRawFile({ githubToken: githubUser.token, githubRepo, path }) - const url = URL.createObjectURL(file) + const file = await readFile(`${ROOT_DIR}${path}`) + + let url = "" - setFile(file) - setUrl(url) + // If file is tracked with Git LFS, resolve the pointer + if (await isTrackedWithGitLfs(file)) { + url = await resolveGitLfsPointer({ file, githubUser, githubRepo }) + } else { + url = URL.createObjectURL(file) + } - // Cache the file and base64 data - fileCache.set(path, { file, url }) + if (!ignore) { + setFile(file) + setUrl(url) + // Cache the file and its URL + fileCache.set(path, { file, url }) + } } catch (error) { console.error(error) } finally { setIsLoading(false) } } + + loadFile() + + return () => { + ignore = true + } }, [file, githubUser, githubRepo, path]) if (!file) { diff --git a/src/global-state.ts b/src/global-state.ts index 3eb54ea0..c32db3b5 100644 --- a/src/global-state.ts +++ b/src/global-state.ts @@ -23,7 +23,7 @@ import { removeTemplateFrontmatter } from "./utils/remove-template-frontmatter" // Constants // ----------------------------------------------------------------------------- -const ROOT_DIR = "/root" +export const ROOT_DIR = "/root" const DEFAULT_BRANCH = "main" const GITHUB_USER_KEY = "github_user" const MARKDOWN_FILES_KEY = "markdown_files" diff --git a/src/utils/fs.ts b/src/utils/fs.ts index 01419886..3a28a4f8 100644 --- a/src/utils/fs.ts +++ b/src/utils/fs.ts @@ -1,4 +1,5 @@ import LightningFS from "@isomorphic-git/lightning-fs" +import mime from "mime" const DB_NAME = "fs" @@ -10,3 +11,20 @@ export const fs = new LightningFS(DB_NAME) export function fsWipe() { window.indexedDB.deleteDatabase(DB_NAME) } + +/** + * The same as fs.promises.readFile(), + * but it returns a File object instead of string or Uint8Array + */ +export async function readFile(path: string) { + let content = await fs.promises.readFile(path) + + // If content is a string, convert it to a Uint8Array + if (typeof content === "string") { + content = new TextEncoder().encode(content) + } + + const mimeType = mime.getType(path) ?? "" + const filename = path.split("/").pop() ?? "" + return new File([content], filename, { type: mimeType }) +} diff --git a/src/utils/git-lfs.ts b/src/utils/git-lfs.ts new file mode 100644 index 00000000..4b5110f1 --- /dev/null +++ b/src/utils/git-lfs.ts @@ -0,0 +1,57 @@ +import { GitHubRepository, GitHubUser } from "../types" + +/** + * Check if a file is tracked with Git LFS by checking + * if the file is actually a Git LFS pointer + */ +// TODO: Use .gitattributes file to determine if file is tracked with Git LFS +export async function isTrackedWithGitLfs(file: File) { + const text = await file.text() + return text.startsWith("version https://git-lfs.github.com/spec/") +} + +/** Resolve a Git LFS pointer to a file URL */ +export async function resolveGitLfsPointer({ + file, + githubUser, + githubRepo, +}: { + file: File + githubUser: GitHubUser + githubRepo: GitHubRepository +}) { + const text = await file.text() + + // Parse the Git LFS pointer + const oid = text.match(/oid sha256:(?[a-f0-9]{64})/)?.groups?.oid + const size = text.match(/size (?\d+)/)?.groups?.size + + if (!oid || !size) { + throw new Error("Invalid Git LFS pointer") + } + + // Fetch the file URL from GitHub + // TODO: Use proxy to avoid CORS issues + const response = await fetch( + `https://github.com/${githubRepo.owner}/${githubRepo.name}.git/info/lfs/objects/batch`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/vnd.git-lfs+json", + Authorization: `Bearer ${githubUser.token}`, + }, + body: JSON.stringify({ + operation: "download", + transfers: ["basic"], + objects: [{ oid, size }], + }), + }, + ) + + const json = await response.json() + + console.log(json) + + return "" +}