From 6d9eb43a62c1a02aa45a176ea514bb0f3f85a8b6 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Mon, 26 Aug 2024 20:27:41 -0700 Subject: [PATCH 1/5] file.size --- docs/convert.md | 1 + docs/files.md | 4 +- package.json | 2 +- src/client/stdlib/fileAttachment.js | 19 ++++-- src/client/stdlib/inputs.js | 61 ++++++++++++++++++- src/dataloader.ts | 10 +++ src/javascript/module.ts | 7 ++- src/preview.ts | 17 +++++- src/render.ts | 16 +++-- test/build-test.ts | 2 +- test/javascript/module-test.ts | 6 +- test/output/build/archives.posix/tar.html | 8 +-- test/output/build/archives.posix/zip.html | 4 +- test/output/build/fetches/foo.html | 4 +- test/output/build/fetches/top.html | 8 +-- test/output/build/files/files.html | 6 +- .../build/files/subsection/subfiles.html | 4 +- test/output/build/imports/foo/foo.html | 2 +- test/output/build/multi/index.html | 4 +- test/output/build/simple/simple.html | 2 +- 20 files changed, 140 insertions(+), 47 deletions(-) diff --git a/docs/convert.md b/docs/convert.md index 6289bf260..db194acfb 100644 --- a/docs/convert.md +++ b/docs/convert.md @@ -341,6 +341,7 @@ The Framework standard library also includes several new methods that are not av Framework’s [`FileAttachment`](./files) includes a few new features: - `file.href` +- `file.size` - `file.lastModified` - `file.mimeType` is always defined - `file.text` now supports an `encoding` option diff --git a/docs/files.md b/docs/files.md index ceeb585ef..8b66f2991 100644 --- a/docs/files.md +++ b/docs/files.md @@ -10,7 +10,7 @@ Load files — whether static or generated dynamically by a [data loader](./load import {FileAttachment} from "npm:@observablehq/stdlib"; ``` -The `FileAttachment` function takes a path and returns a file handle. This handle exposes the file’s name, [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types), and modification time (represented as the number of milliseconds since UNIX epoch). +The `FileAttachment` function takes a path and returns a file handle. This handle exposes the file’s name, [MIME type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types), size in bytes , and modification time (represented as the number of milliseconds since UNIX epoch). ```js echo FileAttachment("volcano.json") @@ -52,7 +52,7 @@ const frames = [ None of the files in `frames` above are loaded until a [content method](#supported-formats) is invoked, for example by saying `frames[0].image()`. -For missing files, `file.lastModified` is undefined. The `file.mimeType` is determined by checking the file extension against the [`mime-db` media type database](https://github.com/jshttp/mime-db); it defaults to `application/octet-stream`. +For missing files, `file.size` and `file.lastModified` are undefined. The `file.mimeType` is determined by checking the file extension against the [`mime-db` media type database](https://github.com/jshttp/mime-db); it defaults to `application/octet-stream`. ## Supported formats diff --git a/package.json b/package.json index 162c1e202..297e8528d 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ }, "dependencies": { "@clack/prompts": "^0.7.0", - "@observablehq/inputs": "^0.11.0", + "@observablehq/inputs": "^0.12.0", "@observablehq/runtime": "^5.9.4", "@rollup/plugin-commonjs": "^25.0.7", "@rollup/plugin-json": "^6.1.0", diff --git a/src/client/stdlib/fileAttachment.js b/src/client/stdlib/fileAttachment.js index 6466d1cc1..a36920597 100644 --- a/src/client/stdlib/fileAttachment.js +++ b/src/client/stdlib/fileAttachment.js @@ -5,8 +5,14 @@ export function registerFile(name, info) { if (info == null) { files.delete(href); } else { - const {path, mimeType, lastModified} = info; - const file = new FileAttachmentImpl(new URL(path, location).href, name.split("/").pop(), mimeType, lastModified); + const {path, mimeType, lastModified, size} = info; + const file = new FileAttachmentImpl( + new URL(path, location).href, + name.split("/").pop(), + mimeType, + lastModified, + size + ); files.set(href, file); } } @@ -25,11 +31,12 @@ async function remote_fetch(file) { } export class AbstractFile { - constructor(name, mimeType = "application/octet-stream", lastModified) { + constructor(name, mimeType = "application/octet-stream", lastModified, size) { Object.defineProperties(this, { name: {value: `${name}`, enumerable: true}, mimeType: {value: `${mimeType}`, enumerable: true}, - lastModified: {value: +lastModified, enumerable: true} + lastModified: {value: +lastModified, enumerable: true}, + size: {value: +size, enumerable: true} }); } async blob() { @@ -131,8 +138,8 @@ export class AbstractFile { } class FileAttachmentImpl extends AbstractFile { - constructor(href, name, mimeType, lastModified) { - super(name, mimeType, lastModified); + constructor(href, name, mimeType, lastModified, size) { + super(name, mimeType, lastModified, size); Object.defineProperty(this, "href", {value: href}); } async url() { diff --git a/src/client/stdlib/inputs.js b/src/client/stdlib/inputs.js index d03fb8a51..21de73dba 100644 --- a/src/client/stdlib/inputs.js +++ b/src/client/stdlib/inputs.js @@ -1,5 +1,60 @@ -import {fileOf} from "@observablehq/inputs"; +import {file as _file} from "@observablehq/inputs"; import {AbstractFile} from "npm:@observablehq/stdlib"; -export * from "@observablehq/inputs"; -export const file = fileOf(AbstractFile); +export { + button, + checkbox, + radio, + toggle, + color, + date, + datetime, + form, + range, + number, + search, + searchFilter, + select, + table, + text, + email, + tel, + url, + password, + textarea, + input, + bind, + disposal, + formatDate, + formatLocaleAuto, + formatLocaleNumber, + formatTrim, + formatAuto, + formatNumber +} from "@observablehq/inputs"; + +export const file = (options) => _file({...options, transform: localFile}); + +function localFile(file) { + return new LocalFile(file); +} + +class LocalFile extends AbstractFile { + constructor(file) { + super(file.name, file.type, file.lastModified, file.size); + Object.defineProperty(this, "_", {value: file}); + Object.defineProperty(this, "_url", {writable: true}); + } + get href() { + return (this._url ??= URL.createObjectURL(this._)); + } + async url() { + return this.href; + } + async blob() { + return this._; + } + async stream() { + return this._.stream(); + } +} diff --git a/src/dataloader.ts b/src/dataloader.ts index dc86d95a2..8d8843863 100644 --- a/src/dataloader.ts +++ b/src/dataloader.ts @@ -183,11 +183,21 @@ export class LoaderResolver { return entry && Math.floor(entry.mtimeMs); } + getSourceSize(name: string): number | undefined { + const entry = getFileInfo(this.root, this.getSourceFilePath(name)); + return entry && Math.floor(entry.size); + } + getOutputLastModified(name: string): number | undefined { const entry = getFileInfo(this.root, this.getOutputFilePath(name)); return entry && Math.floor(entry.mtimeMs); } + getOutputSize(name: string): number | undefined { + const entry = getFileInfo(this.root, this.getOutputFilePath(name)); + return entry && Math.floor(entry.size); + } + resolveFilePath(path: string): string { return `/${join("_file", path)}?sha=${this.getSourceFileHash(path)}`; } diff --git a/src/javascript/module.ts b/src/javascript/module.ts index a96ac0406..d3668f15d 100644 --- a/src/javascript/module.ts +++ b/src/javascript/module.ts @@ -16,6 +16,8 @@ import {parseProgram} from "./parse.js"; export type FileInfo = { /** The last-modified time of the file; used to invalidate the cache. */ mtimeMs: number; + /** The size of the file in bytes. */ + size: number; /** The SHA-256 content hash of the file contents. */ hash: string; }; @@ -186,11 +188,12 @@ export function getFileHash(root: string, path: string): string { export function getFileInfo(root: string, path: string): FileInfo | undefined { const key = join(root, path); let mtimeMs: number; + let size: number; try { const stat = statSync(key); if (!stat.isFile()) return; // ignore non-files accessSync(key, constants.R_OK); // verify that file is readable - ({mtimeMs} = stat); + ({mtimeMs, size} = stat); } catch { fileInfoCache.delete(key); // delete stale entry return; // ignore missing, non-readable file @@ -199,7 +202,7 @@ export function getFileInfo(root: string, path: string): FileInfo | undefined { if (!entry || entry.mtimeMs < mtimeMs) { const contents = readFileSync(key); const hash = createHash("sha256").update(contents).digest("hex"); - fileInfoCache.set(key, (entry = {mtimeMs, hash})); + fileInfoCache.set(key, (entry = {mtimeMs, size, hash})); } return entry; } diff --git a/src/preview.ts b/src/preview.ts index 46a7d1fea..98a3a1b48 100644 --- a/src/preview.ts +++ b/src/preview.ts @@ -354,7 +354,12 @@ function handleWatch(socket: WebSocket, req: IncomingMessage, configPromise: Pro type: "update", html: diffHtml(previousHtml, html), code: diffCode(previousCode, code), - files: diffFiles(previousFiles, files, getLastModifiedResolver(loaders, path)), + files: diffFiles( + previousFiles, + files, + getLastModifiedResolver(loaders, path), + getSizeResolver(loaders, path) + ), tables: diffTables(previousTables, tables, previousFiles, files), stylesheets: diffStylesheets(previousStylesheets, stylesheets), hash: {previous: previousHash, current: hash} @@ -486,13 +491,14 @@ function diffCode(oldCode: Map, newCode: Map): C return patch; } -type FileDeclaration = {name: string; mimeType: string; lastModified: number; path: string}; +type FileDeclaration = {name: string; mimeType: string; lastModified: number; size: number; path: string}; type FilePatch = {removed: string[]; added: FileDeclaration[]}; function diffFiles( oldFiles: Map, newFiles: Map, - getLastModified: (name: string) => number | undefined + getLastModified: (name: string) => number | undefined, + getSize: (name: string) => number | undefined ): FilePatch { const patch: FilePatch = {removed: [], added: []}; for (const [name, path] of oldFiles) { @@ -506,6 +512,7 @@ function diffFiles( name, mimeType: mime.getType(name) ?? "application/octet-stream", lastModified: getLastModified(name) ?? NaN, + size: getSize(name) ?? NaN, path }); } @@ -517,6 +524,10 @@ function getLastModifiedResolver(loaders: LoaderResolver, path: string): (name: return (name) => loaders.getSourceLastModified(resolvePath(path, name)); } +function getSizeResolver(loaders: LoaderResolver, path: string): (name: string) => number | undefined { + return (name) => loaders.getSourceSize(resolvePath(path, name)); +} + type TableDeclaration = {name: string; path: string}; type TablePatch = {removed: string[]; added: TableDeclaration[]}; diff --git a/src/render.ts b/src/render.ts index 937dad9e7..0bbdff2fc 100644 --- a/src/render.ts +++ b/src/render.ts @@ -69,7 +69,10 @@ import ${preview || page.code.length ? `{${preview ? "open, " : ""}define} from resolveFile, preview ? (name) => loaders.getSourceLastModified(resolvePath(path, name)) - : (name) => loaders.getOutputLastModified(resolvePath(path, name)) + : (name) => loaders.getOutputLastModified(resolvePath(path, name)), + preview + ? (name) => loaders.getSourceSize(resolvePath(path, name)) + : (name) => loaders.getOutputSize(resolvePath(path, name)) )}` : "" }${data?.sql ? `\n${registerTables(data.sql, options)}` : ""} @@ -103,24 +106,27 @@ function registerTable(name: string, source: string, {path}: RenderOptions): str function registerFiles( files: Iterable, resolve: (name: string) => string, - getLastModified: (name: string) => number | undefined + getLastModified: (name: string) => number | undefined, + getSize: (name: string) => number | undefined ): string { return Array.from(files) .sort() - .map((f) => registerFile(f, resolve, getLastModified)) + .map((f) => registerFile(f, resolve, getLastModified, getSize)) .join(""); } function registerFile( name: string, resolve: (name: string) => string, - getLastModified: (name: string) => number | undefined + getLastModified: (name: string) => number | undefined, + getSize: (name: string) => number | undefined ): string { return `\nregisterFile(${JSON.stringify(name)}, ${JSON.stringify({ name, mimeType: mime.getType(name) ?? undefined, path: resolve(name), - lastModified: getLastModified(name) + lastModified: getLastModified(name), + size: getSize(name) })});`; } diff --git a/test/build-test.ts b/test/build-test.ts index 28106c9c6..c6c860dc5 100644 --- a/test/build-test.ts +++ b/test/build-test.ts @@ -162,7 +162,7 @@ class TestEffects extends FileBuildEffects { async writeFile(outputPath: string, contents: string | Buffer): Promise { if (typeof contents === "string" && outputPath.endsWith(".html")) { contents = contents.replace(/^(\s*