diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 00000000..04df5fef --- /dev/null +++ b/.prettierrc @@ -0,0 +1,9 @@ +{ + "trailingComma": "es5", + "tabWidth": 4, + "semi": true, + "singleQuote": false, + "printWidth": 120, + "useTabs": false, + "bracketSpacing": true +} \ No newline at end of file diff --git a/src/api/plugin-api.ts b/src/api/plugin-api.ts index 009ff529..c3f524c1 100644 --- a/src/api/plugin-api.ts +++ b/src/api/plugin-api.ts @@ -1,6 +1,6 @@ +import { Datacore } from "index/datacore"; + /** Exterally visible API for datacore. */ export class DatacoreApi { - public constructor() { - - } -} \ No newline at end of file + public constructor(public core: Datacore) {} +} diff --git a/src/expression/deferred.ts b/src/expression/deferred.ts new file mode 100644 index 00000000..61c06532 --- /dev/null +++ b/src/expression/deferred.ts @@ -0,0 +1,22 @@ +/** A promise that can be resolved directly. */ +export type Deferred = Promise & { + resolve: (value: T) => void; + reject: (error: any) => void; +}; + +/** Create a new deferred object, which is a resolvable promise. */ +export function deferred(): Deferred { + let resolve: (value: T) => void; + let reject: (error: any) => void; + + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + + const deferred = promise as any as Deferred; + deferred.resolve = resolve!; + deferred.reject = reject!; + + return deferred; +} diff --git a/src/expression/link.ts b/src/expression/link.ts index b4ce8c97..c1e5c680 100644 --- a/src/expression/link.ts +++ b/src/expression/link.ts @@ -108,7 +108,13 @@ export class Link { /** Convert this link to a raw object which is serialization-friendly. */ public toObject(): Record { - return { path: this.path, type: this.type, subpath: this.subpath, display: this.display, embed: this.embed }; + return { + path: this.path, + type: this.type, + subpath: this.subpath, + display: this.display, + embed: this.embed, + }; } /** Convert any link into a link to its file. */ @@ -153,4 +159,4 @@ export class Link { public fileName(): string { return getFileTitle(this.path); } -} \ No newline at end of file +} diff --git a/src/expression/literal.ts b/src/expression/literal.ts index be482d6a..20b27076 100644 --- a/src/expression/literal.ts +++ b/src/expression/literal.ts @@ -100,7 +100,7 @@ export namespace Literals { /** Sane, English-based defaults for date formats. */ export const DEFAULT_TO_STRING: ToStringSettings = { - nullRepresentation: "\-", + nullRepresentation: "-", dateFormat: "MMMM dd, yyyy", dateTimeFormat: "h:mm a - MMMM dd, yyyy", @@ -134,14 +134,14 @@ export namespace Literals { case "array": let result = ""; if (recursive) result += "["; - result += wrapped.value.map(f => toString(f, setting, true)).join(", "); + result += wrapped.value.map((f) => toString(f, setting, true)).join(", "); if (recursive) result += "]"; return result; case "object": return ( "{ " + Object.entries(wrapped.value) - .map(e => e[0] + ": " + toString(e[1], setting, true)) + .map((e) => e[0] + ": " + toString(e[1], setting, true)) .join(", ") + " }" ); @@ -322,7 +322,7 @@ export namespace Literals { if (field === null || field === undefined) return field; if (Literals.isArray(field)) { - return ([] as Literal[]).concat(field.map(v => deepCopy(v))) as T; + return ([] as Literal[]).concat(field.map((v) => deepCopy(v))) as T; } else if (Literals.isObject(field)) { let result: Record = {}; for (let [key, value] of Object.entries(field)) result[key] = deepCopy(value); @@ -407,7 +407,6 @@ export namespace Literals { } } - /** * A trivial base class which just defines the '$widget' identifier type. Subtypes of * widget are responsible for adding whatever metadata is relevant. If you want your widget diff --git a/src/expression/normalize.ts b/src/expression/normalize.ts index f5884594..282b2b5c 100644 --- a/src/expression/normalize.ts +++ b/src/expression/normalize.ts @@ -122,12 +122,12 @@ export function escapeRegex(str: string) { /** A parsimmon parser which canonicalizes variable names while properly respecting emoji. */ const VAR_NAME_CANONICALIZER: P.Parser = P.alt( P.regex(new RegExp(emojiRegex(), "")), - P.regex(/[0-9\p{Letter}_-]+/u).map(str => str.toLocaleLowerCase()), - P.whitespace.map(_ => "-"), - P.any.map(_ => "") + P.regex(/[0-9\p{Letter}_-]+/u).map((str) => str.toLocaleLowerCase()), + P.whitespace.map((_) => "-"), + P.any.map((_) => "") ) .many() - .map(result => result.join("")); + .map((result) => result.join("")); /** Convert an arbitrary variable name into something JS/query friendly. */ export function canonicalizeVarName(name: string): string { @@ -137,11 +137,11 @@ export function canonicalizeVarName(name: string): string { const HEADER_CANONICALIZER: P.Parser = P.alt( P.regex(new RegExp(emojiRegex(), "")), P.regex(/[0-9\p{Letter}_-]+/u), - P.whitespace.map(_ => " "), - P.any.map(_ => " ") + P.whitespace.map((_) => " "), + P.any.map((_) => " ") ) .many() - .map(result => { + .map((result) => { return result.join("").split(/\s+/).join(" ").trim(); }); @@ -161,4 +161,4 @@ export function setsEqual(first: Set, second: Set): boolean { for (let elem of first) if (!second.has(elem)) return false; return true; -} \ No newline at end of file +} diff --git a/src/expression/widgets.ts b/src/expression/widgets.ts index ee617371..fa8e35ed 100644 --- a/src/expression/widgets.ts +++ b/src/expression/widgets.ts @@ -1,4 +1,4 @@ -import { type Literal, Literals, Widget } from "./literal"; +import { Literal, Literals, Widget } from "./literal"; /** A trivial widget which renders a (key, value) pair, and allows accessing the key and value. */ export class ListPairWidget extends Widget { @@ -46,4 +46,4 @@ export namespace Widgets { export function isBuiltin(widget: Widget): boolean { return isListPair(widget) || isExternalLink(widget); } -} \ No newline at end of file +} diff --git a/src/index/datacore.ts b/src/index/datacore.ts index ef9a7733..fc34ca44 100644 --- a/src/index/datacore.ts +++ b/src/index/datacore.ts @@ -1,3 +1,4 @@ +import { Deferred } from "expression/deferred"; import { Datastore } from "index/datastore"; import { LocalStorageCache } from "index/persister"; import { FileImporter, ImportThrottle } from "index/web-worker/importer"; @@ -18,6 +19,8 @@ export class Datacore extends Component { importer: FileImporter; /** Local-storage backed cache of metadata objects. */ persister: LocalStorageCache; + /** Only set when datacore is in the midst of initialization; tracks current progress. */ + initializer?: DatacoreInitializer; /** If true, datacore is fully hydrated and all files have been indexed. */ initialized: boolean; @@ -27,12 +30,14 @@ export class Datacore extends Component { this.datastore = new Datastore(); this.initialized = false; - this.addChild(this.importer = new FileImporter(app.vault, app.metadataCache, () => { - return { - workers: settings.importerNumThreads, - utilization: Math.max(0.1, Math.min(1.0, settings.importerUtilization)) - } as ImportThrottle; - })); + this.addChild( + (this.importer = new FileImporter(app.vault, app.metadataCache, () => { + return { + workers: settings.importerNumThreads, + utilization: Math.max(0.1, Math.min(1.0, settings.importerUtilization)), + } as ImportThrottle; + })) + ); } /** Obtain the current index revision, for determining if anything has changed. */ @@ -43,20 +48,30 @@ export class Datacore extends Component { /** Initialize datacore by scanning persisted caches and all available files, and queueing parses as needed. */ initialize() { // The metadata cache is updated on initial file index and file loads. - this.registerEvent(this.metadataCache.on("resolve", file => this.reload(file))); + this.registerEvent(this.metadataCache.on("resolve", (file) => this.reload(file))); // Renames do not set off the metadata cache; catch these explicitly. this.registerEvent(this.vault.on("rename", this.rename, this)); // File creation does cause a metadata change, but deletes do not. Clear the caches for this. - this.registerEvent(this.vault.on("delete", af => { - // TODO: Update index. - })); + this.registerEvent( + this.vault.on("delete", (af) => { + // TODO: Update index. + }) + ); // Asynchronously initialize actual content in the background using a lifecycle-respecting object. - const init = new DatacoreInitializer(this, this.vault.getMarkdownFiles(), () => { + const init = (this.initializer = new DatacoreInitializer(this)); + init.finished().then((stats) => { this.initialized = true; + this.initializer = undefined; this.removeChild(init); + + const durationSecs = (stats.durationMs / 1000.0).toFixed(3); + console.log( + `Datacore: Imported all files in the vault in ${durationSecs}s ` + + `(${stats.imported} imported, ${stats.skipped} skipped)` + ); }); this.addChild(init); @@ -68,13 +83,14 @@ export class Datacore extends Component { /** Queue a file for reloading; this is done asynchronously in the background and may take a few seconds. */ private async reload(file: TFile): Promise<{}> { + await this.import(file); return {}; } /** Perform an asynchronous data import of the given file, adding it to the index when finished. */ private async import(file: TFile): Promise { - return this.importer.import(file).then(result => { - // TODO: Add to index. + return this.importer.import(file).then((result) => { + console.log(`Imported: ${file.path}: ${JSON.stringify(result)}`); }); } } @@ -85,30 +101,127 @@ export class DatacoreInitializer extends Component { static BATCH_SIZE: number = 8; /** Whether the initializer should continue to run. */ - private active: boolean; + active: boolean; + /** Queue of files to still import. */ - private queue: TFile[]; + queue: TFile[]; /** The files actively being imported. */ - private current: TFile[]; - - constructor(public core: Datacore, files: TFile[], public finish: () => void) { + current: TFile[]; + /** Deferred promise which resolves when importing is done. */ + done: Deferred; + + /** The time that init started in milliseconds. */ + start: number; + /** Total number of files to import. */ + files: number; + /** Total number of imported files so far. */ + initialized: number; + /** Total number of imported files. */ + imported: number; + /** Total number of skipped files. */ + skipped: number; + + constructor(public core: Datacore) { super(); - this.active = true; - this.queue = ([] as TFile[]).concat(files); + this.active = false; + this.queue = this.core.vault.getMarkdownFiles(); + this.files = this.queue.length; + this.start = Date.now(); + this.current = []; + + this.initialized = this.imported = this.skipped = 0; } async onload() { - // On load, start loading files one by one and importing them into datacore. - // Start by queueing BATCH_SIZE elements. - } + // Queue BATCH_SIZE elements from the queue to import. + this.active = true; - /** Handle a specific file. */ - async handle(file: TFile) { + this.runNext(); + } + /** Promise which resolves when the initialization completes. */ + finished(): Promise { + return this.done; } + /** Cancel initialization. */ onunload() { - this.active = false; + if (this.active) { + this.active = false; + this.done.reject("Initialization was cancelled before completing."); + } + } + + /** Poll for another task to execute from the queue. */ + private runNext() { + // Do nothing if max number of concurrent operations already running. + if (!this.active || this.current.length >= DatacoreInitializer.BATCH_SIZE) { + return; + } + + // There is space available to execute another. + const next = this.queue.pop(); + if (next) { + this.current.push(next); + this.init(next) + .then((result) => this.handleResult(next, result)) + .catch((result) => this.handleResult(next, result)); + + this.runNext(); + } else if (!next && this.current.length == 0) { + this.active = false; + + // All work is done, resolve. + this.done.resolve({ + durationMs: Date.now() - this.start, + files: this.files, + imported: this.imported, + skipped: this.skipped, + }); + } } -} \ No newline at end of file + + /** Process the result of an initialization and queue more runs. */ + private handleResult(file: TFile, result: InitializationResult) { + this.current.remove(file); + this.initialized++; + + if (result.status === "skipped") this.skipped++; + else if (result.status === "imported") this.imported++; + + // Queue more jobs for processing. + this.runNext(); + } + + /** Initialize a specific file. */ + private async init(file: TFile): Promise { + try { + const metadata = this.core.metadataCache.getFileCache(file); + if (!metadata) return { status: "skipped" }; + + await this.core.importer.import(file); + return { status: "imported" }; + } catch (ex) { + console.log("Datacore: Failed to import file: ", ex); + return { status: "skipped" }; + } + } +} + +/** Statistics about a successful vault initialization. */ +export interface InitializationStats { + /** How long initializaton took in miliseconds. */ + durationMs: number; + /** Total number of files that were imported */ + files: number; + /** The number of files that were loaded and imported via background workers. */ + imported: number; + /** The number of files that were skipped due to no longer existing or not being ready. */ + skipped: number; +} + +/** The result of initializing a file. */ +interface InitializationResult { + status: "skipped" | "imported"; +} diff --git a/src/index/datastore.ts b/src/index/datastore.ts index 528e9cd1..dad134ca 100644 --- a/src/index/datastore.ts +++ b/src/index/datastore.ts @@ -5,7 +5,6 @@ import BTree from "sorted-btree"; /** Central, index storage for datacore values. */ export class Datastore { - /** The current store revision. */ public revision: number; /** The master collection of ALL indexed objects. */ @@ -32,7 +31,6 @@ export class Datastore { return; } - } /** Completely clear the datastore of all values. */ @@ -45,10 +43,10 @@ export class Datastore { /** * Search the datastore using the given query, returning an iterator over results. - * + * * Datastore queries return (ordered) lists of results which match the given query. */ - public* search(query: DatastoreQuery): Iterable { + public *search(query: DatastoreQuery): Iterable { yield { $id: "1", $types: ["yes"] }; } -} \ No newline at end of file +} diff --git a/src/index/import/markdown.ts b/src/index/import/markdown.ts index 76ac58ba..6168085e 100644 --- a/src/index/import/markdown.ts +++ b/src/index/import/markdown.ts @@ -5,8 +5,13 @@ import { CachedMetadata, FileStats } from "obsidian"; * Given the raw source and Obsidian metadata for a given markdown file, * return full markdown file metadata. */ -export function markdownImport(path: string, markdown: string, metadata: CachedMetadata, stats: FileStats): MarkdownFile { +export function markdownImport( + path: string, + markdown: string, + metadata: CachedMetadata, + stats: FileStats +): MarkdownFile { return { - path + path, } as any as MarkdownFile; -} \ No newline at end of file +} diff --git a/src/index/persister.ts b/src/index/persister.ts new file mode 100644 index 00000000..1df2bafc --- /dev/null +++ b/src/index/persister.ts @@ -0,0 +1,81 @@ +import { Transferable } from "index/web-worker/transferable"; +import localforage from "localforage"; + +/** A piece of data that has been cached for a specific version and time. */ +export interface Cached { + /** The version of the plugin that the data was written to cache with. */ + version: string; + /** The UNIX epoch time in milliseconds that the data was written to cache. */ + time: number; + /** The data that was cached. */ + data: T; +} + +/** Simpler wrapper for a file-backed cache for arbitrary metadata. */ +export class LocalStorageCache { + public persister: LocalForage; + + public constructor(public appId: string, public version: string) { + this.persister = localforage.createInstance({ + name: "datacore/cache/" + appId, + driver: [localforage.INDEXEDDB], + description: "Cache metadata about files and sections in the datacore index.", + }); + } + + /** Drop the entire cache instance and re-create a new fresh instance. */ + public async recreate() { + await localforage.dropInstance({ name: "datacore/cache/" + this.appId }); + + this.persister = localforage.createInstance({ + name: "datacore/cache/" + this.appId, + driver: [localforage.INDEXEDDB], + description: "Cache metadata about files and sections in the datacore index.", + }); + } + + /** Load file metadata by path. */ + public async loadFile(path: string): Promise> | null | undefined> { + return this.persister.getItem(this.fileKey(path)).then((raw) => { + let result = raw as any as Cached>; + if (result) result.data = Transferable.value(result.data); + return result; + }); + } + + /** Store file metadata by path. */ + public async storeFile(path: string, data: Partial): Promise { + await this.persister.setItem(this.fileKey(path), { + version: this.version, + time: Date.now(), + data: Transferable.transferable(data), + }); + } + + /** Drop old file keys that no longer exist. */ + public async synchronize(existing: string[] | Set): Promise> { + let keys = new Set(await this.allFiles()); + for (let exist of existing) keys.delete(exist); + + // Any keys remaining after deleting existing keys are non-existent keys that should be cleared from cache. + for (let key of keys) await this.persister.removeItem(this.fileKey(key)); + + return keys; + } + + /** Obtain a list of all metadata keys. */ + public async allKeys(): Promise { + return this.persister.keys(); + } + + /** Obtain a list of all persisted files. */ + public async allFiles(): Promise { + let keys = await this.allKeys(); + return keys.filter((k) => k.startsWith("file:")).map((k) => k.substring(5)); + } + + /** Get a unique key for a given file path. */ + public fileKey(path: string): string { + return "file:" + path; + } +} diff --git a/src/index/types/index-query.ts b/src/index/types/index-query.ts index bdb133e7..c4d8183e 100644 --- a/src/index/types/index-query.ts +++ b/src/index/types/index-query.ts @@ -31,4 +31,4 @@ export interface DatastoreCompare { } /** An index query over the data store. */ -export type DatastoreQuery = DatastoreAnd | DatastoreOr; \ No newline at end of file +export type DatastoreQuery = DatastoreAnd | DatastoreOr; diff --git a/src/index/types/markdown.ts b/src/index/types/markdown.ts index f0f71c65..b366a88f 100644 --- a/src/index/types/markdown.ts +++ b/src/index/types/markdown.ts @@ -117,7 +117,7 @@ export interface MarkdownInlineField { /** The character position of just the value of the inline field. */ valuePosition: CharacterSpan; /** How the inline field is delimited. */ - wrapping: '[' | '(' | 'line'; + wrapping: "[" | "(" | "line"; } /** A union type over all possible markdown fields, including inline objects, frontmatter, and inline fields. */ @@ -146,4 +146,4 @@ export interface CharacterSpan { start: number; /** The exclusive end character offset. */ end: number; -} \ No newline at end of file +} diff --git a/src/index/web-worker/importer.ts b/src/index/web-worker/importer.ts index 2908a70c..a32fe17e 100644 --- a/src/index/web-worker/importer.ts +++ b/src/index/web-worker/importer.ts @@ -15,8 +15,8 @@ export interface ImportThrottle { /** Default throttle configuration. */ export const DEFAULT_THROTTLE: ImportThrottle = { workers: 2, - utilization: 0.75 -} + utilization: 0.75, +}; /** Multi-threaded file parser which debounces rapid file requests automatically. */ export class FileImporter extends Component { @@ -80,7 +80,7 @@ export class FileImporter extends Component { const [file, resolve, reject] = this.queue.shift()!; worker.active = [file, resolve, reject, Date.now()]; - this.vault.cachedRead(file).then(c => + this.vault.cachedRead(file).then((c) => worker!.worker.postMessage({ path: file.path, contents: c, @@ -155,7 +155,7 @@ export class FileImporter extends Component { worker: new ImportWorker(), }; - worker.worker.onmessage = evt => this.finish(worker, Transferable.value(evt.data)); + worker.worker.onmessage = (evt) => this.finish(worker, Transferable.value(evt.data)); return worker; } @@ -191,4 +191,4 @@ function terminate(worker: PoolWorker) { if (worker.active) worker.active[2]("Terminated"); worker.active = undefined; -} \ No newline at end of file +} diff --git a/src/index/web-worker/importer.worker.ts b/src/index/web-worker/importer.worker.ts index 980d6b89..8245a8ac 100644 --- a/src/index/web-worker/importer.worker.ts +++ b/src/index/web-worker/importer.worker.ts @@ -8,11 +8,13 @@ onmessage = (event) => { if (message.type === "markdown") { const markdown = markdownImport(message.path, message.contents, message.metadata, message.stat); - postMessage(Transferable.transferable({ - type: "markdown", - result: markdown - } as MarkdownImportResult)); + postMessage( + Transferable.transferable({ + type: "markdown", + result: markdown, + } as MarkdownImportResult) + ); } else if (message.type === "canvas") { - postMessage({ "$error": "Unsupported import method." }); + postMessage({ $error: "Unsupported import method." }); } -}; \ No newline at end of file +}; diff --git a/src/index/web-worker/message.ts b/src/index/web-worker/message.ts index c74a1a9d..44ee0424 100644 --- a/src/index/web-worker/message.ts +++ b/src/index/web-worker/message.ts @@ -13,7 +13,7 @@ export interface MarkdownImport { stat: FileStats; /** Metadata for the file. */ metadata: CachedMetadata; -} +} /** A command to import a canvas file. */ export interface CanvasImport { @@ -43,4 +43,4 @@ export interface ImportFailure { $error: string; } -export type ImportResult = MarkdownImportResult | ImportFailure; \ No newline at end of file +export type ImportResult = MarkdownImportResult | ImportFailure; diff --git a/src/index/web-worker/transferable.ts b/src/index/web-worker/transferable.ts index f9420157..88a5a0bc 100644 --- a/src/index/web-worker/transferable.ts +++ b/src/index/web-worker/transferable.ts @@ -34,11 +34,17 @@ export namespace Transferable { }, }; case "duration": - return { "$transfer-type": "duration", value: transferable(wrapped.value.toObject()) }; + return { + "$transfer-type": "duration", + value: transferable(wrapped.value.toObject()), + }; case "array": - return wrapped.value.map(v => transferable(v)); + return wrapped.value.map((v) => transferable(v)); case "link": - return { "$transfer-type": "link", value: transferable(wrapped.value.toObject()) }; + return { + "$transfer-type": "link", + value: transferable(wrapped.value.toObject()), + }; case "object": let result: Record = {}; for (let [key, value] of Object.entries(wrapped.value)) result[key] = transferable(value); @@ -61,7 +67,7 @@ export namespace Transferable { for (let val of transferable) real.add(value(val)); return real; } else if (Array.isArray(transferable)) { - return transferable.map(v => value(v)); + return transferable.map((v) => value(v)); } else if (typeof transferable === "object") { if ("$transfer-type" in transferable) { switch (transferable["$transfer-type"]) { diff --git a/src/main.ts b/src/main.ts index 6db24c2b..62277d33 100644 --- a/src/main.ts +++ b/src/main.ts @@ -17,8 +17,8 @@ export default class DatacorePlugin extends Plugin { this.settings = Object.assign(DEFAULT_SETTINGS, (await this.loadData()) ?? {}); this.addSettingTab(new GeneralSettingsTab(this.app, this)); - this.addChild(this.core = new Datacore(this.app, this.manifest.version, this.settings)); - this.api = new DatacoreApi(); + this.addChild((this.core = new Datacore(this.app, this.manifest.version, this.settings))); + this.api = new DatacoreApi(this.core); if (!this.app.workspace.layoutReady) { this.app.workspace.onLayoutReady(async () => this.core.initialize()); @@ -26,6 +26,9 @@ export default class DatacorePlugin extends Plugin { this.core.initialize(); } + // Make the API globally accessible from any context. + window.datacore = this.api; + console.log(`Datacore: version ${this.manifest.version} (requires obsidian ${this.manifest.minAppVersion})`); } @@ -52,21 +55,21 @@ class GeneralSettingsTab extends PluginSettingTab { new Setting(this.containerEl) .setName("Importer Threads") .setDesc("The number of importer threads to use for parsing metadata.") - .addSlider(slider => { + .addSlider((slider) => { slider .setLimits(1, 8, 1) .setValue(this.plugin.settings.importerNumThreads) - .onChange(async value => await this.plugin.updateSettings({ importerNumThreads: value })); + .onChange(async (value) => await this.plugin.updateSettings({ importerNumThreads: value })); }); new Setting(this.containerEl) .setName("Importer Utilization") .setDesc("How much CPU time each importer thread should use (10% - 100%).") - .addSlider(slider => { + .addSlider((slider) => { slider .setLimits(0.1, 1.0, 0.1) .setValue(this.plugin.settings.importerUtilization) - .onChange(async value => await this.plugin.updateSettings({ importerUtilization: value })); + .onChange(async (value) => await this.plugin.updateSettings({ importerUtilization: value })); }); } -} \ No newline at end of file +} diff --git a/src/settings.ts b/src/settings.ts index 395a126d..caa02d2a 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -9,5 +9,5 @@ export interface Settings { /** Default settings for the plugin. */ export const DEFAULT_SETTINGS: Readonly = Object.freeze({ importerNumThreads: 2, - importerUtilization: 0.75 -}); \ No newline at end of file + importerUtilization: 0.75, +});