Add proper linter + formatter, add async initializer

.prettierrc

@@ -0,0 +1,9 @@
+{
+    "trailingComma": "es5",
+    "tabWidth": 4,
+    "semi": true,
+    "singleQuote": false,
+    "printWidth": 120,
+    "useTabs": false,
+    "bracketSpacing": true
+}

src/api/plugin-api.ts

@@ -1,6 +1,6 @@
+import { Datacore } from "index/datacore";
+
 /** Exterally visible API for datacore. */
 export class DatacoreApi {
-    public constructor() {
-
-    }
-}
+    public constructor(public core: Datacore) {}
+}

src/expression/deferred.ts

@@ -0,0 +1,22 @@
+/** A promise that can be resolved directly. */
+export type Deferred<T> = Promise<T> & {
+    resolve: (value: T) => void;
+    reject: (error: any) => void;
+};
+
+/** Create a new deferred object, which is a resolvable promise. */
+export function deferred<T>(): Deferred<T> {
+    let resolve: (value: T) => void;
+    let reject: (error: any) => void;
+
+    const promise = new Promise((res, rej) => {
+        resolve = res;
+        reject = rej;
+    });
+
+    const deferred = promise as any as Deferred<T>;
+    deferred.resolve = resolve!;
+    deferred.reject = reject!;
+
+    return deferred;
+}

src/expression/link.ts

@@ -108,7 +108,13 @@ export class Link {
 
     /** Convert this link to a raw object which is serialization-friendly. */
     public toObject(): Record<string, any> {
-        return { path: this.path, type: this.type, subpath: this.subpath, display: this.display, embed: this.embed };
+        return {
+            path: this.path,
+            type: this.type,
+            subpath: this.subpath,
+            display: this.display,
+            embed: this.embed,
+        };
     }
 
     /** Convert any link into a link to its file. */
@@ -153,4 +159,4 @@ export class Link {
     public fileName(): string {
         return getFileTitle(this.path);
     }
-}
+}

src/expression/literal.ts

@@ -100,7 +100,7 @@ export namespace Literals {
 
     /** Sane, English-based defaults for date formats. */
     export const DEFAULT_TO_STRING: ToStringSettings = {
-        nullRepresentation: "\-",
+        nullRepresentation: "-",
 
         dateFormat: "MMMM dd, yyyy",
         dateTimeFormat: "h:mm a - MMMM dd, yyyy",
@@ -134,14 +134,14 @@ export namespace Literals {
             case "array":
                 let result = "";
                 if (recursive) result += "[";
-                result += wrapped.value.map(f => toString(f, setting, true)).join(", ");
+                result += wrapped.value.map((f) => toString(f, setting, true)).join(", ");
                 if (recursive) result += "]";
                 return result;
             case "object":
                 return (
                     "{ " +
                     Object.entries(wrapped.value)
-                        .map(e => e[0] + ": " + toString(e[1], setting, true))
+                        .map((e) => e[0] + ": " + toString(e[1], setting, true))
                         .join(", ") +
                     " }"
                 );
@@ -322,7 +322,7 @@ export namespace Literals {
         if (field === null || field === undefined) return field;
 
         if (Literals.isArray(field)) {
-            return ([] as Literal[]).concat(field.map(v => deepCopy(v))) as T;
+            return ([] as Literal[]).concat(field.map((v) => deepCopy(v))) as T;
         } else if (Literals.isObject(field)) {
             let result: Record<string, Literal> = {};
             for (let [key, value] of Object.entries(field)) result[key] = deepCopy(value);
@@ -407,7 +407,6 @@ export namespace Literals {
     }
 }
 
-
 /**
  * A trivial base class which just defines the '$widget' identifier type. Subtypes of
  * widget are responsible for adding whatever metadata is relevant. If you want your widget

src/expression/normalize.ts

@@ -122,12 +122,12 @@ export function escapeRegex(str: string) {
 /** A parsimmon parser which canonicalizes variable names while properly respecting emoji. */
 const VAR_NAME_CANONICALIZER: P.Parser<string> = P.alt(
     P.regex(new RegExp(emojiRegex(), "")),
-    P.regex(/[0-9\p{Letter}_-]+/u).map(str => str.toLocaleLowerCase()),
-    P.whitespace.map(_ => "-"),
-    P.any.map(_ => "")
+    P.regex(/[0-9\p{Letter}_-]+/u).map((str) => str.toLocaleLowerCase()),
+    P.whitespace.map((_) => "-"),
+    P.any.map((_) => "")
 )
     .many()
-    .map(result => result.join(""));
+    .map((result) => result.join(""));
 
 /** Convert an arbitrary variable name into something JS/query friendly. */
 export function canonicalizeVarName(name: string): string {
@@ -137,11 +137,11 @@ export function canonicalizeVarName(name: string): string {
 const HEADER_CANONICALIZER: P.Parser<string> = P.alt(
     P.regex(new RegExp(emojiRegex(), "")),
     P.regex(/[0-9\p{Letter}_-]+/u),
-    P.whitespace.map(_ => " "),
-    P.any.map(_ => " ")
+    P.whitespace.map((_) => " "),
+    P.any.map((_) => " ")
 )
     .many()
-    .map(result => {
+    .map((result) => {
         return result.join("").split(/\s+/).join(" ").trim();
     });
 
@@ -161,4 +161,4 @@ export function setsEqual<T>(first: Set<T>, second: Set<T>): boolean {
     for (let elem of first) if (!second.has(elem)) return false;
 
     return true;
-}
+}

src/expression/widgets.ts

@@ -1,4 +1,4 @@
-import { type Literal, Literals, Widget } from "./literal";
+import { Literal, Literals, Widget } from "./literal";
 
 /** A trivial widget which renders a (key, value) pair, and allows accessing the key and value. */
 export class ListPairWidget extends Widget {
@@ -46,4 +46,4 @@ export namespace Widgets {
     export function isBuiltin(widget: Widget): boolean {
         return isListPair(widget) || isExternalLink(widget);
     }
-}
+}

src/index/datacore.ts

@@ -1,3 +1,4 @@
+import { Deferred } from "expression/deferred";
 import { Datastore } from "index/datastore";
 import { LocalStorageCache } from "index/persister";
 import { FileImporter, ImportThrottle } from "index/web-worker/importer";
@@ -18,6 +19,8 @@ export class Datacore extends Component {
     importer: FileImporter;
     /** Local-storage backed cache of metadata objects. */
     persister: LocalStorageCache;
+    /** Only set when datacore is in the midst of initialization; tracks current progress. */
+    initializer?: DatacoreInitializer;
     /** If true, datacore is fully hydrated and all files have been indexed. */
     initialized: boolean;
 
@@ -27,12 +30,14 @@ export class Datacore extends Component {
         this.datastore = new Datastore();
         this.initialized = false;
 
-        this.addChild(this.importer = new FileImporter(app.vault, app.metadataCache, () => {
-            return {
-                workers: settings.importerNumThreads,
-                utilization: Math.max(0.1, Math.min(1.0, settings.importerUtilization))
-            } as ImportThrottle;
-        }));
+        this.addChild(
+            (this.importer = new FileImporter(app.vault, app.metadataCache, () => {
+                return {
+                    workers: settings.importerNumThreads,
+                    utilization: Math.max(0.1, Math.min(1.0, settings.importerUtilization)),
+                } as ImportThrottle;
+            }))
+        );
     }
 
     /** Obtain the current index revision, for determining if anything has changed. */
@@ -43,20 +48,30 @@ export class Datacore extends Component {
     /** Initialize datacore by scanning persisted caches and all available files, and queueing parses as needed. */
     initialize() {
         // The metadata cache is updated on initial file index and file loads.
-        this.registerEvent(this.metadataCache.on("resolve", file => this.reload(file)));
+        this.registerEvent(this.metadataCache.on("resolve", (file) => this.reload(file)));
 
         // Renames do not set off the metadata cache; catch these explicitly.
         this.registerEvent(this.vault.on("rename", this.rename, this));
 
         // File creation does cause a metadata change, but deletes do not. Clear the caches for this.
-        this.registerEvent(this.vault.on("delete", af => {
-            // TODO: Update index.
-        }));
+        this.registerEvent(
+            this.vault.on("delete", (af) => {
+                // TODO: Update index.
+            })
+        );
 
         // Asynchronously initialize actual content in the background using a lifecycle-respecting object.
-        const init = new DatacoreInitializer(this, this.vault.getMarkdownFiles(), () => {
+        const init = (this.initializer = new DatacoreInitializer(this));
+        init.finished().then((stats) => {
             this.initialized = true;
+            this.initializer = undefined;
             this.removeChild(init);
+
+            const durationSecs = (stats.durationMs / 1000.0).toFixed(3);
+            console.log(
+                `Datacore: Imported all files in the vault in ${durationSecs}s ` +
+                    `(${stats.imported} imported, ${stats.skipped} skipped)`
+            );
         });
 
         this.addChild(init);
@@ -68,13 +83,14 @@ export class Datacore extends Component {
 
     /** Queue a file for reloading; this is done asynchronously in the background and may take a few seconds. */
     private async reload(file: TFile): Promise<{}> {
+        await this.import(file);
         return {};
     }
 
     /** Perform an asynchronous data import of the given file, adding it to the index when finished. */
     private async import(file: TFile): Promise<void> {
-        return this.importer.import<ImportResult>(file).then(result => {
-            // TODO: Add to index.
+        return this.importer.import<ImportResult>(file).then((result) => {
+            console.log(`Imported: ${file.path}: ${JSON.stringify(result)}`);
         });
     }
 }
@@ -85,30 +101,127 @@ export class DatacoreInitializer extends Component {
     static BATCH_SIZE: number = 8;
 
     /** Whether the initializer should continue to run. */
-    private active: boolean;
+    active: boolean;
+
     /** Queue of files to still import. */
-    private queue: TFile[];
+    queue: TFile[];
     /** The files actively being imported. */
-    private current: TFile[];
-
-    constructor(public core: Datacore, files: TFile[], public finish: () => void) {
+    current: TFile[];
+    /** Deferred promise which resolves when importing is done. */
+    done: Deferred<InitializationStats>;
+
+    /** The time that init started in milliseconds. */
+    start: number;
+    /** Total number of files to import. */
+    files: number;
+    /** Total number of imported files so far. */
+    initialized: number;
+    /** Total number of imported files. */
+    imported: number;
+    /** Total number of skipped files. */
+    skipped: number;
+
+    constructor(public core: Datacore) {
         super();
 
-        this.active = true;
-        this.queue = ([] as TFile[]).concat(files);
+        this.active = false;
+        this.queue = this.core.vault.getMarkdownFiles();
+        this.files = this.queue.length;
+        this.start = Date.now();
+        this.current = [];
+
+        this.initialized = this.imported = this.skipped = 0;
     }
 
     async onload() {
-        // On load, start loading files one by one and importing them into datacore.
-        // Start by queueing BATCH_SIZE elements.
-    }
+        // Queue BATCH_SIZE elements from the queue to import.
+        this.active = true;
 
-    /** Handle a specific file. */
-    async handle(file: TFile) {
+        this.runNext();
+    }
 
+    /** Promise which resolves when the initialization completes. */
+    finished(): Promise<InitializationStats> {
+        return this.done;
     }
 
+    /** Cancel initialization. */
     onunload() {
-        this.active = false;
+        if (this.active) {
+            this.active = false;
+            this.done.reject("Initialization was cancelled before completing.");
+        }
+    }
+
+    /** Poll for another task to execute from the queue. */
+    private runNext() {
+        // Do nothing if max number of concurrent operations already running.
+        if (!this.active || this.current.length >= DatacoreInitializer.BATCH_SIZE) {
+            return;
+        }
+
+        // There is space available to execute another.
+        const next = this.queue.pop();
+        if (next) {
+            this.current.push(next);
+            this.init(next)
+                .then((result) => this.handleResult(next, result))
+                .catch((result) => this.handleResult(next, result));
+
+            this.runNext();
+        } else if (!next && this.current.length == 0) {
+            this.active = false;
+
+            // All work is done, resolve.
+            this.done.resolve({
+                durationMs: Date.now() - this.start,
+                files: this.files,
+                imported: this.imported,
+                skipped: this.skipped,
+            });
+        }
     }
-}
+
+    /** Process the result of an initialization and queue more runs. */
+    private handleResult(file: TFile, result: InitializationResult) {
+        this.current.remove(file);
+        this.initialized++;
+
+        if (result.status === "skipped") this.skipped++;
+        else if (result.status === "imported") this.imported++;
+
+        // Queue more jobs for processing.
+        this.runNext();
+    }
+
+    /** Initialize a specific file. */
+    private async init(file: TFile): Promise<InitializationResult> {
+        try {
+            const metadata = this.core.metadataCache.getFileCache(file);
+            if (!metadata) return { status: "skipped" };
+
+            await this.core.importer.import(file);
+            return { status: "imported" };
+        } catch (ex) {
+            console.log("Datacore: Failed to import file: ", ex);
+            return { status: "skipped" };
+        }
+    }
+}
+
+/** Statistics about a successful vault initialization. */
+export interface InitializationStats {
+    /** How long initializaton took in miliseconds. */
+    durationMs: number;
+    /** Total number of files that were imported */
+    files: number;
+    /** The number of files that were loaded and imported via background workers. */
+    imported: number;
+    /** The number of files that were skipped due to no longer existing or not being ready. */
+    skipped: number;
+}
+
+/** The result of initializing a file. */
+interface InitializationResult {
+    status: "skipped" | "imported";
+}