Skip to content

Commit 25ddd66

Browse files
dljsjrdljsjr
committed
feat: Improved ergonomics for import/require
This commit improves the ergonomics aroud importing/requiring scripts from other files or blocks, while remaining backwards compatible with the existing functionality in Datacore. The specifics of the changes are: - Paths and wikilink-like strings passed to `dc.require()` will be checked against all elements in the `scriptRoots` Setting if the path fails to resolve against the vault root - The `imports` transform is now enabled on the Sucrase `transform` operations. This causes `import { foo } from "bar"` syntax to be transformed in to `const _foo = require("bar")` during transpiling. - The script transpilation pass will convert any `foo = require("bar")` statements in to the form `foo = await dc.require("bar")` before evaluation. This covers both explicitly authored CommonJS require syntax, as well as the output of the Sucrase transformation for ES Module imports. - The script transpilation pass will resolve relative paths (e.g. `./foo`) against the script that called `dc.require()`, allowing for relative imports to now be utilized. This happens as part of the same new transform sa the one above that converts all `require` statements in to `dc.require` statements. - The `imports` transform also allows for the use of the `export` keyword in scripts, which gets transformed in to keys on an `exports` object. - The `exports` object is injected via the context used to construct the `Function` evaluator. - Scripts can now choose to either `return { ... }`, as is the current pattern, *or* they can use `export`. - Returned values take precedence over the exports object. - Datacore code blocks that return a View to be rendered can now use `export default` by leveraging the same effects introduced by the `imports` transform. - Similarly to scripts loaded with `dc.require()`, the `return function ...` form is still supported. - Also similarly to the above, the `return function ...` form is given precedence over the `export default` form.
1 parent 63f96c0 commit 25ddd66

8 files changed

Lines changed: 242 additions & 112 deletions

File tree

‎package.json‎

Lines changed: 61 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,61 +1,63 @@
11
{
2-
"name": "datacore",
3-
"version": "0.1.19",
4-
"description": "Reactive data engine for Obsidian.md.",
5-
"main": "lib/index.js",
6-
"types": "lib/index.d.ts",
7-
"files": [
8-
"lib/**/*"
9-
],
10-
"scripts": {
11-
"dev": "node esbuild.config.mjs",
12-
"build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
13-
"test": "yarn run jest",
14-
"test-watch": "yarn run jest -i --watch --no-cache",
15-
"check-format": "yarn run prettier --check src",
16-
"format": "yarn run prettier --write src"
17-
},
18-
"keywords": [
19-
"obsidian",
20-
"datacore",
21-
"dataview",
22-
"pkm"
23-
],
24-
"author": "Michael Brenan",
25-
"license": "MIT",
26-
"devDependencies": {
27-
"@codemirror/language": "https://github.com/lishid/cm-language",
28-
"@codemirror/state": "^6.0.1",
29-
"@codemirror/view": "^6.0.1",
30-
"@types/jest": "^27.0.1",
31-
"@types/luxon": "^2.3.2",
32-
"@types/node": "^16.7.13",
33-
"@types/parsimmon": "^1.10.6",
34-
"builtin-modules": "3.3.0",
35-
"esbuild": "^0.16.11",
36-
"esbuild-plugin-inline-worker": "https://github.com/mitschabaude/esbuild-plugin-inline-worker",
37-
"jest": "^27.1.0",
38-
"obsidian": "^1.6.6",
39-
"prettier": "2.3.2",
40-
"ts-jest": "^27.0.5",
41-
"ts-node": "^10.9.2",
42-
"tslib": "^2.3.1",
43-
"typescript": "^5.4.2"
44-
},
45-
"dependencies": {
46-
"@datastructures-js/queue": "^4.2.3",
47-
"@fortawesome/fontawesome-svg-core": "^6.4.0",
48-
"@fortawesome/free-solid-svg-icons": "^6.4.0",
49-
"@fortawesome/react-fontawesome": "^0.2.0",
50-
"emoji-regex": "^10.2.1",
51-
"flatqueue": "^2.0.3",
52-
"localforage": "1.10.0",
53-
"luxon": "^2.4.0",
54-
"parsimmon": "^1.18.0",
55-
"preact": "^10.17.1",
56-
"react-select": "^5.8.0",
57-
"sorted-btree": "^1.8.1",
58-
"sucrase": "3.35.0",
59-
"yaml": "^2.3.3"
60-
}
2+
"name": "datacore",
3+
"version": "0.1.19",
4+
"description": "Reactive data engine for Obsidian.md.",
5+
"main": "lib/index.js",
6+
"types": "lib/index.d.ts",
7+
"files": [
8+
"lib/**/*"
9+
],
10+
"scripts": {
11+
"dev": "node esbuild.config.mjs",
12+
"build": "tsc -noEmit -skipLibCheck && node esbuild.config.mjs production",
13+
"test": "yarn run jest",
14+
"test-watch": "yarn run jest -i --watch --no-cache",
15+
"check-format": "yarn run prettier --check src",
16+
"format": "yarn run prettier --write src"
17+
},
18+
"keywords": [
19+
"obsidian",
20+
"datacore",
21+
"dataview",
22+
"pkm"
23+
],
24+
"author": "Michael Brenan",
25+
"license": "MIT",
26+
"devDependencies": {
27+
"@codemirror/language": "https://github.com/lishid/cm-language",
28+
"@codemirror/state": "^6.0.1",
29+
"@codemirror/view": "^6.0.1",
30+
"@types/jest": "^27.0.1",
31+
"@types/luxon": "^2.3.2",
32+
"@types/node": "^16.7.13",
33+
"@types/parsimmon": "^1.10.6",
34+
"@types/path-browserify": "^1.0.3",
35+
"builtin-modules": "3.3.0",
36+
"esbuild": "^0.16.11",
37+
"esbuild-plugin-inline-worker": "https://github.com/mitschabaude/esbuild-plugin-inline-worker",
38+
"jest": "^27.1.0",
39+
"obsidian": "^1.6.6",
40+
"prettier": "2.3.2",
41+
"ts-jest": "^27.0.5",
42+
"ts-node": "^10.9.2",
43+
"tslib": "^2.3.1",
44+
"typescript": "^5.4.2"
45+
},
46+
"dependencies": {
47+
"@datastructures-js/queue": "^4.2.3",
48+
"@fortawesome/fontawesome-svg-core": "^6.4.0",
49+
"@fortawesome/free-solid-svg-icons": "^6.4.0",
50+
"@fortawesome/react-fontawesome": "^0.2.0",
51+
"emoji-regex": "^10.2.1",
52+
"flatqueue": "^2.0.3",
53+
"localforage": "1.10.0",
54+
"luxon": "^2.4.0",
55+
"parsimmon": "^1.18.0",
56+
"path-browserify": "^1.0.1",
57+
"preact": "^10.17.1",
58+
"react-select": "^5.8.0",
59+
"sorted-btree": "^1.8.1",
60+
"sucrase": "3.35.0",
61+
"yaml": "^2.3.3"
62+
}
6163
}

‎src/api/local-api.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ export class DatacoreLocalApi {
9090
* ```
9191
*/
9292
public async require(path: string | Link): Promise<any> {
93-
const result = await this.scriptCache.load(path, { dc: this });
93+
const result = await this.scriptCache.load(path, this);
9494
return result.orElseThrow();
9595
}
9696

‎src/api/script-cache.ts‎

Lines changed: 57 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,16 @@ import { Datastore } from "index/datastore";
33
import { Result } from "./result";
44
import { MarkdownCodeblock, MarkdownSection } from "index/types/markdown";
55
import { Deferred, deferred } from "utils/deferred";
6-
import { ScriptLanguage, asyncEvalInContext, transpile } from "utils/javascript";
6+
import {
7+
ScriptDefinition,
8+
ScriptLanguage,
9+
asyncEvalInContext,
10+
defaultScriptLoadingContext,
11+
transpile,
12+
} from "utils/javascript";
713
import { lineRange } from "utils/normalizers";
8-
import { TFile } from "obsidian";
9-
import { Fragment, h } from "preact";
14+
import { normalizePath, TFile } from "obsidian";
15+
import { DatacoreLocalApi } from "./local-api";
1016

1117
/** A script that is currently being loaded. */
1218
export interface LoadingScript {
@@ -56,20 +62,36 @@ export class ScriptCache {
5662
public constructor(private store: Datastore) {}
5763

5864
/** Load the given script at the given path, recursively loading any subscripts as well. */
59-
public async load(path: string | Link, context: Record<string, any>): Promise<Result<any, string>> {
65+
public async load(path: string | Link, api: DatacoreLocalApi): Promise<Result<any, string>> {
66+
// First, attempt to resolve the script against the script roots so we cache a canonical script path as the key.
67+
var linkToLoad = undefined;
68+
const roots = ["", ...api.core.settings.scriptRoots];
69+
for (var i = 0; i < roots.length; i++) {
70+
linkToLoad = this.store.normalizeLinkAgainstSourcePath(path, normalizePath(roots[i]));
71+
if (linkToLoad) {
72+
break;
73+
}
74+
}
75+
76+
const resolvedPath = linkToLoad ?? path;
77+
console.log("path:");
78+
console.log(resolvedPath);
79+
6080
// Always check the cache first.
61-
const key = this.pathkey(path);
81+
const key = this.pathkey(resolvedPath);
6282
const currentScript = this.scripts.get(key);
6383
if (currentScript) {
64-
if (currentScript.type === "loaded") return Result.success(currentScript.object);
84+
if (currentScript.type === "loaded") {
85+
return Result.success(currentScript.object);
86+
}
6587

6688
// TODO: If we try to load an already-loading script, we are almost certainly doing something
6789
// weird. Either the caller is not `await`-ing the load and loading multiple times, OR
6890
// we are in a `require()` loop. Either way, we'll error out for now since we can't handle
6991
// either case currently.
7092
return Result.failure(
71-
`Failed to import script "${path.toString()}", as it is in the middle of being loaded. Do you have
72-
a circular dependency in your require() calls? The currently loaded or loading scripts are:
93+
`Failed to import script "${resolvedPath.toString()}", as it is in the middle of being loaded. Do you have
94+
a circular dependency in your require() calls? The currently loaded or loading scripts are:
7395
${Array.from(this.scripts.values())
7496
.map((sc) => "\t" + sc.path)
7597
.join("\n")}`
@@ -80,7 +102,7 @@ export class ScriptCache {
80102
const deferral = deferred<Result<any, string>>();
81103
this.scripts.set(key, { type: "loading", promise: deferral, path: key });
82104

83-
const result = await this.loadUncached(path, context);
105+
const result = await this.loadUncached(resolvedPath, api);
84106
deferral.resolve(result);
85107

86108
if (result.successful) {
@@ -93,25 +115,30 @@ export class ScriptCache {
93115
}
94116

95117
/** Load a script, directly bypassing the cache. */
96-
private async loadUncached(path: string | Link, context: Record<string, any>): Promise<Result<any, string>> {
97-
const maybeSource = await this.resolveSource(path);
118+
private async loadUncached(scriptPath: string | Link, api: DatacoreLocalApi): Promise<Result<any, string>> {
119+
var maybeSource = await this.resolveSource(scriptPath);
98120
if (!maybeSource.successful) return maybeSource;
99121

100122
// Transpile to vanilla javascript first...
101-
const { code, language } = maybeSource.value;
123+
const scriptDefinition = maybeSource.value;
102124
let basic;
103125
try {
104-
basic = transpile(code, language);
126+
basic = transpile(scriptDefinition);
105127
} catch (error) {
106-
return Result.failure(`Failed to import ${path.toString()} while transpiling from ${language}: ${error}`);
128+
return Result.failure(
129+
`Failed to import ${scriptPath.toString()} while transpiling from ${
130+
scriptDefinition.scriptLanguage
131+
}: ${error}`
132+
);
107133
}
108134

109135
// Then finally execute the script to 'load' it.
110-
const finalContext = Object.assign({ h: h, Fragment: Fragment }, context);
136+
const scriptContext = defaultScriptLoadingContext(api);
111137
try {
112-
return Result.success(await asyncEvalInContext(basic, finalContext));
138+
const loadRet = (await asyncEvalInContext(basic, scriptContext)) ?? scriptContext.exports;
139+
return Result.success(loadRet);
113140
} catch (error) {
114-
return Result.failure(`Failed to execute script '${path.toString()}': ${error}`);
141+
return Result.failure(`Failed to execute script '${scriptPath.toString()}': ${error}`);
115142
}
116143
}
117144

@@ -122,10 +149,8 @@ export class ScriptCache {
122149
}
123150

124151
/** Attempts to resolve the source to load given a path or link to a markdown section. */
125-
private async resolveSource(
126-
path: string | Link
127-
): Promise<Result<{ code: string; language: ScriptLanguage }, string>> {
128-
const object = this.store.resolveLink(path);
152+
private async resolveSource(path: string | Link, sourcePath?: string): Promise<Result<ScriptDefinition, string>> {
153+
const object = this.store.resolveLink(path, sourcePath);
129154
if (!object) return Result.failure("Could not find a script at the given path: " + path.toString());
130155

131156
const tfile = this.store.vault.getFileByPath(object.$file!);
@@ -137,7 +162,7 @@ export class ScriptCache {
137162

138163
try {
139164
const code = await this.store.vault.cachedRead(tfile);
140-
return Result.success({ code, language });
165+
return Result.success({ scriptFile: tfile, scriptLanguage: language, scriptSource: code });
141166
} catch (error) {
142167
return Result.failure("Failed to load javascript/typescript source file: " + error);
143168
}
@@ -158,7 +183,11 @@ export class ScriptCache {
158183
ScriptCache.SCRIPT_LANGUAGES[
159184
maybeBlock.$languages.find((lang) => lang.toLocaleLowerCase() in ScriptCache.SCRIPT_LANGUAGES)!
160185
];
161-
return (await this.readCodeblock(tfile, maybeBlock)).map((code) => ({ code, language }));
186+
return (await this.readCodeblock(tfile, maybeBlock)).map((code) => ({
187+
scriptFile: tfile,
188+
scriptSource: code,
189+
scriptLanguage: language,
190+
}));
162191
} else if (object instanceof MarkdownCodeblock) {
163192
const maybeLanguage = object.$languages.find(
164193
(lang) => lang.toLocaleLowerCase() in ScriptCache.SCRIPT_LANGUAGES
@@ -167,7 +196,11 @@ export class ScriptCache {
167196
return Result.failure(`The codeblock referenced by '${path}' is not a JS/TS codeblock.`);
168197

169198
const language = ScriptCache.SCRIPT_LANGUAGES[maybeLanguage];
170-
return (await this.readCodeblock(tfile, object)).map((code) => ({ code, language }));
199+
return (await this.readCodeblock(tfile, object)).map((code) => ({
200+
scriptFile: tfile,
201+
scriptSource: code,
202+
scriptLanguage: language,
203+
}));
171204
}
172205

173206
return Result.failure(`Cannot import '${path.toString()}: not a JS/TS file or codeblock reference.`);

‎src/index/datastore.ts‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { FolderIndex } from "index/storage/folder";
44
import { InvertedIndex } from "index/storage/inverted";
55
import { IndexPrimitive, IndexQuery, IndexSource } from "index/types/index-query";
66
import { Indexable, LINKABLE_TYPE, LINKBEARING_TYPE, TAGGABLE_TYPE } from "index/types/indexable";
7-
import { MetadataCache, Vault } from "obsidian";
7+
import { MetadataCache, normalizePath, Vault } from "obsidian";
88
import { MarkdownPage } from "./types/markdown";
99
import { extractSubtags, normalizeHeaderForLink } from "utils/normalizers";
1010
import FlatQueue from "flatqueue";
@@ -14,6 +14,7 @@ import { IndexResolver, execute, optimizeQuery } from "index/storage/query-execu
1414
import { Result } from "api/result";
1515
import { Evaluator } from "expression/evaluator";
1616
import { Settings } from "settings";
17+
import path from "path-browserify";
1718

1819
/** Central, index storage for datacore values. */
1920
export class Datastore {
@@ -266,15 +267,33 @@ export class Datastore {
266267
this.revision++;
267268
}
268269

269-
/** Find the corresponding object for a given link. */
270-
public resolveLink(rawLink: string | Link, sourcePath?: string): Indexable | undefined {
270+
public normalizeLinkAgainstSourcePath(rawLink: string | Link, sourcePath?: string): Link {
271271
let link = typeof rawLink === "string" ? Link.parseInner(rawLink) : rawLink;
272-
273272
if (sourcePath) {
274273
const linkdest = this.metadataCache.getFirstLinkpathDest(link.path, sourcePath);
275274
if (linkdest) link = link.withPath(linkdest.path);
276275
}
277276

277+
if (!this.objects.has(link.path)) {
278+
const normalizedSourcePath = normalizePath(sourcePath ?? "/");
279+
const resolvedModuleFile = this.folder
280+
.getExact(normalizePath(path.join(normalizedSourcePath, path.dirname(link.path))), (childPath) => {
281+
const moduleBasename = path.basename(link.path);
282+
const childFileBasename = path.basename(childPath);
283+
return childFileBasename === moduleBasename || childFileBasename.startsWith(`${moduleBasename}.`);
284+
})
285+
.values()
286+
.next().value;
287+
288+
link.withPath(resolvedModuleFile);
289+
}
290+
291+
return link;
292+
}
293+
294+
/** Find the corresponding object for a given link. */
295+
public resolveLink(rawLink: string | Link, sourcePath?: string): Indexable | undefined {
296+
const link = this.normalizeLinkAgainstSourcePath(rawLink, sourcePath);
278297
const file = this.objects.get(link.path);
279298
if (!file) return undefined;
280299

@@ -288,8 +307,9 @@ export class Datastore {
288307
(sec) => normalizeHeaderForLink(sec.$title) == link.subpath || sec.$title == link.subpath
289308
);
290309

291-
if (section) return section;
292-
else return undefined;
310+
if (section) {
311+
return section;
312+
} else return undefined;
293313
} else if (link.type === "block") {
294314
for (const section of file.$sections) {
295315
const block = section.$blocks.find((bl) => bl.$blockId === link.subpath);

‎src/ui/javascript.tsx‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import { ErrorMessage, SimpleErrorBoundary, CURRENT_FILE_CONTEXT, DatacoreContextProvider } from "ui/markdown";
22
import { App, MarkdownRenderChild } from "obsidian";
33
import { DatacoreLocalApi } from "api/local-api";
4-
import { h, render, Fragment, VNode } from "preact";
4+
import { render, VNode } from "preact";
55
import { unmountComponentAtNode } from "preact/compat";
6-
import { ScriptLanguage, asyncEvalInContext, transpile } from "utils/javascript";
6+
import { ScriptLanguage, asyncEvalInContext, defaultScriptLoadingContext, transpile } from "utils/javascript";
77
import { LoadingBoundary, ScriptContainer } from "./loading-boundary";
88
import { Datacore } from "index/datacore";
99

@@ -29,13 +29,16 @@ export class DatacoreJSRenderer extends MarkdownRenderChild {
2929

3030
// Attempt to parse and evaluate the script to produce either a renderable JSX object or a function.
3131
try {
32-
const primitiveScript = transpile(this.script, this.language);
32+
const primitiveScript = transpile({
33+
scriptSource: this.script,
34+
scriptLanguage: this.language,
35+
scriptFile: this.api.app.vault.getFileByPath(this.path)!,
36+
});
37+
const scriptContext = defaultScriptLoadingContext(this.api);
3338
const renderer = async () => {
34-
return await asyncEvalInContext(primitiveScript, {
35-
dc: this.api,
36-
h: h,
37-
Fragment: Fragment,
38-
});
39+
const loadedScript =
40+
(await asyncEvalInContext(primitiveScript, scriptContext)) ?? scriptContext.exports;
41+
return loadedScript;
3942
};
4043

4144
render(

0 commit comments

Comments
 (0)