Merge pull request #73 from ilo-token/master

0.5.1

Author: neverRare

CHANGELOG.md

@@ -15,6 +15,12 @@ On this on-development version, things can be broken.
 </details>
 -->
 
+## 0.5.1
+
+- Fix adverb placement within verbs: "sina o lape pona" can now translate into
+  "You should nicely sleep" instead of "You nicely should sleep".
+- Fix "token is not defined" bug when telo misikeke error messages is turned on.
+
 ## 0.5.0
 
 ilo Token can now translate "o" imperative sentences such as "o toki" and "mi o

CONTRIBUTING.md

@@ -22,7 +22,9 @@ its wiki. You may open an issue for:
 - Fix suggestion to the wiki
 
 Please remember to search first before opening an issue, it might already exist!
-Duplicate issues are unnecessary.
+Also check the [changelog](./CHANGELOG.md) and open the on-development
+changelog, it might be already fixed but hasn't published yet! Duplicate issues
+are unnecessary.
 
 ## [Discussion](https://github.com/ilo-token/ilo-token.github.io/discussions)
 

bundle.ts

@@ -1,21 +1,16 @@
 import { denoPlugins } from "@luca/esbuild-deno-loader";
 import { debounce } from "@std/async/debounce";
 import * as ESBuild from "esbuild";
+import * as Dictionary from "./dictionary/build.ts";
 
 const WATCH = [
-  "./dictionary/build.ts",
-  "./dictionary/dictionary",
-  "./dictionary/misc.ts",
-  "./dictionary/parser.ts",
-  "./dictionary/type.ts",
-  "./telo_misikeke/linku_data.json",
-  "./telo_misikeke/Parser.js",
-  "./telo_misikeke/rules.js",
-  "./telo_misikeke/telo_misikeke.js",
+  "./dictionary/",
+  "./telo_misikeke/",
   "./src/",
   "./project_data.json",
 ];
-const DICTIONARY = /dictionary[/\\][^/\\]+$/;
+const DICTIONARY_DIST_CODE = /[/\\]dictionary[/\\]dictionary\.ts$/;
+const DICTIONARY = /[/\\]dictionary[/\\]dictionary$/;
 
 function buildOptions(minify: boolean): ESBuild.BuildOptions {
   return {
@@ -33,14 +28,12 @@ async function buildAll(
   options: Readonly<{
     minify: boolean;
     buildDictionary: boolean;
-    checkDictionary?: boolean;
   }>,
 ): Promise<void> {
-  const { minify, buildDictionary, checkDictionary } = options;
+  const { minify, buildDictionary } = options;
   try {
     if (buildDictionary) {
-      const Dictionary = await import("./dictionary/build.ts");
-      await Dictionary.build(checkDictionary ?? true);
+      await Dictionary.build();
     }
     // deno-lint-ignore no-console
     console.log("Building main.js...");
@@ -67,19 +60,19 @@ if (import.meta.main) {
       let dictionaryChanged = false;
       const buildDebounced = debounce((buildDictionary: boolean) => {
         task = task.then(async () => {
-          await buildAll({
-            minify: false,
-            buildDictionary,
-            checkDictionary: false,
-          });
+          await buildAll({ minify: false, buildDictionary });
           dictionaryChanged = false;
         });
       }, 500);
       for await (const event of watcher) {
-        if (event.paths.some((path) => DICTIONARY.test(path))) {
-          dictionaryChanged = true;
+        if (
+          !event.paths.every((path) => DICTIONARY_DIST_CODE.test(path))
+        ) {
+          if (event.paths.some((path) => DICTIONARY.test(path))) {
+            dictionaryChanged = true;
+          }
+          buildDebounced(dictionaryChanged);
         }
-        buildDebounced(dictionaryChanged);
       }
       throw new Error("unreachable");
     }

deno.json

@@ -9,7 +9,7 @@
       "dependencies": ["build-dictionary"]
     },
     "start": "deno run --allow-net --allow-read --no-prompt jsr:@std/http/file-server ./dist/",
-    "watch": "deno run --allow-read --allow-write --allow-env --allow-net --allow-run --no-prompt ./bundle.ts watch",
+    "watch": "deno run --allow-read --allow-write --allow-env --allow-net --allow-run --no-prompt --watch ./bundle.ts watch",
     "update": "deno outdated --update && deno run --allow-write --allow-net --no-prompt ./telo_misikeke/update.ts",
     "build-dictionary": "deno run --allow-read --allow-write --no-prompt ./dictionary/build.ts"
   },

dictionary/build.ts

@@ -1,61 +1,26 @@
-import { extractErrorMessage } from "../src/misc.ts";
 import { parseDictionary } from "./parser.ts";
 
 const SOURCE = new URL("./dictionary", import.meta.url);
 const DESTINATION = new URL("./dictionary.ts", import.meta.url);
 
-class ImportError extends Error {
-  constructor(message?: string, options?: { cause: unknown }) {
-    super(message, options);
-    this.name = "ImportError";
-  }
-}
-async function importPrevious(): Promise<string> {
-  try {
-    const { original } = await import("./dictionary.ts");
-    return original;
-  } catch (error) {
-    throw new ImportError(extractErrorMessage(error), { cause: error });
-  }
-}
-export async function build(checkFile: boolean): Promise<void> {
-  const currentPromise = Deno.readTextFile(SOURCE);
-  if (checkFile) {
-    try {
-      const [current, previous] = await Promise.all([
-        currentPromise,
-        importPrevious(),
-      ]);
-      if (current === previous) {
-        return;
-      }
-    } catch (error) {
-      if (!(error instanceof ImportError)) {
-        throw error;
-      }
-    }
-  }
+export async function build(): Promise<void> {
   // deno-lint-ignore no-console
   console.log("Building dictionary...");
-  const text = await currentPromise;
+  const text = await Deno.readTextFile(SOURCE);
   const json = JSON.stringify(
     Object.fromEntries(parseDictionary(text)),
     undefined,
     2,
   );
-  const original = JSON.stringify(text);
   const code = `\
 // This code is autogenerated
 
 import { Dictionary } from "./type.ts";
 
 export const dictionary: Dictionary = new Map(Object.entries(${json}));
-
-export const original = ${original};
 `;
   await Deno.writeTextFile(DESTINATION, code);
 }
 if (import.meta.main) {
-  const checkFile = true;
-  await build(checkFile);
+  await build();
 }

dictionary/parser.ts

@@ -1,3 +1,4 @@
+import { memoize } from "@std/cache/memoize";
 import { escape as escapeHtml } from "@std/html/entities";
 import { escape as escapeRegex } from "@std/regexp/escape";
 import nlp from "compromise/three";
@@ -35,12 +36,30 @@ import {
 const RESERVED_SYMBOLS = "#()*+/:;<=>@[\\]^`{|}~";
 const WORDS = new RegExp(`[^${escapeRegex(RESERVED_SYMBOLS)}]`);
 
-const comment = match(/#[^\n\r]*/, "comment");
-const spaces = sourceOnly(all(choiceOnlyOne(match(/\s/, "space"), comment)));
 function lex<T>(parser: Parser<T>): Parser<T> {
   return parser.skip(spaces);
 }
+const comment = match(/#[^\n\r]*/, "comment");
+const spaces = sourceOnly(all(choiceOnlyOne(match(/\s/, "space"), comment)));
 const backtick = matchString("`", "backtick");
+
+const tokiPonaWord = lex(match(/[a-z][a-zA-Z]*/, "word"));
+const openParenthesis = lex(matchString("(", "open parenthesis"));
+const closeParenthesis = lex(matchString(")", "close parenthesis"));
+const openBracket = lex(matchString("[", "open bracket"));
+const closeBracket = lex(matchString("]", "close bracket"));
+const comma = lex(matchString(",", "comma"));
+const colon = lex(matchString(":", "colon"));
+const semicolon = lex(matchString(";", "semicolon"));
+const slash = lex(matchString("/", "slash"));
+
+const keyword = memoize(<T extends string>(keyword: T): Parser<T> =>
+  lex(match(/[a-z\-]+/, keyword))
+    .filter((that) =>
+      keyword === that ||
+      throwError(new UnexpectedError(`"${that}"`, `"${keyword}"`))
+    ) as Parser<T>
+);
 const unescapedWord = allAtLeastOnce(
   choiceOnlyOne(
     match(WORDS, "word"),
@@ -55,16 +74,8 @@ const unescapedWord = allAtLeastOnce(
     word !== "" || throwError(new ArrayResultError("missing word"))
   );
 const word = unescapedWord.map(escapeHtml);
-const slash = lex(matchString("/", "slash"));
 const forms = sequence(word, all(slash.with(word)))
   .map(([first, rest]) => [first, ...rest]);
-function keyword<T extends string>(keyword: T): Parser<T> {
-  return lex(match(/[a-z\-]+/, keyword))
-    .filter((that) =>
-      keyword === that ||
-      throwError(new UnexpectedError(`"${that}"`, `"${keyword}"`))
-    ) as Parser<T>;
-}
 const number = choiceOnlyOne(keyword("singular"), keyword("plural"));
 const optionalNumber = optionalAll(number);
 const perspective = choiceOnlyOne(
@@ -73,18 +84,12 @@ const perspective = choiceOnlyOne(
   keyword("third"),
 );
 function tag<T>(parser: Parser<T>): Parser<T> {
-  return lex(matchString("(", "open parenthesis"))
-    .with(parser)
-    .skip(lex(matchString(")", "close parenthesis")));
+  return openParenthesis.with(parser).skip(closeParenthesis);
 }
 function template<T>(parser: Parser<T>): Parser<T> {
-  return lex(matchString("[", "open square bracket"))
-    .with(parser)
-    .skip(lex(matchString("]", "close square bracket")));
-}
-function simpleUnit(kind: string): Parser<string> {
-  return word.skip(tag(keyword(kind)));
+  return openBracket.with(parser).skip(closeBracket);
 }
+const simpleUnit = memoize((kind: string) => word.skip(tag(keyword(kind))));
 function detectRepetition(
   source: ReadonlyArray<string>,
 ): { before: string; repeat: string; after: string } {
@@ -324,7 +329,6 @@ function verbOnly(tagInside: Parser<unknown>): Parser<VerbForms> {
       }),
   );
 }
-const semicolon = lex(matchString(";", "semicolon"));
 const definition = choiceOnlyOne<Definition>(
   adjective
     .skip(semicolon)
@@ -492,12 +496,8 @@ const definition = choiceOnlyOne<Definition>(
       type: "filler",
     })),
 );
-const singleWord = lex(match(/[a-z][a-zA-Z]*/, "word"));
-const head = sequence(
-  all(singleWord.skip(lex(matchString(",", "comma")))),
-  singleWord,
-)
-  .skip(matchString(":", "colon"))
+const head = sequence(all(tokiPonaWord.skip(comma)), tokiPonaWord)
+  .skip(colon)
   .map(([init, last]) => [...init, last]);
 const entry = withSource(spaces.with(all(definition)))
   .map(([definitions, src]) => ({ definitions, src: src.trimEnd() }));
@@ -512,6 +512,7 @@ const dictionaryParser = spaces
     )
   )
   .parser();
+
 const definitionExtractor = spaces
   .with(all(optionalAll(lex(head)).with(lex(match(/[^;]*;/, "definition")))))
   .skip(end)

project_data.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.5.0",
+  "version": "0.5.1",
   "onDevelopment": false,
   "releaseDate": "2025-3-9"
 }

src/cache.ts

@@ -1,7 +1,7 @@
 export interface Clearable {
   clear(): void;
 }
-export class Cache {
+export class ClearableCacheSet {
   readonly #caches: Set<WeakRef<Clearable>> = new Set();
   add(cache: Clearable): void {
     this.#caches.add(new WeakRef(cache));

src/dictionary.ts

@@ -2,6 +2,8 @@ import { dictionary as globalDictionary } from "../dictionary/dictionary.ts";
 import { parseDictionary } from "../dictionary/parser.ts";
 import { Definition, Dictionary } from "../dictionary/type.ts";
 
+// All of these global constants are mutable
+
 const customDictionary: Dictionary = new Map();
 export const dictionary: Dictionary = new Map();