myanmar_word_segmenter

Author: p

.gitignore

@@ -0,0 +1,133 @@
+myanmar-words/
+MYWORDS.json
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+.cache
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*

MyWordSegmenter.js

@@ -0,0 +1,69 @@
+# myanmar_word_segmenter
+
+In Myanmar sentences, words are often written without spaces between them. This can make it inconvenient for beginners to find them in dictionaries.
+
+This script utilizes a simplistic approach by using a known Myanmar word list to split long words or phrases into smaller individual words, making it easier to locate them in the dictionary.
+
+## Usage
+
+The [MyWordSegmenter.js](./MyWordSegmenter.js) is **standalone** code file that can be used directly in a browser or as a NodeJs module. Check [example.js](./example.js) file.
+
+```javascript
+
+const MyWordSegmenter = require("./MyWordSegmenter");
+const mSegmenter = new MyWordSegmenter();
+
+const imagineTestWord =
+  "123ညီအစ်ကိုမသိတသိအချိန်ညီအစ်ကိုမသိတသိအချိန်Singaporeမသေကောင်းမပျောက်ကောင်းမှုန်မှုန်မွှားမွှား";
+
+console.log("\nTest input", imagineTestWord);
+console.log("Test output", mSegmenter.word_segment(imagineTestWord));
+
+Expected output:
+[
+  '123',
+  'ညီအစ်ကိုမသိတသိအချိန်',
+  'ညီအစ်ကိုမသိတသိအချိန်',
+  'Singapore',
+  'မသေကောင်းမပျောက်ကောင်း',
+  'မှုန်မှုန်မွှားမွှား'
+]
+
+```
+
+## Attributions
+
+- Myanmar word list is from [myanmartools/myanmar-words](https://github.com/myanmartools/myanmar-words.git) - The MIT License Copyright (c) Myanmar Linguistics team.
+
+- syllable_segment method is adapted from [swanhtet1992/ReSegment](https://github.com/swanhtet1992/ReSegment/blob/master/Javascript/resegment.js)
+
+- Many code/text snippets are generated via [ChatGPT](https://chat.openai.com/chat)
+
+---
+
+## Developer notes
+
+### 1. Download or update the myanmar-word list
+
+Clone or download the [myanmar-words](https://github.com/myanmartools/myanmar-words) repository to the same directory with `prepareWordList.js` file. The relative path to `prepareWordList.js` should be `myanmar-words/json-files`.
+
+```bash
+git clone --depth=1 https://github.com/myanmartools/myanmar-words.git
+
+# then run
+node prepareWordList.js
+```
+
+It will use `dev_standalone_template` to generate `MYWORDS.json` and the standalone code file `MyWordSegmenter.js`.
+
+## 2. Miscellaneous
+
+- **Combine with dictionaries**
+
+It is feasible to integrate Myanmar dictionaries within MyWordSegmenter.js to develop a basic Myanmar Popup dictionary browser extension for Myanmar learners.
+
+- **Work with other languages**
+
+The `theSegmenter` method in `MyWordSegmenter.js` can be easily adapted to work with many other languages with minor modifications.
+
+Try yourself with the Pāḷi language.

README.md

@@ -0,0 +1,105 @@
+/**
+ * Simple Word Segmenter
+ * https://github.com/vpnry/myanmar_word_segmenter
+ * File: dev_NodeJs_MyWordSegmenter.js
+ */
+
+const fs = require("fs");
+const path = require("path");
+const myWordListJsonFile = "MYWORDS.json";
+
+class MyWordSegmenter {
+  /**
+   * The MY_SYLLABLE_REGEX and method syllable_segment are adapted from
+   * https://github.com/swanhtet1992/ReSegment/blob/master/Javascript/resegment.js
+   */
+
+  constructor(wordListJsonFile = myWordListJsonFile) {
+    this.wordListJsonFile = wordListJsonFile;
+    this.puncSpecialChars = "…‘’!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" + "။၊";
+    this.puncRegex = new RegExp(
+      `^[${this.puncSpecialChars}]+|[${this.puncSpecialChars}]+$`,
+      "g"
+    );
+
+    // The regular expression for syllable segmentation
+    this.MY_SYLLABLE_REGEX =
+      /(?:(?<!\u1039)([\u1000-\u102A\u103F\u104A-\u104F]|[\u1040-\u1049]+|[^\u1000-\u104F]+)(?![\u103E\u103B]?[\u1039\u103A\u1037]))/g;
+
+    // Load all the words from the file into a set
+    this.MYWORDS = this.loadWords();
+  }
+
+  // Method to load all the words from the file
+  loadWords() {
+    const mergedFilePath = path.join(__dirname, this.wordListJsonFile);
+    const mergedData = JSON.parse(fs.readFileSync(mergedFilePath, "utf8"));
+    const MYWORDS = new Set(mergedData);
+    console.log(`Words in ${this.wordListJsonFile}: ${MYWORDS.size}`);
+    return MYWORDS;
+  }
+
+  syllable_segment(text) {
+    /**
+     * @author Chan Mrate Ko Ko
+     */
+    var outArray = text.replace(this.MY_SYLLABLE_REGEX, "𝕊$1").split("𝕊");
+    if (outArray.length > 0) {
+      outArray.shift();
+    }
+    return outArray;
+  }
+
+  word_segment(word) {
+    if (this.MYWORDS.has(word)) {
+      return [word];
+    }
+    return this.theSegmenter(word);
+  }
+
+  word_segment_sentence(sentence) {
+    let words = sentence.split(/\s+/);
+    let segmentedWords = [];
+
+    for (let word of words) {
+      segmentedWords.push(...this.word_segment(word));
+    }
+
+    return segmentedWords;
+  }
+
+  theSegmenter(text) {
+    text = text.trim();
+    let syllables = this.syllable_segment(text);
+
+    // filter punctuation, whitespace
+    syllables = syllables
+      .filter(
+        (element) => element.replace(this.puncRegex, "").trim().length > 0
+      )
+      .map((element) => element.trim());
+
+    // Split the word into its constituent syllables and look them up in the loaded words set
+    let wordList = [];
+    let i = 0;
+    while (i < syllables.length) {
+      for (let j = syllables.length; j > i; j--) {
+        let word = syllables.slice(i, j).join("");
+        if (this.MYWORDS.has(word)) {
+          wordList.push(word);
+          i = j;
+          break;
+        } else {
+          if (j === i + 1) {
+            let s = syllables[i];
+            wordList.push(s);
+            i++;
+          }
+        }
+      }
+    }
+    return wordList;
+  }
+}
+
+module.exports = MyWordSegmenter;