refactor

Author: zirkelc

.editorconfig

@@ -0,0 +1,21 @@
+# EditorConfig helps developers define and maintain consistent
+# coding styles between different editors and IDEs
+# editorconfig.org
+#
+# Some of these options are also respected by Prettier
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+# Markdown syntax specifies that trailing whitespaces can be meaningful,
+# so let’s not trim those. e.g. 2 trailing spaces = linebreak (<br />)
+# See https://daringfireball.net/projects/markdown/syntax#p
+[*.md]
+trim_trailing_whitespace = false

.github/workflows/ci.yml

@@ -0,0 +1,89 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  checks: write         # needed for checks
+  id-token: write       # needed for npm publish with provenance
+  contents: write       # needed for github release
+  pull-requests: write  # needed for coverage comment
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Biome
+        uses: biomejs/setup-biome@v2
+
+      - name: Lint
+        run: biome ci --no-errors-on-unmatched .
+
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup and Install
+        uses: zirkelc/setup-and-install@v1
+        with:
+          node-version: 20
+
+      - name: Build
+        run: pnpm build
+
+      - name: Test
+        run: pnpm test
+
+  preview:
+    name: Preview
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: corepack enable
+
+      - name: Setup and Install
+        uses: zirkelc/setup-and-install@v1
+        with:
+          node-version: 20
+
+      - name: Build
+        run: pnpm build
+
+      - name: Publish Preview
+        run: npx pkg-pr-new publish --pnpm --packageManager=pnpm
+
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    needs: [lint, test]
+    if: github.ref == 'refs/heads/main'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup and Install
+        uses: zirkelc/setup-and-install@v1
+        with:
+          node-version: 20
+
+      - name: Build
+        run: pnpm build
+
+      - name: Publish
+        uses: zirkelc/npm-publish@v1
+        with:
+          token: ${{ secrets.NPM_TOKEN }}
+          dry-run: false
+          provenance: true

README.md

@@ -0,0 +1,8 @@
+A very opinionated markdown text splitter to create chunks worth embedding.
+
+It is built around a few core ideas:
+
+1. **Semantic length vs. character length**: the length of a piece of content should be measured in terms of how much information it contains (i.e. words), not how many raw characters. Embeddings store semantics, not characters, therefore calculating the length of a chunk should ignore markdown formatting characters that may contribute to semantics of the text but aren't actual information.
+2. **Markdown as tree**: markdown can be represented as a hierarchical tree, where each node (e.g. heading) potentially has child nodes (e.g. paragraphs, images, links). This relationship between headings and their content can be leveraged to create better chunks.
+3. **Atomic units**: the smallest unit for a chunk is a word. Splitting a word further into its characters does not make sense for embeddings.
+4. **Never break semantics**: certain semantic elements like links or images should never be split across chunks. Embedding a partial description (of an image or link) or an URL will not create a meaningful embedding.

biome.json

@@ -0,0 +1,38 @@
+{
+	"$schema": "https://biomejs.dev/schemas/2.1.3/schema.json",
+	"vcs": {
+		"enabled": false,
+		"clientKind": "git",
+		"useIgnoreFile": false
+	},
+	"files": {
+		"ignoreUnknown": false
+	},
+	"formatter": {
+		"enabled": true,
+		"indentStyle": "space",
+		"useEditorconfig": true
+	},
+	"linter": {
+		"enabled": true,
+		"rules": {
+			"recommended": true
+		}
+	},
+  "javascript": {
+    "formatter": {
+      "quoteStyle": "single",
+      "semicolons": "always",
+      "trailingCommas": "all",
+      "arrowParentheses": "always"
+    }
+  },
+	"assist": {
+		"enabled": true,
+		"actions": {
+			"source": {
+				"organizeImports": "on"
+			}
+		}
+	}
+}

package.json

@@ -5,14 +5,17 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "test": "vitest"
+    "test": "vitest",
+    "lint": "biome check . --write --no-errors-on-unmatched"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "devDependencies": {
+    "@biomejs/biome": "2.1.3",
+    "@total-typescript/tsconfig": "^1.0.4",
     "tsx": "^4.20.3",
     "typescript": "^5.8.3",
     "vitest": "^3.2.4"
   }
-}
+}