Fix emoji rendering in code blocks with UTF-8 decoding and emoji fonts (#79)

* Initial plan

* Initial plan for fixing emoji rendering

Co-authored-by: dwjohnston <[email protected]>

* Fix emoji rendering by adding emoji-capable fonts to code blocks

Co-authored-by: dwjohnston <[email protected]>

* Add example story to the proper story.

* Add example fix.

* Replace deprecated escape() with proper UTF-8 decoding using TextDecoder

The escape/unescape pattern works but uses deprecated methods. This replaces it with the modern TextDecoder API which properly handles UTF-8 multi-byte characters like emojis.

Why the deprecated pattern worked:
- atob() decodes base64 but treats bytes as Latin-1
- escape() percent-encodes the malformed string
- decodeURIComponent() interprets percent-encoded bytes as UTF-8

Modern solution:
- atob() decodes base64 to binary string
- Convert to Uint8Array byte array
- TextDecoder properly interprets bytes as UTF-8

Added comprehensive tests verifying both patterns give identical results.

Co-authored-by: dwjohnston <[email protected]>

* Add note to document AI generated code

* Create sharp-pianos-hammer.md

* Update test

---------

Co-authored-by: copilot-swe-agent[bot] <[email protected]>
Co-authored-by: dwjohnston <[email protected]>
Co-authored-by: David Johnston <[email protected]>

Author: Copilot

.changeset/sharp-pianos-hammer.md

@@ -0,0 +1,5 @@
+---
+"react-github-permalink": patch
+---
+
+Fix emoji rendering in code blocks with UTF-8 decoding

package-lock.json

@@ -37,8 +37,8 @@ export const DifferentLanguages: Story = {
     <p>TSX</p>
     <GithubPermalink permalink="https://github.com/dwjohnston/react-github-permalink/blob/242681a9df549adcc9a7fca0d8421d98b7e312c4/sample_files/sample1.tsx#L1-L11" />
 
-    <p>TSX2</p>
-    <GithubPermalink permalink="https://github.com/dwjohnston/react-renders/blob/b91494bff90774073c10ba7a2a362d37c8d083ef/src/react-renders/ReactRenders3.tsx#L8-L32" />
+    <p>TSX with Emoji</p>
+    <GithubPermalink permalink="    https://github.com/dwjohnston/react-renders/blob/b91494bff90774073c10ba7a2a362d37c8d083ef/src/react-renders/ReactRenders3.tsx#L8-L19" />
 
 
     <p>Docker file</p>

src/library/GithubPermalink/GithubPermalink.stories.tsx

@@ -107,6 +107,39 @@ export const WithLineExclusionsRealCode: Story = {
             excludeLines={[[105, 107]]}
             excludeText="// snip"
 
+            />
+    ),
+};
+
+const codeWithEmoji = `export function ChildrenStyleOne() {
+    const [value, setValue] = React.useState(0)
+    return <div className="some-parent-component">
+        <strong>ChildrenStyleOne</strong>
+        <p>RenderTracker is directly rendered</p>
+        <button onClick={() => {
+            setValue((prev) => prev + 1);;
+        }}>Increase count: {value}</button>
+        {/* 👇     Here we declare the RenderTracker directly in the component */}
+        <RenderTracker />
+    </div>
+}`;
+
+export const WithEmoji: Story = {
+    render: () => (
+        <GithubPermalinkBase
+            permalink="https://github.com/dwjohnston/react-renders/src/react-renders/ReactRenders3.tsx#L8-L32"
+            data={{
+                lines: codeWithEmoji.split('\n'),
+                lineFrom: 8,
+                lineTo: 19,
+                commit: "b91494b",
+                path: "src/react-renders/ReactRenders3.tsx",
+                owner: "dwjohnston",
+                repo: "react-renders",
+                commitUrl: "https://github.com/dwjohnston/react-renders/commit/b91494b",
+                status: "ok"
+            }} 
+            language="typescript"
             />
     ),
 };

src/library/GithubPermalink/GithubPermalinkBase.stories.tsx

@@ -98,6 +98,8 @@
     &>pre {
         /* react-style-highlighter is adding margin via style attribute*/
         margin: 0 !important;
+        /* Ensure emojis render properly by including emoji-capable fonts */
+        font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
     }
 
     pre+.hide-line-numbers {

src/library/GithubPermalink/github-permalink.css

@@ -0,0 +1,29 @@
+import { describe, it, expect } from 'vitest';
+import { decodeBase64WithUTF8 } from './defaultFunctions';
+
+describe('Base64 UTF-8 Decoding', () => {
+    it('should match the deprecated escape/unescape pattern behavior', () => {
+        const text = '// 👇 This is a comment with emoji';
+
+        // We're using the deprecated pattern for a sanity check to see that the AI generated version is correct.
+        const deprecatedEncodedString = btoa(unescape(encodeURIComponent(text)));
+
+        // Verify our solution gives the same result as the deprecated method
+        const encoder = new TextEncoder();
+        const bytes = encoder.encode(text);
+        const binaryString = String.fromCharCode(...bytes);
+        const base64 = btoa(binaryString);
+
+
+        expect(deprecatedEncodedString).toBe(base64);
+        const modernResult = decodeBase64WithUTF8(base64);
+
+        // The deprecated pattern: decodeURIComponent(escape(atob(base64)))
+        // escape() converts the incorrectly-decoded UTF-8 bytes to percent-encoding
+        // decodeURIComponent() then interprets those percent-encoded bytes as UTF-8
+        const deprecatedResult = decodeURIComponent(escape(atob(base64)));
+
+        expect(modernResult).toBe(deprecatedResult);
+        expect(modernResult).toBe(text);
+    });
+});

src/library/config/defaultFunctions.test.ts

@@ -3,6 +3,39 @@ import { parseGithubIssueLink, parseGithubPermalinkUrl } from "../utils/urlParse
 import { GithubPermalinkDataResponse } from "./GithubPermalinkContext";
 import { ErrorResponses } from "./GithubPermalinkContext";
 
+/**
+ * This is AI generated code from GitHub Copilot.
+ * See: https://github.com/dwjohnston/react-github-permalink/pull/79
+ * But based on my reading of this:https://stackoverflow.com/a/56647993/1068446
+ * But the suggested answer is using deprecated functions (escape/unescape)
+ * 
+ * Properly decode base64 string with UTF-8 support.
+ * GitHub API returns base64-encoded content that may contain UTF-8 characters like emojis.
+ * 
+ * The issue: atob() decodes base64 to a binary string, but treats each byte as a Latin-1 character.
+ * For UTF-8 multi-byte characters (like emojis), this corrupts the data.
+ * 
+ * The solution: Convert the binary string to a byte array, then use TextDecoder to properly
+ * interpret those bytes as UTF-8.
+ */
+export function decodeBase64WithUTF8(base64: string): string {
+    // Remove whitespace that GitHub API might include
+    const cleanedBase64 = base64.replace(/\s/g, '');
+
+    // Decode base64 to binary string (each character represents a byte)
+    const binaryString = atob(cleanedBase64);
+
+    // Convert binary string to byte array
+    const bytes = new Uint8Array(binaryString.length);
+    for (let i = 0; i < binaryString.length; i++) {
+        bytes[i] = binaryString.charCodeAt(i);
+    }
+
+    // Decode UTF-8 bytes to string
+    const decoder = new TextDecoder('utf-8');
+    return decoder.decode(bytes);
+}
+
 
 export async function defaultGetIssueFn(issueLink: string, githubToken?: string, onError?: (err: unknown) => void): Promise<GithubIssueLinkDataResponse> {
     const config = parseGithubIssueLink(issueLink);
@@ -32,10 +65,10 @@ export async function defaultGetIssueFn(issueLink: string, githubToken?: string,
         issueState: issueJson.state,
         status: "ok",
         owner: config.owner,
-        repo: config.repo, 
+        repo: config.repo,
         reactions: issueJson.reactions,
     };
-}export async function defaultGetPermalinkFn(permalink: string, githubToken?: string, onError?: (err: unknown) => void): Promise<GithubPermalinkDataResponse> {
+} export async function defaultGetPermalinkFn(permalink: string, githubToken?: string, onError?: (err: unknown) => void): Promise<GithubPermalinkDataResponse> {
     const config = parseGithubPermalinkUrl(permalink);
 
 
@@ -61,7 +94,7 @@ export async function defaultGetIssueFn(issueLink: string, githubToken?: string,
     }
 
     const [contentJson, commitJson] = await Promise.all([contentResult.json(), commitResult.json()]);
-    const content = atob(contentJson.content);
+    const content = decodeBase64WithUTF8(contentJson.content);
     const lines = content.split("\n");
 
     return {
@@ -76,6 +109,9 @@ export async function defaultGetIssueFn(issueLink: string, githubToken?: string,
         status: "ok"
     };
 }
+
+
+
 export function handleResponse(response: Response): ErrorResponses {
     if (response.status === 404) {
         return { status: "404" };
@@ -87,7 +123,7 @@ export function handleResponse(response: Response): ErrorResponses {
         };
     }
 
-    if(response.status === 401) {
+    if (response.status === 401) {
         return {
             status: "unauthorized"
         }