Compress images on web by decoding once

modules/expo-bluesky-image-compressor/README.md

@@ -1,10 +1,11 @@
 # expo-bluesky-image-compressor
 
 Re-encodes an image as a JPEG that fits a pixel and byte budget, for uploading.
-iOS and Android only. The app uses it while the
-`image_uploads:native_compression:enable` gate is on. Otherwise, and on web, it
-falls back to an `expo-image-manipulator` loop
-(`src/lib/media/image/compressWithManipulator.ts`).
+iOS and Android only. Web does the same with a canvas, in
+`src/lib/media/image/compressToFit.web.ts`. The app uses them while the
+`image_uploads:native_compression:enable` and
+`image_uploads:web_compression:enable` gates are on, and otherwise falls back to
+an `expo-image-manipulator` loop (`src/lib/media/image/compressWithManipulator.ts`).
 
 ```ts
 import {compressImage} from '@bsky.app/expo-bluesky-image-compressor'
@@ -39,7 +40,9 @@ image. So some trial and error is unavoidable, but it can be cheap:
 3. **Shrink and retry.** If nothing fits, the long side shrinks by 20% and it
    starts again, up to 4 sizes in total.
 
-`QualitySearch` is duplicated in Swift and Kotlin. Keep them in sync.
+`QualitySearch` is duplicated in Swift, Kotlin and TypeScript
+(`src/lib/media/image/qualitySearch.ts`, which has the tests). Keep them in
+sync.
 
 Output has no metadata (so no location) but keeps the color profile, and
 transparency is flattened onto white.

modules/expo-bluesky-image-compressor/android/src/main/java/expo/modules/blueskyimagecompressor/QualitySearch.kt

@@ -9,7 +9,9 @@ import kotlin.math.roundToInt
  * [record] the result, until [nextQuality] returns null. Then [best] is the
  * result to use, or null if nothing fit.
  *
- * Kept in sync with QualitySearch.swift. See the README for the strategy.
+ * Kept in sync with QualitySearch.swift and
+ * src/lib/media/image/qualitySearch.ts, which has the tests. See the README
+ * for the strategy.
  */
 internal class QualitySearch<Payload>(
   private val maxBytes: Int,

modules/expo-bluesky-image-compressor/ios/QualitySearch.swift

@@ -6,7 +6,8 @@ import Foundation
  the result, until `nextQuality()` returns nil. Then `best` is the result to
  use, or nil if nothing fit.
 
- Kept in sync with QualitySearch.kt. See the README for the strategy.
+ Kept in sync with QualitySearch.kt and src/lib/media/image/qualitySearch.ts,
+ which has the tests. See the README for the strategy.
  */
 struct QualitySearch<Payload> {
   static var firstQuality: Int { 90 }

src/analytics/features/types.ts

@@ -25,6 +25,7 @@ export enum Features {
   AtmosphereScreenEnable = 'atmosphere_screen:enable',
   NotificationsV2Enable = 'notifications_v2:enable',
   ImageUploadsNativeCompressionEnable = 'image_uploads:native_compression:enable',
+  ImageUploadsWebCompressionEnable = 'image_uploads:web_compression:enable',
 
   // values
   StarterPacksConfig = 'starter_packs:config',

src/lib/media/image/compressToFit.web.ts

@@ -1 +1,180 @@
-export {compressWithManipulator as compressToFit} from '#/lib/media/image/compressWithManipulator'
+import {compressWithManipulator} from '#/lib/media/image/compressWithManipulator'
+import {QualitySearch} from '#/lib/media/image/qualitySearch'
+import {type PickerImage} from '#/lib/media/picker.shared'
+import {type ImageMeta} from '#/state/gallery'
+import {Features, features} from '#/analytics/features'
+
+/** Same as expo-bluesky-image-compressor on native. */
+const MAX_DIMENSION_ATTEMPTS = 4
+const SHRINK_FACTOR = 0.8
+
+/**
+ * Re-encodes the image as a JPEG that fits `maxDimension` and `maxSize`. Works
+ * like expo-bluesky-image-compressor on native: it decodes the image once,
+ * draws it at the target size, and tries qualities in memory, making a blob URL
+ * only for the winner.
+ */
+export async function compressToFit(
+  source: ImageMeta,
+  {maxDimension, maxSize}: {maxDimension: number; maxSize: number},
+): Promise<PickerImage> {
+  if (!features.isOn(Features.ImageUploadsWebCompressionEnable)) {
+    return compressWithManipulator(source, {maxDimension, maxSize})
+  }
+
+  const image = await decode(source)
+  try {
+    let longSide = Math.min(Math.max(image.width, image.height), maxDimension)
+
+    for (let attempt = 0; attempt < MAX_DIMENSION_ATTEMPTS; attempt++) {
+      const canvas = draw(image, longSide)
+      try {
+        const fit = await findEncoding(canvas, maxSize)
+        if (fit) {
+          return {
+            path: URL.createObjectURL(fit.payload),
+            width: canvas.width,
+            height: canvas.height,
+            mime: 'image/jpeg',
+            size: fit.size,
+          }
+        }
+        longSide = Math.floor(
+          Math.max(canvas.width, canvas.height) * SHRINK_FACTOR,
+        )
+      } finally {
+        releaseCanvas(canvas)
+      }
+    }
+  } finally {
+    image.close()
+  }
+
+  throw new Error('Unable to compress image')
+}
+
+/**
+ * Decodes at full size, upright. Not an `<img>`: iOS Safari draws large ones
+ * from a lower resolution copy, so the output comes out blurry.
+ */
+async function decode(source: ImageMeta) {
+  const blob = await (await fetch(source.path)).blob()
+  if (blob.type === 'image/svg+xml') {
+    return decodeSvg(blob, source)
+  }
+  return createImageBitmap(blob, {imageOrientation: 'from-image'})
+}
+
+/**
+ * `createImageBitmap` can't decode vector images, so SVGs are drawn with an
+ * `<img>` first, at the size it gives them (or the size the picker measured,
+ * if the SVG has none).
+ */
+async function decodeSvg(blob: Blob, source: ImageMeta) {
+  const url = URL.createObjectURL(blob)
+  try {
+    const image = new Image()
+    image.src = url
+    await image.decode()
+    const {canvas, context} = createCanvas(
+      image.naturalWidth || source.width,
+      image.naturalHeight || source.height,
+    )
+    try {
+      context.drawImage(image, 0, 0, canvas.width, canvas.height)
+      return await createImageBitmap(canvas)
+    } finally {
+      releaseCanvas(canvas)
+    }
+  } finally {
+    URL.revokeObjectURL(url)
+  }
+}
+
+/**
+ * Draws the image with its long side at `longSide` (never upscaling), onto
+ * white since JPEG has no transparency.
+ */
+function draw(image: ImageBitmap, longSide: number) {
+  const scale = Math.min(longSide / Math.max(image.width, image.height), 1)
+  const width = Math.max(1, Math.round(image.width * scale))
+  const height = Math.max(1, Math.round(image.height * scale))
+
+  /*
+   * Halve it first while it's more than twice the target size. Firefox
+   * samples too few pixels when a single draw shrinks an image by more than
+   * half, which turns fine detail like leaves into speckles.
+   */
+  let source: ImageBitmap | HTMLCanvasElement = image
+  let sourceWidth = image.width
+  let sourceHeight = image.height
+  while (sourceWidth / 2 >= width && sourceHeight / 2 >= height) {
+    sourceWidth = Math.round(sourceWidth / 2)
+    sourceHeight = Math.round(sourceHeight / 2)
+    const half = createCanvas(sourceWidth, sourceHeight)
+    half.context.drawImage(source, 0, 0, sourceWidth, sourceHeight)
+    if (source instanceof HTMLCanvasElement) {
+      releaseCanvas(source)
+    }
+    source = half.canvas
+  }
+
+  const {canvas, context} = createCanvas(width, height)
+  context.fillStyle = '#ffffff'
+  context.fillRect(0, 0, width, height)
+  context.drawImage(source, 0, 0, width, height)
+  if (source instanceof HTMLCanvasElement) {
+    releaseCanvas(source)
+  }
+  return canvas
+}
+
+function createCanvas(width: number, height: number) {
+  const canvas = document.createElement('canvas')
+  canvas.width = width
+  canvas.height = height
+  const context = canvas.getContext('2d')
+  if (!context) {
+    throw new Error('Could not create a canvas')
+  }
+  context.imageSmoothingQuality = 'high'
+  return {canvas, context}
+}
+
+async function findEncoding(canvas: HTMLCanvasElement, maxBytes: number) {
+  const search = new QualitySearch<Blob>(maxBytes)
+  for (
+    let quality = search.nextQuality();
+    quality !== null;
+    quality = search.nextQuality()
+  ) {
+    const blob = await encode(canvas, quality)
+    search.record(quality, blob.size, blob)
+  }
+  return search.best
+}
+
+function encode(canvas: HTMLCanvasElement, quality: number) {
+  return new Promise<Blob>((resolve, reject) => {
+    canvas.toBlob(
+      blob => {
+        if (blob) {
+          resolve(blob)
+        } else {
+          reject(new Error('Could not encode the image'))
+        }
+      },
+      'image/jpeg',
+      quality / 100,
+    )
+  })
+}
+
+/**
+ * Safari keeps a canvas's pixels until it's garbage collected unless it's
+ * shrunk to nothing.
+ */
+function releaseCanvas(canvas: HTMLCanvasElement) {
+  canvas.width = 0
+  canvas.height = 0
+}

src/lib/media/image/compressWithManipulator.ts

@@ -8,8 +8,8 @@ import {type ImageMeta} from '#/state/gallery'
 
 /**
  * Re-encodes the image as a JPEG that fits `maxDimension` and `maxSize`, by
- * binary searching the quality with a full manipulator pass per attempt. Used
- * on web, and on native while `image_uploads:native_compression:enable` is off.
+ * binary searching the quality with a full manipulator pass per attempt. This
+ * is the fallback while the native and web compressors' gates are off.
  */
 export async function compressWithManipulator(
   source: ImageMeta,

src/lib/media/image/qualitySearch.test.ts

@@ -0,0 +1,92 @@
+import {describe, expect, it} from '@jest/globals'
+
+import {QualitySearch} from '#/lib/media/image/qualitySearch'
+
+const MAX_BYTES = 2_000_000
+
+/** Runs a search against a fake encoder and returns what it tried. */
+function run(sizeAt: (quality: number) => number, maxBytes = MAX_BYTES) {
+  const search = new QualitySearch<number>(maxBytes)
+  const tried: number[] = []
+  for (
+    let quality = search.nextQuality();
+    quality !== null;
+    quality = search.nextQuality()
+  ) {
+    tried.push(quality)
+    search.record(quality, sizeAt(quality), quality)
+  }
+  return {best: search.best, tried}
+}
+
+/** JPEG size grows roughly exponentially with quality. */
+function exponential(sizeAt25: number, sizeAt100: number) {
+  const growth = Math.log(sizeAt100 / sizeAt25) / 75
+  return (quality: number) =>
+    Math.round(sizeAt25 * Math.exp(growth * (quality - 25)))
+}
+
+describe('QualitySearch', () => {
+  it('stops at the first guess when it is close enough to the budget', () => {
+    const {best, tried} = run(() => MAX_BYTES * 0.95)
+    expect(tried).toEqual([90])
+    expect(best?.quality).toBe(90)
+  })
+
+  it('tries the top quality when the first guess is well under budget', () => {
+    const {best, tried} = run(exponential(100_000, 500_000))
+    expect(tried).toEqual([90, 100])
+    expect(best?.quality).toBe(100)
+  })
+
+  it('checks the floor and gives up when even the floor is too big', () => {
+    const {best, tried} = run(() => MAX_BYTES + 1)
+    expect(tried).toEqual([90, 25])
+    expect(best).toBeNull()
+  })
+
+  it('searches between the floor and the first guess', () => {
+    const {best, tried} = run(exponential(800_000, 8_000_000))
+    expect(tried.slice(0, 2)).toEqual([90, 25])
+    expect(best!.size).toBeLessThanOrEqual(MAX_BYTES)
+    expect(best!.size).toBeGreaterThanOrEqual(MAX_BYTES * 0.9)
+    expect(tried.length).toBeLessThanOrEqual(4)
+  })
+
+  it('stops when the bounds are one apart', () => {
+    // Size jumps from well under to well over the budget between 60 and 61.
+    const {best, tried} = run(q => (q <= 60 ? 1_000_000 : 3_000_000))
+    expect(best?.quality).toBe(60)
+    expect(new Set(tried).size).toBe(tried.length)
+    expect(tried.length).toBeLessThanOrEqual(6)
+  })
+
+  it('always fits, never repeats a quality and encodes at most 6 times', () => {
+    for (let sizeAt25 = 50_000; sizeAt25 <= 3_000_000; sizeAt25 *= 1.37) {
+      for (const ratio of [2, 4, 8, 16, 40]) {
+        const sizeAt = exponential(sizeAt25, sizeAt25 * ratio)
+        const {best, tried} = run(sizeAt)
+
+        expect(tried.length).toBeLessThanOrEqual(6)
+        expect(new Set(tried).size).toBe(tried.length)
+        expect(tried.every(q => q >= 25 && q <= 100)).toBe(true)
+
+        if (sizeAt(25) > MAX_BYTES) {
+          expect(best).toBeNull()
+          continue
+        }
+        expect(best!.size).toBeLessThanOrEqual(MAX_BYTES)
+        // The best quality that fits, found by trying all of them.
+        let optimum = 25
+        for (let q = 25; q <= 100; q++) {
+          if (sizeAt(q) <= MAX_BYTES) optimum = q
+        }
+        const closeEnough =
+          best!.size >= MAX_BYTES * 0.9 ||
+          best!.quality === optimum ||
+          tried.length === 6
+        expect(closeEnough).toBe(true)
+      }
+    }
+  })
+})