feat: use location claims (#98)

Simple change to allow freeway to make use of location claims. Note:
still assumes all content is in carpark, this PR just extracts CAR CID
from URL `pathname` and uses byte range to extract block.

Author: Alan Shaw

package-lock.json

@@ -40,7 +40,7 @@
     "@ipld/dag-cbor": "^9.0.8",
     "@ipld/dag-json": "^10.1.7",
     "@ipld/dag-pb": "^4.0.8",
-    "@web3-storage/content-claims": "^4.0.1",
+    "@web3-storage/content-claims": "^4.0.5",
     "@web3-storage/gateway-lib": "^4.1.1",
     "cardex": "^3.0.0",
     "dagula": "^7.3.0",
@@ -55,7 +55,7 @@
     "@ucanto/principal": "^8.1.0",
     "ava": "^5.3.1",
     "carbites": "^1.0.6",
-    "carstream": "^1.1.1",
+    "carstream": "^2.1.0",
     "dotenv": "^16.3.1",
     "esbuild": "^0.18.20",
     "files-from-path": "^0.2.6",

package.json

@@ -1,6 +1,9 @@
 /* global ReadableStream */
 import * as Link from 'multiformats/link'
 import * as raw from 'multiformats/codecs/raw'
+import { base32 } from 'multiformats/bases/base32'
+import * as Digest from 'multiformats/hashes/digest'
+import { varint } from 'multiformats'
 import * as Claims from '@web3-storage/content-claims/client'
 import { MultihashIndexSortedReader } from 'cardex/multihash-index-sorted'
 import { Map as LinkMap } from 'lnmap'
@@ -94,6 +97,18 @@ export class ContentClaimsIndex {
 
     const claims = await Claims.read(cid, { serviceURL: this.#serviceURL })
     for (const claim of claims) {
+      if (claim.type === 'assert/location' && claim.range?.length != null) {
+        const origin = locationToShardCID(claim.location[0])
+        if (!origin) continue
+
+        const { multihash } = cid
+        // convert offset to CARv2 index offset (start of block header)
+        const offset = claim.range.offset - (varint.encodingLength(cid.bytes.length + claim.range.length) + cid.bytes.length)
+
+        this.#cache.set(cid, { origin, multihash, digest: multihash.bytes, offset })
+        continue
+      }
+
       // skip anything that is not a relation claim, since we know by
       // our naming convention that our CAR files are named after their hash
       // and we don't serve anything that we don't have in our own bucket.
@@ -158,3 +173,26 @@ const decodeIndex = async (origin, bytes) => {
   }
   return entries
 }
+
+/**
+ * Attempts to extract a CAR CID from a bucket location URL.
+ *
+ * @param {string} key
+ */
+const locationToShardCID = key => {
+  const filename = String(key.split('/').at(-1))
+  const [hash] = filename.split('.')
+  try {
+    // recent buckets encode CAR CID in filename
+    const cid = Link.parse(hash).toV1()
+    if (isCARLink(cid)) return cid
+    throw new Error('not a CAR CID')
+  } catch (err) {
+    // older buckets base32 encode a CAR multihash <base32(car-multihash)>.car
+    try {
+      const digestBytes = base32.baseDecode(hash)
+      const digest = Digest.decode(digestBytes)
+      return Link.create(CAR.code, digest)
+    } catch {}
+  }
+}

src/lib/dag-index/content-claims.js

@@ -117,6 +117,43 @@ export const generateClaims = async (signer, dataCid, carCid, carStream, indexCi
   return claims
 }
 
+/**
+ * @param {import('@ucanto/interface').Signer} signer
+ * @param {import('cardex/api').CARLink} carCid
+ * @param {ReadableStream<Uint8Array>} carStream CAR file data
+ */
+export const generateLocationClaims = async (signer, carCid, carStream) => {
+  /** @type {Claims} */
+  const claims = new LinkMap()
+
+  await carStream
+    .pipeThrough(new CARReaderStream())
+    .pipeTo(new WritableStream({
+      async write ({ cid, blockOffset, blockLength }) {
+        const invocation = Assert.location.invoke({
+          issuer: signer,
+          audience: signer,
+          with: signer.did(),
+          nb: {
+            content: cid,
+            location: [
+              /** @type {import('@ucanto/interface').URI<'http:'>} */
+              (`http://localhost/${carCid}/${carCid}.car`)
+            ],
+            range: { offset: blockOffset, length: blockLength }
+          }
+        })
+
+        const blocks = claims.get(cid) ?? []
+        // @ts-expect-error
+        blocks.push(await encode(invocation))
+        claims.set(cid, blocks)
+      }
+    }))
+
+  return claims
+}
+
 /**
  * Encode a claim to a block.
  * @param {import('@ucanto/interface').IssuedInvocationView} invocation