Merge pull request #314 from kodadot/main

🔖 Stick v12

Author: vikiival

README.md

@@ -162,6 +162,14 @@ CHAIN=polkadot # or kusama
 OFFER=<ID_OF_THE_COLLECTION>
 ```
 
+6. debugging the processor
+
+As the processor can run for a longer period of time it is useful to turn off the "features" that you do not need,
+Handlers that need to be always enabled are `createCollection` and `createItem` as they are the base for the rest of the processor.
+
+> [!NOTE]
+> If you do not wish to index `uniques` pallet you can turn it off by setting `UNIQUES_ENABLED=false` in `.env` file
+
 ### Note on Swaps
 
 1. Swaps can be overwritten at any time

db/migrations/1726485003107-Data.js

@@ -0,0 +1,13 @@
+module.exports = class Data1726485003107 {
+    name = 'Data1726485003107'
+
+    async up(db) {
+        await db.query(`ALTER TABLE "metadata_entity" ADD "kind" character varying(6)`)
+        await db.query(`ALTER TABLE "collection_entity" ADD "kind" character varying(6)`)
+    }
+
+    async down(db) {
+        await db.query(`ALTER TABLE "metadata_entity" DROP COLUMN "kind"`)
+        await db.query(`ALTER TABLE "collection_entity" DROP COLUMN "kind"`)
+    }
+}

package-lock.json

@@ -16,13 +16,13 @@
     "query-node:start": "squid-graphql-server --subscriptions --max-response-size 10000 --dumb-cache in-memory --dumb-cache-ttl 1000 --dumb-cache-size 100 --dumb-cache-max-age 1000"
   },
   "dependencies": {
-    "@kodadot1/hyperdata": "^0.0.1-rc.4",
+    "@kodadot1/hyperdata": "^0.0.1-rc.5",
     "@kodadot1/metasquid": "^0.3.1-rc.0",
     "@kodadot1/minipfs": "^0.4.3-rc.2",
     "@subsquid/archive-registry": "3.3.2",
-    "@subsquid/graphql-server": "4.6.0",
+    "@subsquid/graphql-server": "4.7.0",
     "@subsquid/ss58": "2.0.2",
-    "@subsquid/substrate-processor": "8.4.1",
+    "@subsquid/substrate-processor": "8.5.1",
     "@subsquid/typeorm-migration": "1.3.0",
     "@subsquid/typeorm-store": "1.5.1",
     "dotenv": "^16.4.5",
@@ -33,9 +33,9 @@
     "typeorm": "0.3.17"
   },
   "devDependencies": {
-    "@subsquid/substrate-metadata-explorer": "3.1.2",
+    "@subsquid/substrate-metadata-explorer": "3.2.0",
     "@subsquid/substrate-typegen": "8.1.0",
-    "@subsquid/typeorm-codegen": "2.0.1",
+    "@subsquid/typeorm-codegen": "2.0.2",
     "@types/md5": "^2.3.5",
     "@types/node": "18.11.18",
     "@types/pg": "^8.11.4",

package.json

@@ -16,6 +16,7 @@ type CollectionEntity @entity {
   id: ID!
   image: String
   issuer: String!
+  kind: Kind
   max: Int
   media: String
   meta: MetadataEntity
@@ -97,6 +98,7 @@ type MetadataEntity @entity {
   animationUrl: String
   type: String
   banner: String
+  kind: Kind
 }
 
 # Entity to represent an Attribute
@@ -259,6 +261,15 @@ enum TradeStatus {
   WITHDRAWN
 }
 
+enum Kind {
+  poap
+  pfp
+  genart
+  mixed
+  # audio
+  # video
+}
+
 # Entity to represent a Fungible Asset
 # defined on chain as  pub type Asset<T: Config<I>, I: 'static = ()>
 # https://github.com/paritytech/polkadot-sdk/blob/99234440f0f8b24f7e4d1d3a0102a9b19a408dd3/substrate/frame/assets/src/lib.rs#L325

schema.graphql

@@ -1,6 +1,6 @@
 manifestVersion: subsquid.io/v0.1
 name: speck
-version: 13
+version: 14
 description: 'SubSquid indexer for Uniques and Assets on Statemint'
 build: 
 deploy:
@@ -16,6 +16,7 @@ deploy:
     env:
       CHAIN: polkadot
       OFFER: 174
+      UNIQUES_ENABLED: true
   api:
     cmd:
       - npx

speck.yaml

@@ -16,6 +16,7 @@ deploy:
     env:
       CHAIN: kusama
       OFFER: 464
+      UNIQUES_ENABLED: true
   api:
     cmd:
       - npx

squid.yaml

@@ -3,9 +3,10 @@ export type Chain = 'kusama' | 'rococo' | 'polkadot'
 export const CHAIN: Chain = process.env.CHAIN as Chain || 'kusama'
 export const COLLECTION_OFFER: string = process.env.OFFER || ''
 
+export const UNIQUES_ENABLED = process.env.UNIQUES === 'true'
 const UNIQUE_STARTING_BLOCK = 323_750 // 618838;
-// const _NFT_STARTING_BLOCK = 4_556_552
-export const STARTING_BLOCK = UNIQUE_STARTING_BLOCK
+const NFT_STARTING_BLOCK = 4_556_552
+export const STARTING_BLOCK = UNIQUES_ENABLED ? UNIQUE_STARTING_BLOCK : NFT_STARTING_BLOCK
 
 // Asset Hub
 const ARCHIVE_URL = `https://v2.archive.subsquid.io/network/asset-hub-${CHAIN}`
@@ -16,6 +17,7 @@ export const isProd = CHAIN !== 'rococo'
 console.table({
   CHAIN, ARCHIVE_URL, NODE_URL, STARTING_BLOCK,
   COLLECTION_OFFER,
+  UNIQUES_ENABLED,
   disabledRPC: false,
   environment: isProd ? 'production' : 'development',
 })

src/environment.ts

@@ -8,8 +8,7 @@ import * as a from './assets'
 import * as n from './nfts'
 import * as u from './uniques'
 import { BatchContext, Context, SelectedEvent } from './utils/types'
-import { updateCache } from './utils/cache'
-import { logError } from './utils/logger'
+import { updateOfferCache } from './utils/cache'
 
 type HandlerFunction = <T extends SelectedEvent>(item: T, ctx: Context) => Promise<void>
 
@@ -234,105 +233,26 @@ export async function mainFrame(ctx: BatchContext<Store>): Promise<void> {
     }
   }
 
-  // const lastDate = new Date(ctx.blocks[ctx.blocks.length - 1].header.timestamp || start)
-  // await updateCache(lastDate, ctx.store)  
-
-  // const { contracts, tokens } = uniqueEntitySets(items)
-  // const collections = await finalizeCollections(contracts, ctx)
-  // const finish = await whatToDoWithTokens({ tokens, collections, items }, ctx)
-  // const complete = await completeTokens(ctx, finish)
-
-  // logger.info(`Batch completed, ${complete.length} tokens saved`)
+  if (ctx.isHead) {
+    const lastBlock = ctx.blocks[ctx.blocks.length - 1].header
+    const lastDate = new Date(lastBlock.timestamp || Date.now())
+    logger.info(`Found head block, updating cache`)
+    await updateOfferCache(lastDate, lastBlock.height, ctx.store)  
+  }
 }
 
-// function unwrapLog(log: Log, block: BlockData) {
-//   switch (log.topics[0]) {
-//     case ERC721_TRANSFER:
+// class Head {
+//   #height: number
 
-//       if (log.address !== Contracts.HueNft) {
-//         return null
-//       }
-//       return handle721Token(log, block)
-//     default:
-//       // console.log('unknown log', log.topics[0])
-//       return null
-//     // throw new Error('unknown log')
+//   constructor(height: number) {
+//     this.#height = height
 //   }
-// }
-
-type What = {
-  // tokens: Set<string>,
-  // collections: EnMap<CE>,
-  // items: ItemStateUpdate[],
-}
 
-export async function whatToDoWithTokens(x: What, ctx: Context) {
-  // // ctx.store.findBy(CE, {id: In([...collectionMap.keys()])})
-  // const knownTokens = await findByIdListAsMap(ctx.store, NE, tokens)
-  // const events: EventEntity[] = []
-  // for (const item of items) {
-  //   logger.debug(`APPLY ${item.interaction} on ${item.id}`)
-  //   let knownToken = knownTokens.get(item.id) ?? create(NE, item.id, {})
-  //   if (item.applyFrom) {
-  //     const collection = collections.get(item.contract)!
-  //     item.applyFrom(collection)
-  //   }
-  //   if (item.applyTo) {
-  //     knownToken = item.applyTo(knownToken)
-  //   }
-  //   events.push(item.event)
-  //   knownTokens.set(item.id, knownToken)
-  // }
-  // const values = [...knownTokens.values()]
-  // await ctx.store.upsert(values)
-  // await ctx.store.save(events)
-  // return knownTokens
-}
+//   get height() {
+//     return this.#height
+//   }
 
-type EnMap<T> = Map<string, T>
-// TODO: do only if event was mint.
-async function completeTokens(ctx: Context, tokenMap: EnMap<NE>) {
-  //   const collections = groupedItemsByCollection(tokenMap.keys())
-  //   const final: NE[] = []
-  //   const metadataFutures: Promise<Optional<MetadataEntity>>[] = []
-  //   for (const [contract, ids] of collections.entries()) {
-  //     const list = Array.from(ids)
-  //     const tokens = await multicallMetadataFetch(ctx, contract, list)
-  //     for (const [i, id] of list.entries()) {
-  //       const realId = createTokenId(contract, id)
-  //       const token = tokenMap.get(realId)!
-  //       if (!token.metadata) {
-  //         const metadata = tokens[i]
-  //         token.metadata = metadata
-  //         const getMeta = handleMetadata(metadata, ctx.store).then(m => {
-  //           if (m) {
-  //             token.meta = m
-  //             token.name = m.name
-  //             token.image = m.image
-  //             token.media = m.animationUrl
-  //           }
-  //           return m
-  //         })
-  //         metadataFutures.push(getMeta)
-  //         final.push(token)
-  //       }
-  //     }
-  //   }
-  //   const metaList = await Promise.all(metadataFutures)
-  //   const filtered = metaList.filter(m => m) as MetadataEntity[]
-  //   logger.debug(`Saving ${filtered.length} metadata`)
-  //   await ctx.store.save(filtered)
-  //   await ctx.store.save(final)
-  //   return final
-  // }
-  // async function multicallMetadataFetch(ctx: Context, collection: string, tokens: Array<string>): Promise<string[]> {
-  //   const tokenIds = tokens.map((id) => [BigInt(id)])
-  //   const contract = new Multicall(ctx, lastBatchBlock(ctx), MULTICALL_ADDRESS)
-  //   const metadata = await contract.aggregate(
-  //     erc721.functions.tokenURI,
-  //     collection,
-  //     tokenIds,
-  //     MULTICALL_BATCH_SIZE
-  //   )
-  //   return metadata
-}
+//   set height(height: number) {
+//     this.#height = height
+//   }
+// }

src/mappings/index.ts

@@ -26,7 +26,7 @@ export async function handleTokenBuy(context: Context): Promise<void> {
   const originalPrice = event.price
   const originalOwner = entity.currentOwner ?? undefined
 
-  entity.price = BigInt(0)
+  entity.price = null
   entity.currentOwner = event.caller
   entity.updatedAt = event.timestamp
 

src/mappings/nfts/buy.ts

@@ -1,6 +1,6 @@
 import { getOrCreate } from '@kodadot1/metasquid/entity'
 import md5 from 'md5'
-import { CollectionEntity as CE } from '../../model'
+import { CollectionEntity as CE, Kind } from '../../model'
 import { handleMetadata } from '../shared/metadata'
 import { unwrap } from '../utils/extract'
 import { debug, pending, success } from '../utils/logger'
@@ -50,6 +50,7 @@ export async function handleCollectionCreate(context: Context): Promise<void> {
     final.name = metadata?.name
     final.image = metadata?.image
     final.media = metadata?.animationUrl
+    final.kind = metadata?.kind || Kind.mixed
   }
 
   success(OPERATION, `[COLLECTION] ${final.id}`)

src/mappings/nfts/create.ts

@@ -23,7 +23,7 @@ export async function handleTokenList(context: Context): Promise<void> {
   const id = createTokenId(event.collectionId, event.sn)
   const entity = await getWith(context.store, NE, id, { collection: true })
 
-  entity.price = event.price
+  entity.price = Boolean(event.price) ? event.price : null
 
   if (event.price && (entity.collection.floor === 0n || event.price < entity.collection.floor)) {
     entity.collection.floor = event.price

src/mappings/nfts/list.ts

@@ -43,7 +43,7 @@ export async function handleTokenCreate(context: Context): Promise<void> {
   final.collection = collection
   final.sn = BigInt(event.sn)
   final.metadata = event.metadata || tokenUri(collection.baseUri, event.sn) || collection.metadata
-  final.price = BigInt(0)
+  final.price = null
   final.burned = false
   final.createdAt = event.timestamp
   final.updatedAt = event.timestamp

src/mappings/nfts/mint.ts

@@ -41,7 +41,7 @@ export async function handleTokenTransfer(context: Context): Promise<void> {
   const entity = await getWith(context.store, NE, id, { collection: true })
 
   const oldOwner = entity.currentOwner
-  entity.price = BigInt(0)
+  entity.price = null
   entity.currentOwner = event.to
   entity.updatedAt = event.timestamp
 

src/mappings/nfts/transfer.ts

@@ -3,7 +3,7 @@ import { Optional } from '@kodadot1/metasquid/types'
 
 import type { Content } from '@kodadot1/hyperdata'
 import { logger } from '@kodadot1/metasquid/logger'
-import { MetadataEntity as Metadata } from '../../model/generated'
+import { Kind, MetadataEntity as Metadata } from '../../model/generated'
 import { isEmpty } from '../utils/helper'
 import { fetchMetadata } from '../utils/metadata'
 import { Store, attributeFrom } from '../utils/types'
@@ -31,6 +31,7 @@ export async function handleMetadata(id: string, store: Store): Promise<Optional
     name: metadata.name || '',
     type: metadata.type || '',
     banner: metadata.banner || '',
+    kind: metadata.kind as Kind || Kind.mixed,
   }
 
   const final = create<Metadata>(Metadata, id, partial)

src/mappings/shared/metadata.ts

@@ -26,7 +26,7 @@ export async function handleTokenBuy(context: Context): Promise<void> {
   const originalPrice = event.price
   const originalOwner = entity.currentOwner ?? undefined
 
-  entity.price = BigInt(0)
+  entity.price = null
   entity.currentOwner = event.caller
   entity.updatedAt = event.timestamp
 

src/mappings/uniques/buy.ts

@@ -1,6 +1,6 @@
 import { getOrCreate } from '@kodadot1/metasquid/entity'
 import md5 from 'md5'
-import { CollectionEntity as CE } from '../../model'
+import { CollectionEntity as CE, Kind } from '../../model'
 import { handleMetadata } from '../shared/metadata'
 import { unwrap } from '../utils/extract'
 import { debug, pending, success } from '../utils/logger'
@@ -50,6 +50,7 @@ export async function handleCollectionCreate(context: Context): Promise<void> {
     final.name = metadata?.name
     final.image = metadata?.image
     final.media = metadata?.animationUrl
+    final.kind = metadata?.kind || Kind.mixed
   }
 
   success(OPERATION, `[COLLECTION] ${final.id}`)

src/mappings/uniques/create.ts

@@ -23,7 +23,7 @@ export async function handleTokenList(context: Context): Promise<void> {
   const id = createTokenId(event.collectionId, event.sn)
   const entity = await getWith(context.store, NE, id, { collection: true })
 
-  entity.price = event.price
+  entity.price = Boolean(event.price) ? event.price : null
 
   if (event.price && (entity.collection.floor === 0n || event.price < entity.collection.floor)) {
     entity.collection.floor = event.price

src/mappings/uniques/list.ts

@@ -43,7 +43,7 @@ export async function handleTokenCreate(context: Context): Promise<void> {
   final.collection = collection
   final.sn = BigInt(event.sn)
   final.metadata = event.metadata || collection.metadata
-  final.price = BigInt(0)
+  final.price = null
   final.burned = false
   final.createdAt = event.timestamp
   final.updatedAt = event.timestamp

src/mappings/uniques/mint.ts

@@ -24,7 +24,7 @@ export async function handleTokenTransfer(context: Context): Promise<void> {
   const entity = await getWith(context.store, NE, id, { collection: true })
 
   const oldOwner = entity.currentOwner
-  entity.price = BigInt(0)
+  entity.price = null
   entity.currentOwner = event.to
   entity.updatedAt = event.timestamp