feat!: use activePieceCount for accurate piece tracking (#517)

BREAKING: EnhancedDataSetInfo.nextPieceId removed, s/currentPieceCount/activePieceCount

This exposes getActivePieceCount from PDPVerifier, replaces the inaccurate
nextPieceId-based counting with actual active piece count, and removes
nextPieceId from the data set details struct to avoid extra eth_call for a low-
value piece of information.

Author: rvagg

docs/src/content/docs/developer-guides/storage/storage-operations.mdx

@@ -115,7 +115,7 @@ for (const ds of dataSets) {
   console.log(`Dataset ${ds.pdpVerifierDataSetId}:`, {
     live: ds.isLive,
     cdn: ds.withCDN,
-    pieces: ds.currentPieceCount,
+    pieces: ds.activePieceCount,
     metadata: ds.metadata
   });
 }

packages/synapse-core/src/mocks/jsonrpc/index.ts

@@ -475,6 +475,7 @@ export const presets = {
       dataSetLive: () => [true],
       getDataSetListener: () => [ADDRESSES.calibration.warmStorage],
       getNextPieceId: () => [2n],
+      getActivePieceCount: () => [2n],
       getActivePieces: () => [[], [], false],
       getDataSetStorageProvider: () => [ADDRESSES.serviceProvider1, ADDRESSES.zero],
       getDataSetLeafCount: () => [0n],

packages/synapse-core/src/mocks/jsonrpc/pdp.ts

@@ -6,6 +6,7 @@ import * as Abis from '../../abis/index.ts'
 import type { AbiToType, JSONRPCOptions } from './types.ts'
 
 export type getNextPieceId = ExtractAbiFunction<typeof Abis.pdp, 'getNextPieceId'>
+export type getActivePieceCount = ExtractAbiFunction<typeof Abis.pdp, 'getActivePieceCount'>
 export type dataSetLive = ExtractAbiFunction<typeof Abis.pdp, 'dataSetLive'>
 export type getDataSetListener = ExtractAbiFunction<typeof Abis.pdp, 'getDataSetListener'>
 export type getActivePieces = ExtractAbiFunction<typeof Abis.pdp, 'getActivePieces'>
@@ -17,6 +18,7 @@ export interface PDPVerifierOptions {
   dataSetLive?: (args: AbiToType<dataSetLive['inputs']>) => AbiToType<dataSetLive['outputs']>
   getDataSetListener?: (args: AbiToType<getDataSetListener['inputs']>) => AbiToType<getDataSetListener['outputs']>
   getNextPieceId?: (args: AbiToType<getNextPieceId['inputs']>) => AbiToType<getNextPieceId['outputs']>
+  getActivePieceCount?: (args: AbiToType<getActivePieceCount['inputs']>) => AbiToType<getActivePieceCount['outputs']>
   getActivePieces?: (args: AbiToType<getActivePieces['inputs']>) => AbiToType<getActivePieces['outputs']>
   getDataSetStorageProvider?: (
     args: AbiToType<getDataSetStorageProvider['inputs']>
@@ -65,6 +67,14 @@ export function pdpVerifierCallHandler(data: Hex, options: JSONRPCOptions): Hex
         Abis.pdp.find((abi) => abi.type === 'function' && abi.name === 'getNextPieceId')!.outputs,
         options.pdpVerifier.getNextPieceId(args)
       )
+    case 'getActivePieceCount':
+      if (!options.pdpVerifier?.getActivePieceCount) {
+        throw new Error('PDP Verifier: getActivePieceCount is not defined')
+      }
+      return encodeAbiParameters(
+        Abis.pdp.find((abi) => abi.type === 'function' && abi.name === 'getActivePieceCount')!.outputs,
+        options.pdpVerifier.getActivePieceCount(args)
+      )
     case 'getActivePieces': {
       if (!options.pdpVerifier?.getActivePieces) {
         throw new Error('PDP Verifier: getActivePieces is not defined')

packages/synapse-sdk/src/pdp/verifier.ts

@@ -46,13 +46,23 @@ export class PDPVerifier {
   /**
    * Get the next piece ID for a data set
    * @param dataSetId - The PDPVerifier data set ID
-   * @returns The next piece ID (which equals the current piece count)
+   * @returns The next piece ID to assign (total pieces ever added; does not decrease when pieces are removed)
    */
   async getNextPieceId(dataSetId: number): Promise<number> {
     const nextPieceId = await this._contract.getNextPieceId(dataSetId)
     return Number(nextPieceId)
   }
 
+  /**
+   * Get the count of active pieces (non-zero leaf count) for a data set
+   * @param dataSetId - The PDPVerifier data set ID
+   * @returns The number of active pieces in the data set
+   */
+  async getActivePieceCount(dataSetId: number): Promise<number> {
+    const count = await this._contract.getActivePieceCount(dataSetId)
+    return Number(count)
+  }
+
   /**
    * Get the data set listener (record keeper)
    * @param dataSetId - The PDPVerifier data set ID

packages/synapse-sdk/src/retriever/chain.ts

@@ -44,7 +44,7 @@ export class ChainRetriever implements PieceRetriever {
     const dataSets = await this.warmStorageService.getClientDataSetsWithDetails(client)
 
     // Filter for live data sets with pieces
-    const validDataSets = dataSets.filter((ds) => ds.isLive && ds.currentPieceCount > 0)
+    const validDataSets = dataSets.filter((ds) => ds.isLive && ds.activePieceCount > 0)
 
     if (validDataSets.length === 0) {
       throw createError('ChainRetriever', 'findProviders', `No active data sets with data found for client ${client}`)

packages/synapse-sdk/src/storage/context.ts

@@ -539,7 +539,7 @@ export class StorageContext {
     type EvaluatedDataSet = {
       dataSetId: number
       dataSetMetadata: Record<string, string>
-      currentPieceCount: number
+      activePieceCount: number
     }
 
     // Sort ascending by ID (oldest first) for deterministic selection
@@ -558,9 +558,9 @@ export class StorageContext {
         sortedDataSets.slice(i, i + BATCH_SIZE).map(async (dataSet) => {
           const dataSetId = Number(dataSet.dataSetId)
           try {
-            const [dataSetMetadata, currentPieceCount] = await Promise.all([
+            const [dataSetMetadata, activePieceCount] = await Promise.all([
               warmStorageService.getDataSetMetadata(dataSetId),
-              warmStorageService.getNextPieceId(dataSetId),
+              warmStorageService.getActivePieceCount(dataSetId),
               warmStorageService.validateDataSet(dataSetId),
             ])
 
@@ -571,7 +571,7 @@ export class StorageContext {
             return {
               dataSetId,
               dataSetMetadata,
-              currentPieceCount,
+              activePieceCount,
             }
           } catch (error) {
             console.warn(
@@ -587,7 +587,7 @@ export class StorageContext {
         if (result == null) continue
 
         // select the first dataset with pieces and break out of the inner loop
-        if (result.currentPieceCount > 0) {
+        if (result.activePieceCount > 0) {
           selectedDataSet = result
           break
         }
@@ -599,7 +599,7 @@ export class StorageContext {
       }
 
       // early exit if we found a dataset with pieces; break out of the outer loop
-      if (selectedDataSet != null && selectedDataSet.currentPieceCount > 0) {
+      if (selectedDataSet != null && selectedDataSet.activePieceCount > 0) {
         break
       }
     }
@@ -689,8 +689,8 @@ export class StorageContext {
     if (managedDataSets.length > 0 && !forceCreateDataSet) {
       // Prefer data sets with pieces, sort by ID (older first)
       const sorted = managedDataSets.sort((a, b) => {
-        if (a.currentPieceCount > 0 && b.currentPieceCount === 0) return -1
-        if (b.currentPieceCount > 0 && a.currentPieceCount === 0) return 1
+        if (a.activePieceCount > 0 && b.activePieceCount === 0) return -1
+        if (b.activePieceCount > 0 && a.activePieceCount === 0) return 1
         return a.pdpVerifierDataSetId - b.pdpVerifierDataSetId
       })
 

packages/synapse-sdk/src/test/retriever-chain.test.ts

@@ -572,7 +572,7 @@ describe('ChainRetriever', () => {
               return [dataSetId !== 1n] // Data set 1 not live
             },
             getDataSetListener: () => [Mocks.ADDRESSES.calibration.warmStorage],
-            getNextPieceId: (args) => {
+            getActivePieceCount: (args) => {
               const [dataSetId] = args
               return [dataSetId === 2n ? 0n : 1n] // Data set 2 has no pieces
             },

packages/synapse-sdk/src/test/storage.test.ts

@@ -429,7 +429,7 @@ describe('StorageService', () => {
           ...Mocks.presets.basic,
           pdpVerifier: {
             ...Mocks.presets.basic.pdpVerifier,
-            getNextPieceId: (args) => {
+            getActivePieceCount: (args) => {
               const [dataSetId] = args
               if (dataSetId === 2n) {
                 return [2n]

packages/synapse-sdk/src/test/warm-storage-service.test.ts

@@ -255,8 +255,7 @@ describe('WarmStorageService', () => {
       assert.lengthOf(detailedDataSets, 1)
       assert.equal(detailedDataSets[0].pdpRailId, 48)
       assert.equal(detailedDataSets[0].pdpVerifierDataSetId, 242)
-      assert.equal(detailedDataSets[0].nextPieceId, 2)
-      assert.equal(detailedDataSets[0].currentPieceCount, 2)
+      assert.equal(detailedDataSets[0].activePieceCount, 2)
       assert.isTrue(detailedDataSets[0].isLive)
       assert.isTrue(detailedDataSets[0].isManaged)
     })

packages/synapse-sdk/src/types.ts

@@ -229,10 +229,8 @@ export interface DataSetInfo {
 export interface EnhancedDataSetInfo extends DataSetInfo {
   /** PDPVerifier global data set ID */
   pdpVerifierDataSetId: number
-  /** Next piece ID to use when adding pieces */
-  nextPieceId: number
-  /** Current number of pieces in the data set */
-  currentPieceCount: number
+  /** Number of active pieces in the data set (excludes removed pieces) */
+  activePieceCount: number
   /** Whether the data set is live on-chain */
   isLive: boolean
   /** Whether this data set is managed by the current Warm Storage contract */