fix(coin:tezos): api iterates over all results

Author: jprudent

.vscode/launch.json

@@ -16,6 +16,19 @@
       "type": "node",
       "request": "attach",
       "skipFiles": ["<node_internals>/**"]
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug Jest Tests",
+      "program": "${workspaceFolder}/node_modules/.bin/jest",
+      "args": ["--runInBand", "coin-tezoz", "api/index.test.ts"],
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "neverOpen",
+      "skipFiles": ["<node_internals>/**"],
+      "runtimeArgs": ["--inspect-brk"],
+      "sourceMaps": true,
+      "outFiles": ["${workspaceFolder}/dist/**/*.js"]
     }
   ]
 }

libs/coin-modules/coin-tezos/src/api/index.test.ts

@@ -0,0 +1,71 @@
+import { Operation } from "@ledgerhq/coin-framework/lib/api/types";
+import { createApi } from "./index";
+
+const logicGetTransactions = jest.fn();
+jest.mock("../logic", () => ({
+  listOperations: async () => {
+    return logicGetTransactions();
+  },
+}));
+
+const api = createApi({
+  baker: {
+    url: "https://baker.example.com",
+  },
+  explorer: {
+    url: "foo",
+    maxTxQuery: 1,
+  },
+  node: {
+    url: "bar",
+  },
+  fees: {
+    minGasLimit: 1,
+    minRevealGasLimit: 1,
+    minStorageLimit: 1,
+    minFees: 1,
+    minEstimatedFees: 2,
+  },
+});
+
+describe("get operations", () => {
+  afterEach(() => {
+    logicGetTransactions.mockClear();
+  });
+
+  it("operations", async () => {
+    logicGetTransactions.mockResolvedValue([[], ""]);
+
+    // When
+    const [operations, token] = await api.listOperations("addr", { minHeight: 100 });
+
+    // Then
+    expect(operations).toEqual([]);
+    expect(token).toEqual("");
+  });
+
+  const op: Operation = {
+    hash: "opHash",
+    address: "tz1...",
+    type: "transaction",
+    value: BigInt(1000),
+    fee: BigInt(100),
+    block: {
+      hash: "blockHash",
+      height: 123456,
+      time: new Date(),
+    },
+    senders: ["tz1Sender"],
+    recipients: ["tz1Recipient"],
+    date: new Date(),
+    transactionSequenceNumber: 1,
+  };
+
+  it("stops iterating after 10 iterations", async () => {
+    logicGetTransactions.mockResolvedValue([[op], "888"]);
+    const [operations, token] = await api.listOperations("addr", { minHeight: 100 });
+    expect(logicGetTransactions).toHaveBeenCalledTimes(10);
+    expect(operations.length).toBe(10);
+    expect(token).toEqual("888");
+  });
+});

libs/coin-modules/coin-tezos/src/api/index.ts

@@ -1,5 +1,6 @@
 import {
   IncorrectTypeError,
+  Operation,
   Pagination,
   type Api,
   type Transaction as ApiTransaction,
@@ -16,6 +17,8 @@ import {
   rawEncode,
 } from "../logic";
 import api from "../network/tzkt";
+import { assert } from "console";
+import { log } from "@ledgerhq/logs";
 
 export function createApi(config: TezosConfig): Api {
   coinConfig.setCoinConfig(() => ({ ...config, status: { type: "active" } }));
@@ -65,7 +68,62 @@ async function estimate(addr: string, amount: bigint): Promise<bigint> {
   return estimatedFees.estimatedFees;
 }
 
-function operations(address: string, _pagination: Pagination) {
-  //TODO implement properly with https://github.com/LedgerHQ/ledger-live/pull/8875
-  return listOperations(address, {});
+type PaginationState = {
+  readonly pageSize: number;
+  readonly maxIterations: number; // a security to avoid infinite loop
+  currentIteration: number;
+  readonly minHeight: number;
+  continueIterations: boolean;
+  nextCursor?: string;
+  accumulator: Operation[];
+};
+
+async function fetchNextPage(address: string, state: PaginationState): Promise<PaginationState> {
+  const [operations, newNextCursor] = await listOperations(address, {
+    limit: state.pageSize,
+    token: state.nextCursor,
+    sort: "Ascending",
+    minHeight: state.minHeight,
+  });
+  const newCurrentIteration = state.currentIteration + 1;
+  let continueIteration = newNextCursor !== "";
+  if (newCurrentIteration >= state.maxIterations) {
+    log("coin:tezos", "(api/operations): max iterations reached", state.maxIterations);
+    continueIteration = false;
+  }
+  const accumulated = operations.concat(state.accumulator);
+  return {
+    ...state,
+    continueIterations: continueIteration,
+    currentIteration: newCurrentIteration,
+    nextCursor: newNextCursor,
+    accumulator: accumulated,
+  };
+}
+
+async function operationsFromHeight(
+  address: string,
+  start: number,
+): Promise<[Operation[], string]> {
+  const firstState: PaginationState = {
+    pageSize: 200,
+    maxIterations: 10,
+    currentIteration: 0,
+    minHeight: start,
+    continueIterations: true,
+    accumulator: [],
+  };
+
+  let state = await fetchNextPage(address, firstState);
+  while (state.continueIterations) {
+    state = await fetchNextPage(address, state);
+  }
+  return [state.accumulator, state.nextCursor || ""];
+}
+
+async function operations(
+  address: string,
+  { minHeight }: Pagination,
+): Promise<[Operation[], string]> {
+  return operationsFromHeight(address, minHeight);
 }

libs/coin-modules/coin-tezos/src/logic/listOperations.test.ts

@@ -10,6 +10,11 @@ jest.mock("../network", () => ({
   },
 }));
 
+const options: { sort: "Ascending" | "Descending"; minHeight: number } = {
+  sort: "Ascending",
+  minHeight: 0,
+};
+
 describe("listOperations", () => {
   afterEach(() => {
     mockNetworkGetTransactions.mockClear();
@@ -19,7 +24,7 @@ describe("listOperations", () => {
     // Given
     mockNetworkGetTransactions.mockResolvedValue([]);
     // When
-    const [results, token] = await listOperations("any address", {});
+    const [results, token] = await listOperations("any address", options);
     // Then
     expect(results).toEqual([]);
     expect(token).toEqual("");
@@ -65,7 +70,7 @@ describe("listOperations", () => {
       // Given
       mockNetworkGetTransactions.mockResolvedValue([operation]);
       // When
-      const [results, token] = await listOperations("any address", {});
+      const [results, token] = await listOperations("any address", options);
       // Then
       expect(results.length).toEqual(1);
       expect(results[0].recipients).toEqual([someDestinationAddress]);
@@ -80,7 +85,7 @@ describe("listOperations", () => {
     // Given
     mockNetworkGetTransactions.mockResolvedValue([operation]);
     // When
-    const [results, token] = await listOperations("any address", {});
+    const [results, token] = await listOperations("any address", options);
     // Then
     expect(results.length).toEqual(1);
     expect(results[0].recipients).toEqual([]);
@@ -92,10 +97,18 @@ describe("listOperations", () => {
     const operation = { ...undelegate, sender: null };
     mockNetworkGetTransactions.mockResolvedValue([operation]);
     // When
-    const [results, token] = await listOperations("any address", {});
+    const [results, token] = await listOperations("any address", options);
     // Then
     expect(results.length).toEqual(1);
     expect(results[0].senders).toEqual([]);
     expect(token).toEqual(JSON.stringify(operation.id));
   });
+
+  it("should order the results in descending order even if the sort option is set to ascending", async () => {
+    const op1 = { ...undelegate, level: "1" };
+    const op2 = { ...undelegate, level: "2" };
+    mockNetworkGetTransactions.mockResolvedValue([op1, op2]);
+    const [results, _] = await listOperations("any address", options);
+    expect(results.map(op => op.block.height)).toEqual(["2", "1"]);
+  });
 });

libs/coin-modules/coin-tezos/src/logic/listOperations.ts

@@ -3,6 +3,7 @@ import { log } from "@ledgerhq/logs";
 import {
   type APIDelegationType,
   type APITransactionType,
+  AccountsGetOperationsOptions,
   isAPIDelegationType,
   isAPITransactionType,
 } from "../network/types";
@@ -24,23 +25,44 @@ export type Operation = {
   transactionSequenceNumber: number;
 };
 
+/**
+ * Returns list of "Transfer", "Delegate" and "Undelegate" Operations associated to an account.
+ * @param address Account address
+ * @param limit the maximum number of operations to return. Beware that's a weak limit, as explorers might not respect it.
+ * @param order whether to return operations starting from the top block or from the oldest block.
+ *   "Descending" returns newest operation first, "Ascending" returns oldest operation first.
+ *   It doesn't control the order of the operations in the result list:
+ *     operations are always returned sorted in descending order (newest first).
+ * @param minHeight retrieve operations from a specific block height until top most (inclusive).
+ * @param token a token to be used for pagination
+ * @returns a list of operations is descending (newest first) order and a token to be used for pagination
+ */
 export async function listOperations(
   address: string,
-  { token, limit }: { limit?: number; token?: string },
+  {
+    token,
+    limit,
+    sort,
+    minHeight,
+  }: { limit?: number; token?: string; sort: "Ascending" | "Descending"; minHeight: number },
 ): Promise<[Operation[], string]> {
-  let options: { lastId?: number; limit?: number } = { limit: limit };
+  let options: AccountsGetOperationsOptions = { limit, sort, "level.ge": minHeight };
   if (token) {
     options = { ...options, lastId: JSON.parse(token) };
   }
   const operations = await tzkt.getAccountOperations(address, options);
   const lastOperation = operations.slice(-1)[0];
+  // it's important to get the last id from the **unfiltered** operation list
+  // otherwise we might miss operations
   const nextToken = lastOperation ? JSON.stringify(lastOperation?.id) : "";
-  return [
-    operations
-      .filter(op => isAPITransactionType(op) || isAPIDelegationType(op))
-      .reduce((acc, op) => acc.concat(convertOperation(address, op)), [] as Operation[]),
-    nextToken,
-  ];
+  const filteredOperations = operations
+    .filter(op => isAPITransactionType(op) || isAPIDelegationType(op))
+    .reduce((acc, op) => acc.concat(convertOperation(address, op)), [] as Operation[]);
+  if (sort === "Ascending") {
+    //results are always sorted in descending order
+    filteredOperations.reverse();
+  }
+  return [filteredOperations, nextToken];
 }
 
 // note that "initiator" of APITransactionType is never used in the conversion

libs/coin-modules/coin-tezos/src/network/types.ts

@@ -61,6 +61,16 @@ export type APIDelegationType = CommonOperationType & {
 export function isAPIDelegationType(op: APIOperation): op is APIDelegationType {
   return op.type === "delegation";
 }
+
+// https://api.tzkt.io/#operation/Accounts_GetOperations
+export type AccountsGetOperationsOptions = {
+  lastId?: number; // used as a pagination cursor to fetch more transactions
+  limit?: number;
+  sort?: "Descending" | "Ascending";
+  // the minimum height of the block the operation is in
+  "level.ge": number;
+};
+
 export type APIOperation =
   | APITransactionType
   | (CommonOperationType & {

libs/coin-modules/coin-tezos/src/network/tzkt.ts

@@ -2,7 +2,7 @@ import URL from "url";
 import { log } from "@ledgerhq/logs";
 import network from "@ledgerhq/live-network";
 import coinConfig from "../config";
-import { APIAccount, APIBlock, APIOperation } from "./types";
+import { APIAccount, APIBlock, APIOperation, AccountsGetOperationsOptions } from "./types";
 
 const getExplorerUrl = () => coinConfig.getCoinConfig().explorer.url;
 
@@ -36,11 +36,7 @@ const api = {
   // https://api.tzkt.io/#operation/Accounts_GetOperations
   async getAccountOperations(
     address: string,
-    query: {
-      lastId?: number;
-      sort?: number;
-      limit?: number;
-    },
+    query: AccountsGetOperationsOptions,
   ): Promise<APIOperation[]> {
     // Remove undefined from query
     Object.entries(query).forEach(
@@ -56,10 +52,7 @@ const api = {
   },
 };
 
-const sortOperation = {
-  ascending: 0,
-  descending: 1,
-};
+// TODO this has same purpose as api/listOperations
 export const fetchAllTransactions = async (
   address: string,
   lastId?: number,
@@ -69,7 +62,8 @@ export const fetchAllTransactions = async (
   do {
     const newOps = await api.getAccountOperations(address, {
       lastId,
-      sort: sortOperation.ascending,
+      sort: "Ascending",
+      "level.ge": 0,
     });
     if (newOps.length === 0) return ops;
     ops = ops.concat(newOps);

libs/coin-modules/coin-xrp/src/api/index.ts

@@ -44,10 +44,10 @@ async function estimate(_addr: string, _amount: bigint): Promise<bigint> {
 }
 
 type PaginationState = {
-  pageSize: number; // must be large enough to avoid unnecessary calls to the underlying explorer
-  maxIterations: number; // a security to avoid infinite loop
+  readonly pageSize: number; // must be large enough to avoid unnecessary calls to the underlying explorer
+  readonly maxIterations: number; // a security to avoid infinite loop
   currentIteration: number;
-  minHeight: number;
+  readonly minHeight: number;
   continueIterations: boolean;
   apiNextCursor?: string;
   accumulator: XrpOperation[];