Add pagination support in xata pull (#1477)

Author: SferaDev

cli/src/commands/pull/index.ts

@@ -51,20 +51,22 @@ export default class Pull extends BaseCommand<typeof Pull> {
 
     const details = await getBranchDetailsWithPgRoll(xata, { workspace, region, database, branch });
 
-    let logs: Schemas.MigrationHistoryItem[] | Schemas.Commit[] = [];
+    let logs: (Schemas.MigrationHistoryItem | Schemas.Commit)[] = [];
+    let cursor = undefined;
     if (isBranchPgRollEnabled(details)) {
-      const { migrations } = await xata.api.migrations.getMigrationHistory({
-        pathParams: { workspace, region, dbBranchName: `${database}:${branch}` }
-      });
-      logs = migrations;
+      do {
+        const { migrations, cursor: newCursor } = await xata.api.migrations.getMigrationHistory({
+          pathParams: { workspace, region, dbBranchName: `${database}:${branch}` },
+          queryParams: { cursor, limit: 200 }
+        });
+
+        logs = logs.concat(migrations);
+        cursor = newCursor;
+      } while (cursor !== undefined);
     } else {
       const data = await xata.api.migrations.getBranchSchemaHistory({
         pathParams: { workspace, region, dbBranchName: `${database}:${branch}` },
-        body: {
-          // TODO: Fix pagination in the API to start from last known migration and not from the beginning
-          // Also paginate until we get all migrations
-          page: { size: 200 }
-        }
+        body: { page: { size: 200 } }
       });
       logs = data.logs;
     }

cli/src/commands/pull/pull.test.ts

@@ -230,7 +230,7 @@ const pgrollFetchSingle = (url: string, request: any) => {
         schema: { tables: [{ name: 'table1', columns: [{ name: 'a', type: 'string' }] }] }
       })
     };
-  } else if (url === `${baseUrl}/migrations/history` && request.method === 'GET') {
+  } else if (url === `${baseUrl}/migrations/history?limit=200` && request.method === 'GET') {
     return {
       ok: true,
       json: async () => ({
@@ -251,7 +251,7 @@ const pgrollFetchMultiple = (url: string, request: any) => {
         schema: { tables: [{ name: 'table1', columns: [{ name: 'a', type: 'string' }] }] }
       })
     };
-  } else if (url === `${baseUrl}/migrations/history` && request.method === 'GET') {
+  } else if (url === `${baseUrl}/migrations/history?limit=200` && request.method === 'GET') {
     return {
       ok: true,
       json: async () => ({

cli/src/commands/push/index.ts

@@ -48,20 +48,22 @@ export default class Push extends BaseCommand<typeof Push> {
 
     const details = await getBranchDetailsWithPgRoll(xata, { workspace, region, database, branch });
 
-    let logs: Schemas.MigrationHistoryItem[] | Schemas.Commit[] = [];
+    let logs: (Schemas.MigrationHistoryItem | Schemas.Commit)[] = [];
+    let cursor = undefined;
     if (isBranchPgRollEnabled(details)) {
-      const { migrations } = await xata.api.migrations.getMigrationHistory({
-        pathParams: { workspace, region, dbBranchName: `${database}:${branch}` }
-      });
-      logs = migrations;
+      do {
+        const { migrations, cursor: newCursor } = await xata.api.migrations.getMigrationHistory({
+          pathParams: { workspace, region, dbBranchName: `${database}:${branch}` },
+          queryParams: { cursor, limit: 200 }
+        });
+
+        logs = logs.concat(migrations);
+        cursor = newCursor;
+      } while (cursor !== undefined);
     } else {
       const data = await xata.api.migrations.getBranchSchemaHistory({
         pathParams: { workspace, region, dbBranchName: `${database}:${branch}` },
-        body: {
-          // TODO: Fix pagination in the API to start from last known migration and not from the beginning
-          // Also paginate until we get all migrations
-          page: { size: 200 }
-        }
+        body: { page: { size: 200 } }
       });
       logs = data.logs;
     }

cli/src/commands/push/push.test.ts

@@ -244,7 +244,7 @@ const pgrollFetchSingle = (url: string, request: any, type: 'inferred' | 'pgroll
         schema: { tables: [{ name: 'table1', columns: [{ name: 'a', type: 'string' }] }] }
       })
     };
-  } else if (url === `${baseUrl}/migrations/history` && request.method === 'GET') {
+  } else if (url === `${baseUrl}/migrations/history?limit=200` && request.method === 'GET') {
     return {
       ok: true,
       json: async () => (type === 'inferred' ? { migrations: [pgrollMigration3] } : { migrations: [pgrollMigration1] })
@@ -263,7 +263,7 @@ const pgrollFetchEmpty = (url: string, request: any) => {
         schema: { tables: [{ name: 'table1', columns: [{ name: 'a', type: 'string' }] }] }
       })
     };
-  } else if (url === `${baseUrl}/migrations/history` && request.method === 'GET') {
+  } else if (url === `${baseUrl}/migrations/history?limit=200` && request.method === 'GET') {
     return {
       ok: true,
       json: async () => ({

cli/src/migrations/files.ts

@@ -97,7 +97,7 @@ export async function removeLocalMigrations() {
   }
 }
 
-export function commitToMigrationFile(logs: Schemas.Commit[] | Schemas.MigrationHistoryItem[]): LocalMigrationFile[] {
+export function commitToMigrationFile(logs: (Schemas.Commit | Schemas.MigrationHistoryItem)[]): LocalMigrationFile[] {
   // Schema history comes in reverse order, so we need to reverse it
   return logs.reverse().map((log) =>
     isMigrationPgRollFormat(log)