Skip to content

Commit af38c4b

Browse files
SferaDevSferaDev
authored
Add pagination support in xata pull (#1477)
1 parent e109eee commit af38c4b

File tree

10 files changed

+5068
-2989
lines changed

10 files changed

+5068
-2989
lines changed

‎cli/src/commands/pull/index.ts‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -51,20 +51,22 @@ export default class Pull extends BaseCommand<typeof Pull> {
5151

5252
const details = await getBranchDetailsWithPgRoll(xata, { workspace, region, database, branch });
5353

54-
let logs: Schemas.MigrationHistoryItem[] | Schemas.Commit[] = [];
54+
let logs: (Schemas.MigrationHistoryItem | Schemas.Commit)[] = [];
55+
let cursor = undefined;
5556
if (isBranchPgRollEnabled(details)) {
56-
const { migrations } = await xata.api.migrations.getMigrationHistory({
57-
pathParams: { workspace, region, dbBranchName: `${database}:${branch}` }
58-
});
59-
logs = migrations;
57+
do {
58+
const { migrations, cursor: newCursor } = await xata.api.migrations.getMigrationHistory({
59+
pathParams: { workspace, region, dbBranchName: `${database}:${branch}` },
60+
queryParams: { cursor, limit: 200 }
61+
});
62+
63+
logs = logs.concat(migrations);
64+
cursor = newCursor;
65+
} while (cursor !== undefined);
6066
} else {
6167
const data = await xata.api.migrations.getBranchSchemaHistory({
6268
pathParams: { workspace, region, dbBranchName: `${database}:${branch}` },
63-
body: {
64-
// TODO: Fix pagination in the API to start from last known migration and not from the beginning
65-
// Also paginate until we get all migrations
66-
page: { size: 200 }
67-
}
69+
body: { page: { size: 200 } }
6870
});
6971
logs = data.logs;
7072
}

‎cli/src/commands/pull/pull.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,7 @@ const pgrollFetchSingle = (url: string, request: any) => {
230230
schema: { tables: [{ name: 'table1', columns: [{ name: 'a', type: 'string' }] }] }
231231
})
232232
};
233-
} else if (url === `${baseUrl}/migrations/history` && request.method === 'GET') {
233+
} else if (url === `${baseUrl}/migrations/history?limit=200` && request.method === 'GET') {
234234
return {
235235
ok: true,
236236
json: async () => ({
@@ -251,7 +251,7 @@ const pgrollFetchMultiple = (url: string, request: any) => {
251251
schema: { tables: [{ name: 'table1', columns: [{ name: 'a', type: 'string' }] }] }
252252
})
253253
};
254-
} else if (url === `${baseUrl}/migrations/history` && request.method === 'GET') {
254+
} else if (url === `${baseUrl}/migrations/history?limit=200` && request.method === 'GET') {
255255
return {
256256
ok: true,
257257
json: async () => ({

‎cli/src/commands/push/index.ts‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,20 +48,22 @@ export default class Push extends BaseCommand<typeof Push> {
4848

4949
const details = await getBranchDetailsWithPgRoll(xata, { workspace, region, database, branch });
5050

51-
let logs: Schemas.MigrationHistoryItem[] | Schemas.Commit[] = [];
51+
let logs: (Schemas.MigrationHistoryItem | Schemas.Commit)[] = [];
52+
let cursor = undefined;
5253
if (isBranchPgRollEnabled(details)) {
53-
const { migrations } = await xata.api.migrations.getMigrationHistory({
54-
pathParams: { workspace, region, dbBranchName: `${database}:${branch}` }
55-
});
56-
logs = migrations;
54+
do {
55+
const { migrations, cursor: newCursor } = await xata.api.migrations.getMigrationHistory({
56+
pathParams: { workspace, region, dbBranchName: `${database}:${branch}` },
57+
queryParams: { cursor, limit: 200 }
58+
});
59+
60+
logs = logs.concat(migrations);
61+
cursor = newCursor;
62+
} while (cursor !== undefined);
5763
} else {
5864
const data = await xata.api.migrations.getBranchSchemaHistory({
5965
pathParams: { workspace, region, dbBranchName: `${database}:${branch}` },
60-
body: {
61-
// TODO: Fix pagination in the API to start from last known migration and not from the beginning
62-
// Also paginate until we get all migrations
63-
page: { size: 200 }
64-
}
66+
body: { page: { size: 200 } }
6567
});
6668
logs = data.logs;
6769
}

‎cli/src/commands/push/push.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,7 @@ const pgrollFetchSingle = (url: string, request: any, type: 'inferred' | 'pgroll
244244
schema: { tables: [{ name: 'table1', columns: [{ name: 'a', type: 'string' }] }] }
245245
})
246246
};
247-
} else if (url === `${baseUrl}/migrations/history` && request.method === 'GET') {
247+
} else if (url === `${baseUrl}/migrations/history?limit=200` && request.method === 'GET') {
248248
return {
249249
ok: true,
250250
json: async () => (type === 'inferred' ? { migrations: [pgrollMigration3] } : { migrations: [pgrollMigration1] })
@@ -263,7 +263,7 @@ const pgrollFetchEmpty = (url: string, request: any) => {
263263
schema: { tables: [{ name: 'table1', columns: [{ name: 'a', type: 'string' }] }] }
264264
})
265265
};
266-
} else if (url === `${baseUrl}/migrations/history` && request.method === 'GET') {
266+
} else if (url === `${baseUrl}/migrations/history?limit=200` && request.method === 'GET') {
267267
return {
268268
ok: true,
269269
json: async () => ({

‎cli/src/migrations/files.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ export async function removeLocalMigrations() {
9797
}
9898
}
9999

100-
export function commitToMigrationFile(logs: Schemas.Commit[] | Schemas.MigrationHistoryItem[]): LocalMigrationFile[] {
100+
export function commitToMigrationFile(logs: (Schemas.Commit | Schemas.MigrationHistoryItem)[]): LocalMigrationFile[] {
101101
// Schema history comes in reverse order, so we need to reverse it
102102
return logs.reverse().map((log) =>
103103
isMigrationPgRollFormat(log)

‎openapi-codegen.config.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,13 @@ export default defineConfig({
4343
to: '#/components/schemas/XataRecord'
4444
});
4545

46+
// Avoid conflict with duplicated `PageSize` type
47+
context.openAPIDocument = renameComponent({
48+
openAPIDocument: context.openAPIDocument,
49+
from: '#/components/schemas/PageSize',
50+
to: '#/components/schemas/PaginationPageSize'
51+
});
52+
4653
context.openAPIDocument = removeDeprecatedObjectType({ openAPIDocument: context.openAPIDocument });
4754

4855
// Inject path param in all requests (for now, this should be server url variables)

‎packages/client/src/api/dataPlaneComponents.ts‎

Lines changed: 107 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,98 @@ export const startMigration = (variables: StartMigrationVariables, signal?: Abor
124124
signal
125125
});
126126

127+
export type CompleteMigrationPathParams = {
128+
/**
129+
* The DBBranchName matches the pattern `{db_name}:{branch_name}`.
130+
*/
131+
dbBranchName: Schemas.DBBranchName;
132+
workspace: string;
133+
region: string;
134+
};
135+
136+
export type CompleteMigrationError = Fetcher.ErrorWrapper<
137+
| {
138+
status: 400;
139+
payload: Responses.BadRequestError;
140+
}
141+
| {
142+
status: 401;
143+
payload: Responses.AuthError;
144+
}
145+
| {
146+
status: 404;
147+
payload: Responses.SimpleError;
148+
}
149+
>;
150+
151+
export type CompleteMigrationVariables = {
152+
pathParams: CompleteMigrationPathParams;
153+
} & DataPlaneFetcherExtraProps;
154+
155+
/**
156+
* Complete an active migration on the specified database
157+
*/
158+
export const completeMigration = (variables: CompleteMigrationVariables, signal?: AbortSignal) =>
159+
dataPlaneFetch<
160+
Schemas.CompleteMigrationResponse,
161+
CompleteMigrationError,
162+
undefined,
163+
{},
164+
{},
165+
CompleteMigrationPathParams
166+
>({
167+
url: '/db/{dbBranchName}/migrations/complete',
168+
method: 'post',
169+
...variables,
170+
signal
171+
});
172+
173+
export type RollbackMigrationPathParams = {
174+
/**
175+
* The DBBranchName matches the pattern `{db_name}:{branch_name}`.
176+
*/
177+
dbBranchName: Schemas.DBBranchName;
178+
workspace: string;
179+
region: string;
180+
};
181+
182+
export type RollbackMigrationError = Fetcher.ErrorWrapper<
183+
| {
184+
status: 400;
185+
payload: Responses.BadRequestError;
186+
}
187+
| {
188+
status: 401;
189+
payload: Responses.AuthError;
190+
}
191+
| {
192+
status: 404;
193+
payload: Responses.SimpleError;
194+
}
195+
>;
196+
197+
export type RollbackMigrationVariables = {
198+
pathParams: RollbackMigrationPathParams;
199+
} & DataPlaneFetcherExtraProps;
200+
201+
/**
202+
* Roll back an active migration on the specified database
203+
*/
204+
export const rollbackMigration = (variables: RollbackMigrationVariables, signal?: AbortSignal) =>
205+
dataPlaneFetch<
206+
Schemas.RollbackMigrationResponse,
207+
RollbackMigrationError,
208+
undefined,
209+
{},
210+
{},
211+
RollbackMigrationPathParams
212+
>({
213+
url: '/db/{dbBranchName}/migrations/rollback',
214+
method: 'post',
215+
...variables,
216+
signal
217+
});
218+
127219
export type AdaptTablePathParams = {
128220
/**
129221
* The DBBranchName matches the pattern `{db_name}:{branch_name}`.
@@ -305,6 +397,17 @@ export type GetMigrationHistoryPathParams = {
305397
region: string;
306398
};
307399

400+
export type GetMigrationHistoryQueryParams = {
401+
/**
402+
* @format date-time
403+
*/
404+
cursor?: string;
405+
/**
406+
* Page size
407+
*/
408+
limit?: Schemas.PaginationPageSize;
409+
};
410+
308411
export type GetMigrationHistoryError = Fetcher.ErrorWrapper<
309412
| {
310413
status: 400;
@@ -322,6 +425,7 @@ export type GetMigrationHistoryError = Fetcher.ErrorWrapper<
322425

323426
export type GetMigrationHistoryVariables = {
324427
pathParams: GetMigrationHistoryPathParams;
428+
queryParams?: GetMigrationHistoryQueryParams;
325429
} & DataPlaneFetcherExtraProps;
326430

327431
export const getMigrationHistory = (variables: GetMigrationHistoryVariables, signal?: AbortSignal) =>
@@ -330,7 +434,7 @@ export const getMigrationHistory = (variables: GetMigrationHistoryVariables, sig
330434
GetMigrationHistoryError,
331435
undefined,
332436
{},
333-
{},
437+
GetMigrationHistoryQueryParams,
334438
GetMigrationHistoryPathParams
335439
>({
336440
url: '/db/{dbBranchName}/migrations/history',
@@ -5049,6 +5153,8 @@ export const operationsByTag = {
50495153
migrations: {
50505154
applyMigration,
50515155
startMigration,
5156+
completeMigration,
5157+
rollbackMigration,
50525158
adaptTable,
50535159
adaptAllTables,
50545160
getBranchMigrationJobStatus,

‎packages/client/src/api/dataPlaneParameters.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,13 @@ export type TableNameParam = Schemas.TableName;
1111

1212
export type MigrationJobIDParam = Schemas.MigrationJobID;
1313

14+
/**
15+
* @format date-time
16+
*/
17+
export type TimestampCursorParam = string;
18+
19+
export type LimitParam = Schemas.PaginationPageSize;
20+
1421
export type DBNameParam = Schemas.DBName;
1522

1623
export type MigrationRequestNumberParam = Schemas.MigrationRequestNumber;

0 commit comments

Comments
 (0)