adjust migration

Author: emilwidlund

src/cli.ts

@@ -3,6 +3,7 @@ import { BunContext, BunRuntime } from "@effect/platform-bun";
 import { Effect, Layer } from "effect";
 import { listen } from "./commands/listen";
 import { login } from "./commands/login";
+import { migrate } from "./commands/migrate";
 import { update } from "./commands/update";
 import * as Migration from "./services/migration/migrate";
 import * as OAuth from "./services/oauth";
@@ -13,7 +14,7 @@ import { VERSION } from "./version";
 const mainCommand = Command.make("polar").pipe(
   Command.withSubcommands([
     login,
-    // migrate, Disabled until ready for now
+    migrate,
     listen,
     update
   ])

src/prompts/migration.ts

@@ -38,7 +38,7 @@ export const storePrompt = (provider: LemonSqueezy.LemonSqueezyImpl) =>
 
     return yield* Prompt.select({
       message: "Select Store to Migrate",
-      choices: stores.data.map((store) => ({
+      choices: stores.map((store) => ({
         value: store.id,
         title: store.attributes.name,
       })),

src/services/migration/lemon/provider.ts

@@ -12,7 +12,7 @@ import {
 import { Context, Data, Effect, Layer, Schema } from "effect";
 import { CustomerCreate } from "../../../schemas/Customer";
 import { ProductCreate } from "../../../schemas/Product";
-import { fetchAllPages } from "../../../utils";
+import { fetchAllPages, promiseAllInBatches } from "../../../utils";
 import { parseCustomers, parseVariants } from "./transform";
 
 export class LemonSqueezyError extends Data.TaggedError("LemonSqueezyError")<{
@@ -21,7 +21,11 @@ export class LemonSqueezyError extends Data.TaggedError("LemonSqueezyError")<{
 }> {}
 
 export interface LemonSqueezyImpl {
-  stores: () => Effect.Effect<ListStores, LemonSqueezyError, never>;
+  stores: () => Effect.Effect<
+    ListStores["data"],
+    LemonSqueezyError,
+    never
+  >;
   customers: (
     storeId: string
   ) => Effect.Effect<readonly CustomerCreate[], LemonSqueezyError, never>;
@@ -43,11 +47,16 @@ export const make = (apiKey: string) =>
       stores: () =>
         Effect.tryPromise({
           try: () =>
-            client
-              .listStores()
-              .then(
-                (response) => response.data ?? ([] as unknown as ListStores)
-              ),
+            fetchAllPages((pageNumber: number) =>
+              client
+                .listStores({
+                  page: { number: pageNumber, size: 50 },
+                })
+                .then((response) => ({
+                  data: response.data?.data,
+                  lastPage: response.data?.meta.page.lastPage ?? 1,
+                }))
+            ),
           catch: (error) =>
             new LemonSqueezyError({
               message: "Failed to list Lemon Squeezy stores",
@@ -93,22 +102,41 @@ export const make = (apiKey: string) =>
               })
           )
         ),
-      products: (_storeId: string) =>
+      products: (storeId: string) =>
         Effect.tryPromise({
           try: async () => {
-            return await fetchAllPages((pageNumber: number) =>
+            // Fetch all products for this store
+            const products = await fetchAllPages((pageNumber: number) =>
               client
-                .listVariants({
-                  page: {
-                    number: pageNumber,
-                    size: 50,
-                  },
+                .listProducts({
+                  filter: { storeId },
+                  page: { number: pageNumber, size: 50 },
                 })
                 .then((response) => ({
                   data: response.data?.data,
                   lastPage: response.data?.meta.page.lastPage ?? 1,
                 }))
             );
+
+            // Fetch variants for each product in batches to respect rate limits
+            const variantGroups = await promiseAllInBatches(
+              (product) =>
+                fetchAllPages((pageNumber: number) =>
+                  client
+                    .listVariants({
+                      filter: { productId: product.id },
+                      page: { number: pageNumber, size: 50 },
+                    })
+                    .then((response) => ({
+                      data: response.data?.data,
+                      lastPage: response.data?.meta.page.lastPage ?? 1,
+                    }))
+                ),
+              products,
+              5
+            );
+
+            return variantGroups.flat();
           },
           catch: (error) =>
             new LemonSqueezyError({

src/utils.ts

@@ -39,22 +39,61 @@ export async function promiseAllInBatches<A, B>(
   return results;
 }
 
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const withRetry = async <T>(
+  fn: () => Promise<T>,
+  maxRetries = 5,
+  baseDelay = 1000
+): Promise<T> => {
+  let lastError: unknown;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error;
+      if (attempt < maxRetries) {
+        const delay = baseDelay * Math.pow(2, attempt);
+        await sleep(delay);
+      }
+    }
+  }
+  throw lastError;
+};
+
 export const fetchAllPages = async <T>(
   task: (pageNumber: number) => Promise<{
     data?: T[];
     lastPage: number;
-  }>
+  }>,
+  options: { batchSize?: number; delayMs?: number } = {}
 ): Promise<T[]> => {
-  const { data, lastPage } = await task(1);
-  const allItems: T[] = [...(data ?? [])];
+  const { batchSize = 5, delayMs = 250 } = options;
 
-  if (!data) {
+  const firstPage = await withRetry(() => task(1));
+  const allItems: T[] = [...(firstPage.data ?? [])];
+
+  if (!firstPage.data || firstPage.lastPage <= 1) {
     return allItems;
   }
 
-  const results = await Promise.all(
-    Array.from({ length: lastPage - 1 }, (_, i) => task(i + 2))
+  const remainingPages = Array.from(
+    { length: firstPage.lastPage - 1 },
+    (_, i) => i + 2
   );
 
-  return [...allItems, ...results.flatMap((result) => result.data ?? [])];
+  for (let i = 0; i < remainingPages.length; i += batchSize) {
+    const batch = remainingPages.slice(i, i + batchSize);
+    const results = await Promise.all(
+      batch.map((pageNumber) => withRetry(() => task(pageNumber)))
+    );
+    for (const result of results) {
+      allItems.push(...(result.data ?? []));
+    }
+    if (i + batchSize < remainingPages.length) {
+      await sleep(delayMs);
+    }
+  }
+
+  return allItems;
 };