handle retrys and hasMore

Author: disnet

src/lib/services/api.ts

@@ -10,6 +10,16 @@ import type {
 
 const API_BASE = import.meta.env.VITE_API_URL || 'http://127.0.0.1:8787';
 
+export class RateLimitError extends Error {
+	retryAfter: number;
+
+	constructor(retryAfter: number) {
+		super(`Rate limit exceeded. Try again in ${retryAfter} seconds.`);
+		this.name = 'RateLimitError';
+		this.retryAfter = retryAfter;
+	}
+}
+
 class ApiClient {
 	private onUnauthorized: (() => void) | null = null;
 
@@ -41,6 +51,12 @@ class ApiClient {
 				throw new Error('Session expired');
 			}
 
+			// Handle 429 - rate limit exceeded
+			if (response.status === 429) {
+				const retryAfter = parseInt(response.headers.get('Retry-After') || '60', 10);
+				throw new RateLimitError(retryAfter);
+			}
+
 			const error = await response.json().catch(() => ({ error: 'Request failed' }));
 			throw new Error((error as { error: string }).error || `HTTP ${response.status}`);
 		}
@@ -440,23 +456,17 @@ class ApiClient {
 			pushedToPds: number;
 			skipped: number;
 			warnings: string[];
-		};
-		readPositions?: {
-			success: boolean;
-			pulledFromPds: number;
-			pushedToPds: number;
-			skipped: number;
-			warnings: string[];
+			hasMore?: boolean;
 		};
 		error?: string;
+		hasMore?: boolean;
 	}> {
 		return this.fetch('/api/sync/full', { method: 'POST' });
 	}
 
 	async getSyncStatus(): Promise<{
 		pdsSyncEnabled: boolean;
 		lastSyncSubscriptions: number | null;
-		lastSyncReadPositions: number | null;
 	}> {
 		return this.fetch('/api/sync/status');
 	}

src/routes/settings/+page.svelte

@@ -11,7 +11,7 @@
 	import ImportOPMLModal from '$lib/components/ImportOPMLModal.svelte';
 	import PageHeader from '$lib/components/common/PageHeader.svelte';
 	import { downloadOPML } from '$lib/utils/opml-exporter';
-	import { api } from '$lib/services/api';
+	import { api, RateLimitError } from '$lib/services/api';
 
 	const fontOptions: { value: ArticleFont; label: string }[] = [
 		{ value: 'sans-serif', label: 'Sans Serif' },
@@ -32,7 +32,6 @@
 	// PDS Sync state
 	let pdsSyncEnabled = $state(false);
 	let lastSyncSubscriptions = $state<number | null>(null);
-	let lastSyncReadPositions = $state<number | null>(null);
 	let isSyncLoading = $state(false);
 	let isSyncing = $state(false);
 	let syncError = $state<string | null>(null);
@@ -58,7 +57,6 @@
 			const settings = await api.getSettings();
 			pdsSyncEnabled = settings.pdsSyncEnabled;
 			lastSyncSubscriptions = settings.lastPdsSyncSubscriptions;
-			lastSyncReadPositions = settings.lastPdsSyncReadPositions;
 		} catch (error) {
 			console.error('Failed to load sync settings:', error);
 		} finally {
@@ -89,43 +87,81 @@
 		}
 	}
 
+	function sleep(ms: number): Promise<void> {
+		return new Promise((resolve) => setTimeout(resolve, ms));
+	}
+
 	async function handleSync() {
 		if (isSyncing) return;
 
 		isSyncing = true;
 		syncError = null;
 		syncSuccess = null;
 
+		// Track totals across multiple sync calls (for batched hasMore syncs)
+		let totalPulled = 0;
+		let totalPushed = 0;
+		let allWarnings: string[] = [];
+		let batchCount = 0;
+		const maxBatches = 50; // Safety limit to prevent infinite loops
+
 		try {
-			const result = await api.triggerFullSync();
-
-			if (result.success) {
-				const subsPulled = result.subscriptions?.pulledFromPds || 0;
-				const subsPushed = result.subscriptions?.pushedToPds || 0;
-				const readPulled = result.readPositions?.pulledFromPds || 0;
-				const readPushed = result.readPositions?.pushedToPds || 0;
-
-				syncSuccess = `Sync complete: ${subsPulled + readPulled} pulled, ${subsPushed + readPushed} pushed`;
-
-				// Show warnings if any
-				const warnings = [
-					...(result.subscriptions?.warnings || []),
-					...(result.readPositions?.warnings || []),
-				];
-				if (warnings.length > 0) {
-					syncSuccess += `. Warning: ${warnings.join(', ')}`;
+			let hasMore = true;
+
+			while (hasMore && batchCount < maxBatches) {
+				batchCount++;
+				if (batchCount > 1) {
+					syncSuccess = `Syncing batch ${batchCount}...`;
+				}
+
+				let result;
+				try {
+					result = await api.triggerFullSync();
+				} catch (error) {
+					// Handle rate limit by waiting and retrying
+					if (error instanceof RateLimitError) {
+						const waitSeconds = Math.min(error.retryAfter, 300); // Cap at 5 minutes
+						syncSuccess = `Rate limit reached. Waiting ${waitSeconds}s before continuing...`;
+						await sleep(waitSeconds * 1000);
+						// Retry this batch
+						batchCount--;
+						continue;
+					}
+					throw error;
+				}
+
+				if (!result.success) {
+					syncError = result.error || 'Sync failed';
+					return;
 				}
 
-				// Refresh sync status
-				const status = await api.getSyncStatus();
-				lastSyncSubscriptions = status.lastSyncSubscriptions;
-				lastSyncReadPositions = status.lastSyncReadPositions;
+				// Accumulate totals
+				totalPulled += result.subscriptions?.pulledFromPds || 0;
+				totalPushed += result.subscriptions?.pushedToPds || 0;
 
-				// Reload subscriptions to show any pulled items
-				await subscriptionsStore.load();
-			} else {
-				syncError = result.error || 'Sync failed';
+				// Collect warnings
+				allWarnings = [...allWarnings, ...(result.subscriptions?.warnings || [])];
+
+				// Check if there's more to sync
+				hasMore = result.hasMore || false;
+			}
+
+			syncSuccess = `Sync complete: ${totalPulled} pulled, ${totalPushed} pushed`;
+			if (batchCount > 1) {
+				syncSuccess += ` (${batchCount} batches)`;
+			}
+
+			// Show warnings if any
+			if (allWarnings.length > 0) {
+				syncSuccess += `. Warning: ${allWarnings.join(', ')}`;
 			}
+
+			// Refresh sync status
+			const status = await api.getSyncStatus();
+			lastSyncSubscriptions = status.lastSyncSubscriptions;
+
+			// Reload subscriptions to show any pulled items
+			await subscriptionsStore.load();
 		} catch (error) {
 			console.error('Sync error:', error);
 			syncError = error instanceof Error ? error.message : 'Sync failed';
@@ -211,11 +247,12 @@
 			<div class="sync-toggle-section">
 				<label class="toggle-setting">
 					<input type="checkbox" checked={pdsSyncEnabled} onchange={handleTogglePdsSync} />
-					<span>Also sync subscriptions and reading data</span>
+					<span>Also sync subscriptions</span>
 				</label>
 				<p class="setting-description">
-					Optionally store your feed subscriptions and read/starred articles in your PDS. Note: this
-					data will be <strong>publicly visible</strong> on your PDS, but gives you full backup and portability.
+					Optionally store your feed subscriptions in your PDS. Note: this data will be <strong
+						>publicly visible</strong
+					> on your PDS, but gives you full backup and portability.
 				</p>
 			</div>
 
@@ -224,7 +261,6 @@
 					<p class="sync-time">
 						Subscriptions last synced: {formatSyncTime(lastSyncSubscriptions)}
 					</p>
-					<p class="sync-time">Reading data last synced: {formatSyncTime(lastSyncReadPositions)}</p>
 				</div>
 
 				<button class="btn btn-secondary" onclick={handleSync} disabled={isSyncing}>