Skip to content

Commit 7ceae06

Browse files
bealqiuclaude
andcommitted
fix(sync): reduce concurrency for WebDAV and retry on 401/403
Some WebDAV servers (e.g. fnOS) fail with 401 under concurrent requests due to poor session handling. This change: - Reduces WebDAV upload concurrency from 5 to 2 - Reduces WebDAV download concurrency from 8 to 2 - Adds automatic single retry with 800ms delay on 401/403 errors - S3 backends keep the original higher concurrency Fixes #132 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent cbedab3 commit 7ceae06

1 file changed

Lines changed: 31 additions & 11 deletions

File tree

packages/core/src/sync/sync-files.ts

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,20 @@ export interface SyncFilesOptions {
1616
disableRemoteDeletes?: boolean;
1717
}
1818

19+
/** Retry an async operation once after a short delay if it fails with a retryable error (401/403) */
20+
async function withRetry<T>(fn: () => Promise<T>, delayMs = 800): Promise<T> {
21+
try {
22+
return await fn();
23+
} catch (e: unknown) {
24+
const msg = e instanceof Error ? e.message : String(e);
25+
if (msg.includes("401") || msg.includes("403")) {
26+
await new Promise((r) => setTimeout(r, delayMs));
27+
return fn();
28+
}
29+
throw e;
30+
}
31+
}
32+
1933
function isAbsoluteOrProtocolPath(path: string): boolean {
2034
return (
2135
path.startsWith("/") ||
@@ -153,7 +167,7 @@ export async function syncFiles(
153167
console.log(`[Sync] 📤 Uploading book: ${bookTitle} (${remoteName})`);
154168
const data = await adapter.readFileBytes(localPath);
155169
const sizeMB = (data.length / 1024 / 1024).toFixed(2);
156-
await backend.put(`${REMOTE_FILES}/${remoteName}`, data);
170+
await withRetry(() => backend.put(`${REMOTE_FILES}/${remoteName}`, data));
157171
console.log(
158172
`[Sync] ✓ Uploaded "${bookTitle}" (${sizeMB} MB) in ${Date.now() - taskStart}ms`,
159173
);
@@ -171,7 +185,7 @@ export async function syncFiles(
171185
const bookTitle = book.title || "未知书籍";
172186
try {
173187
console.log(`[Sync] 📥 Downloading book: ${bookTitle} (${remoteName})`);
174-
const data = await backend.get(`${REMOTE_FILES}/${remoteName}`);
188+
const data = await withRetry(() => backend.get(`${REMOTE_FILES}/${remoteName}`));
175189
const sizeMB = (data.length / 1024 / 1024).toFixed(2);
176190
const dir = localPath.substring(0, localPath.lastIndexOf("/"));
177191
if (dir) await adapter.ensureDir(dir);
@@ -218,7 +232,7 @@ export async function syncFiles(
218232
console.log(`[Sync] 📤 Uploading cover: ${bookTitle} (${coverRemoteName})`);
219233
const data = await adapter.readFileBytes(coverLocalPath);
220234
const sizeKB = (data.length / 1024).toFixed(2);
221-
await backend.put(`${REMOTE_COVERS}/${coverRemoteName}`, data);
235+
await withRetry(() => backend.put(`${REMOTE_COVERS}/${coverRemoteName}`, data));
222236
console.log(
223237
`[Sync] ✓ Uploaded cover "${bookTitle}" (${sizeKB} KB) in ${Date.now() - taskStart}ms`,
224238
);
@@ -236,7 +250,7 @@ export async function syncFiles(
236250
const bookTitle = book.title || "未知书籍";
237251
try {
238252
console.log(`[Sync] 📥 Downloading cover: ${bookTitle} (${coverRemoteName})`);
239-
const data = await backend.get(`${REMOTE_COVERS}/${coverRemoteName}`);
253+
const data = await withRetry(() => backend.get(`${REMOTE_COVERS}/${coverRemoteName}`));
240254
const sizeKB = (data.length / 1024).toFixed(2);
241255
const dir = coverLocalPath.substring(0, coverLocalPath.lastIndexOf("/"));
242256
if (dir) await adapter.ensureDir(dir);
@@ -260,9 +274,15 @@ export async function syncFiles(
260274
`upload: ${uploadTasks.length}, download: ${downloadTasks.length}`,
261275
);
262276

263-
// Execute uploads in parallel (limit: 5 concurrent)
277+
// Concurrency limits: WebDAV servers (especially NAS devices like fnOS)
278+
// often fail with 401 under concurrent requests due to poor session handling.
279+
const isWebDav = backend.type === "webdav";
280+
const uploadConcurrency = isWebDav ? 2 : 5;
281+
const downloadConcurrency = isWebDav ? 2 : 8;
282+
283+
// Execute uploads in parallel (limit: uploadConcurrency concurrent)
264284
if (uploadTasks.length > 0) {
265-
console.log(`[Sync] 📤 Starting upload of ${uploadTasks.length} files (5 concurrent)...`);
285+
console.log(`[Sync] 📤 Starting upload of ${uploadTasks.length} files (${uploadConcurrency} concurrent)...`);
266286
const uploadStart = Date.now();
267287
let completed = 0;
268288
const total = uploadTasks.length;
@@ -279,17 +299,17 @@ export async function syncFiles(
279299
completed++;
280300
return result;
281301
});
282-
const uploadResults = await parallelLimit(tasksWithProgress, 5);
302+
const uploadResults = await parallelLimit(tasksWithProgress, uploadConcurrency);
283303
filesUploaded = uploadResults.filter((r) => r).length;
284304
const uploadFailed = uploadResults.length - filesUploaded;
285305
console.log(
286306
`[Sync] ✅ Upload completed: ${filesUploaded} succeeded, ${uploadFailed} failed in ${Date.now() - uploadStart}ms`,
287307
);
288308
}
289309

290-
// Execute downloads in parallel (limit: 8 concurrent)
310+
// Execute downloads in parallel (limit: downloadConcurrency concurrent)
291311
if (downloadTasks.length > 0) {
292-
console.log(`[Sync] 📥 Starting download of ${downloadTasks.length} files (8 concurrent)...`);
312+
console.log(`[Sync] 📥 Starting download of ${downloadTasks.length} files (${downloadConcurrency} concurrent)...`);
293313
const downloadStart = Date.now();
294314
let completed = 0;
295315
const total = downloadTasks.length;
@@ -306,7 +326,7 @@ export async function syncFiles(
306326
completed++;
307327
return result;
308328
});
309-
const downloadResults = await parallelLimit(tasksWithProgress, 8);
329+
const downloadResults = await parallelLimit(tasksWithProgress, downloadConcurrency);
310330
filesDownloaded = downloadResults.filter((r) => r).length;
311331
const downloadFailed = downloadResults.length - filesDownloaded;
312332
console.log(
@@ -437,7 +457,7 @@ export async function downloadBookFile(
437457
console.log(`[Sync] Downloading book file: ${remoteName}`);
438458
onProgress?.({ downloaded: 0, total: 100 });
439459

440-
const data = await backend.get(remotePath);
460+
const data = await withRetry(() => backend.get(remotePath));
441461
const sizeMB = (data.length / 1024 / 1024).toFixed(2);
442462
console.log(`[Sync] Downloaded ${remoteName} (${sizeMB} MB)`);
443463

0 commit comments

Comments
 (0)