Skip to content
This repository has been archived by the owner on Mar 10, 2024. It is now read-only.

Commit

Permalink
fix: handle pagination errors correctly
Browse files Browse the repository at this point in the history
  • Loading branch information
lucasmarshall committed Dec 19, 2023
1 parent e789ff0 commit 845661a
Showing 1 changed file with 8 additions and 3 deletions.
11 changes: 8 additions & 3 deletions packages/core/remotes/utils/paginator.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { Readable } from 'stream';
import { PassThrough } from 'stream';
import { logger } from '../../lib/logger';

type PaginatorImpl<T> = (cursor?: string) => Promise<T>;

Expand Down Expand Up @@ -62,7 +63,7 @@ export async function paginator<T>(
response = await pageFetcher(cursor);
} catch (e: any) {
if (e.problemType === 'SG_TERMINAL_TOO_MANY_REQUESTS_ERROR') {
passThrough.emit('error', e);
passThrough.destroy(e);
return;
}
throw e;
Expand All @@ -72,13 +73,17 @@ export async function paginator<T>(
cursor = getNextCursorFromPage(response);

readable.pipe(passThrough, { end: index === lastIndex && !cursor });
readable.on('error', (err) => passThrough.emit('error', err));
readable.on('error', (err) => passThrough.destroy(err));

await new Promise((resolve) => readable.on('end', resolve));
} while (cursor);
}
})().catch((err) => {
passThrough.emit('error', err);
passThrough.destroy(err);
});

passThrough.on('error', (err) => {
logger.error({ err }, 'Error in paginator stream');
});

return passThrough;
Expand Down

0 comments on commit 845661a

Please sign in to comment.