const requestList = await RequestList.open('My-ReqList', allUrls, { persistStateKey: 'My-ReqList' });
console.log(requestList.length())
const crawler = new CheerioCrawler({
requestList,
proxyConfiguration,
requestHandler: router,
minConcurrency: 32,
maxConcurrency: 256,
maxRequestRetries: 20,
navigationTimeoutSecs: 6,
loggingInterval: 30,
useSessionPool: true,
failedRequestHandler({ request }) {
log.debug(`Request ${request.url} failed 20 times.`);
},
});
await crawler.run()
const requestList = await RequestList.open('My-ReqList', allUrls, { persistStateKey: 'My-ReqList' });
console.log(requestList.length())
const crawler = new CheerioCrawler({
requestList,
proxyConfiguration,
requestHandler: router,
minConcurrency: 32,
maxConcurrency: 256,
maxRequestRetries: 20,
navigationTimeoutSecs: 6,
loggingInterval: 30,
useSessionPool: true,
failedRequestHandler({ request }) {
log.debug(`Request ${request.url} failed 20 times.`);
},
});
await crawler.run()