splitAndExecute({
callback: async (urlBatch, batchIndex) => {
// log that we are enqueuing the nth batch of preview jobs from job-id etc
logger.info(`Linkedin/Scraper - Enqueuing ${urlBatch.length} jobs to detail page handler - Batch ${batchIndex + 1}`);
await enqueueLinks({
urls: urlBatch,
label: LinkedinRouterLabels.JOB_DETAIL_PAGE,
userData: createLinkedinRouterUserData(payload),
waitForAllRequestsToBeAdded: false
});
const minSleepTime = 2000 * (batchIndex + 1);
const maxSleepTime = 3000 * (batchIndex + 1);
await random_sleep(minSleepTime, maxSleepTime);
},
urls: jobDetailPageUrls,
maxRequestsPerBatch: 2,
});
splitAndExecute({
callback: async (urlBatch, batchIndex) => {
// log that we are enqueuing the nth batch of preview jobs from job-id etc
logger.info(`Linkedin/Scraper - Enqueuing ${urlBatch.length} jobs to detail page handler - Batch ${batchIndex + 1}`);
await enqueueLinks({
urls: urlBatch,
label: LinkedinRouterLabels.JOB_DETAIL_PAGE,
userData: createLinkedinRouterUserData(payload),
waitForAllRequestsToBeAdded: false
});
const minSleepTime = 2000 * (batchIndex + 1);
const maxSleepTime = 3000 * (batchIndex + 1);
await random_sleep(minSleepTime, maxSleepTime);
},
urls: jobDetailPageUrls,
maxRequestsPerBatch: 2,
});