N
P
AN
H
M
P
.NHNNHNHNNNHNNN
crawler.addRequestscrawler.addRequestsenqueueLinksenqueueLinkscrawler.addRequestconst mcmasterQueue2 = await RequestQueue.open('mcmaster2');
const mcmasterQueue3 = await RequestQueue.open('mcmaster3');
const mcmasterQueue4 = await RequestQueue.open('mcmaster4');
const mcmasterQueue5 = await RequestQueue.open('mcmaster5');
const mcmasterQueue6 = await RequestQueue.open('mcmaster6');
const requestQueues = [
mcmasterQueue2,
mcmasterQueue3,
mcmasterQueue4,
mcmasterQueue5,
mcmasterQueue6
];
router.addDefaultHandler(async ({ enqueueLinks, log, page, request, parseWithCheerio, crawler }) => {
totalRequestCount += crawler.requestQueue.assumedTotalCount
console.log(totalRequestCount)
// console.log(`Total request count: ${totalRequestCount}`)
let queueIndex = 0;
if (totalRequestCount > 15000000 ) {
totalRequestCount = 0
// Select the next queue to use
let newQueue = requestQueues[queueIndex]
crawler.requestQueue = newQueue;
requestQueues.splice(0, 1)
// Use the selected queue
// console.log(`Using ${requestQueues[queueIndex]}`);
console.log(crawler.requestQueue.name)
console.log(totalRequestCount)
// Check if we've reached the final queue
if (queueIndex === requestQueues.length - 1) {
console.log("Reached the final queue");
}
}