await page.$eval('main', (main) => {
main.querySelector('.c-endless-paginator__button').click();
})
await page.waitForTimeout(5000);
const products = await page.$$eval('.c-product-list-container-products__item', ($products) => {
const scrapedUrls = [];
$products.forEach(async ($product) => {
await scrapedUrls.push({
url: $product.querySelector('.c-product-list-item--grid__title').href
});
});
return scrapedUrls;
});
for (let i = 0; i < products.length; i++) {
await requestQueue.addRequest({
url: products[i].url,
userData: {
depth: request.userData.depth + 1
}
})
}
await page.$eval('main', (main) => {
main.querySelector('.c-endless-paginator__button').click();
})
await page.waitForTimeout(5000);
const products = await page.$$eval('.c-product-list-container-products__item', ($products) => {
const scrapedUrls = [];
$products.forEach(async ($product) => {
await scrapedUrls.push({
url: $product.querySelector('.c-product-list-item--grid__title').href
});
});
return scrapedUrls;
});
for (let i = 0; i < products.length; i++) {
await requestQueue.addRequest({
url: products[i].url,
userData: {
depth: request.userData.depth + 1
}
})
}