infinite scrolling
trying to get infinite scrolling to render in all products while scraping them as the page is being scrolled down
i looked at the documentation but didnt understand how to do this:
i looked at the documentation but didnt understand how to do this:
kotnRouter.addHandler('KOTN_DETAIL', async ({ log, page, parseWithCheerio }) => {
log.info(`Scraping product URLs`);
const $ = await parseWithCheerio()
const productUrls: string[] = [];
$('a').each((_, el) => {
let productUrl = $(el).attr('href');
if (productUrl) {
if (!productUrl.startsWith('https://')) {
productUrl = 'https://www.kotn.com' + productUrl;
if(productUrl.includes('/products')){
productUrls.push(productUrl);
}
}
}
});
// Push unique URLs to the dataset
const uniqueProductUrls = Array.from(new Set(productUrls));
await Dataset.pushData({
urls: uniqueProductUrls,
});
await Promise.all(uniqueProductUrls.map(link => kotnPw.addRequests([{ url: link, label: 'KOTN_PRODUCT' }])));
linksCount += uniqueProductUrls.length;
await infiniteScroll(page, {
maxScrollHeight: 0,
});
console.log(uniqueProductUrls);
console.log(`Total product links scraped so far: ${linksCount}`);
// Run bronPuppet crawler once after pushing the first product requests
if (linksCount === uniqueProductUrls.length) {
await kotnPw.run();
}
});kotnRouter.addHandler('KOTN_DETAIL', async ({ log, page, parseWithCheerio }) => {
log.info(`Scraping product URLs`);
const $ = await parseWithCheerio()
const productUrls: string[] = [];
$('a').each((_, el) => {
let productUrl = $(el).attr('href');
if (productUrl) {
if (!productUrl.startsWith('https://')) {
productUrl = 'https://www.kotn.com' + productUrl;
if(productUrl.includes('/products')){
productUrls.push(productUrl);
}
}
}
});
// Push unique URLs to the dataset
const uniqueProductUrls = Array.from(new Set(productUrls));
await Dataset.pushData({
urls: uniqueProductUrls,
});
await Promise.all(uniqueProductUrls.map(link => kotnPw.addRequests([{ url: link, label: 'KOTN_PRODUCT' }])));
linksCount += uniqueProductUrls.length;
await infiniteScroll(page, {
maxScrollHeight: 0,
});
console.log(uniqueProductUrls);
console.log(`Total product links scraped so far: ${linksCount}`);
// Run bronPuppet crawler once after pushing the first product requests
if (linksCount === uniqueProductUrls.length) {
await kotnPw.run();
}
});
