7 replies

infinite scrolling of pages

i have a crawler that goes through collection pages of stores

and scrapes their product links

and goes through those product page links to get product data

when getting the product links in the collection pages,

many sites utilize an infinite scrolling to render in all the products

how do i implement infinite scrolling into this specific crawler route handler here below

while scraping the product page urls to render in all the products to make sure i scraped all the products on the page:

kotnRouter.addHandler('KOTN_DETAIL', async ({ page, log }) => {
    log.info('Scraping product URLs');
    
    await page.goto(page.url(), { waitUntil: 'domcontentloaded' })

  
    const productUrls: string[] = [];
  
    const links = await page.$$eval('a', (elements) =>
      elements.map((el) => el.getAttribute('href'))
    );
  
    for (const link of links) {
      if (link && !link.startsWith('https://')) {
        const productUrl = 'https://www.kotn.com' + link;
        if (productUrl.includes('/products')) {
          productUrls.push(productUrl);
        }
      }
    }
  
    // Push unique URLs to the dataset
    const uniqueProductUrls = Array.from(new Set(productUrls));
    console.log(uniqueProductUrls);
    await Dataset.pushData({
      urls: uniqueProductUrls,
    });
  
    await Promise.all(
      uniqueProductUrls.map((link) => kotnCrawler.addRequests([{ url: link, label: 'KOTN_PRODUCT' }]))
    );
  
    linksCount += uniqueProductUrls.length;
  
    console.log(uniqueProductUrls);
    console.log(`Total product links scraped so far: ${linksCount}`);
  
});
z

kotnRouter.addHandler('KOTN_DETAIL', async ({ page, log }) => {
    log.info('Scraping product URLs');
    
    await page.goto(page.url(), { waitUntil: 'domcontentloaded' })

  
    const productUrls: string[] = [];
  
    const links = await page.$$eval('a', (elements) =>
      elements.map((el) => el.getAttribute('href'))
    );
  
    for (const link of links) {
      if (link && !link.startsWith('https://')) {
        const productUrl = 'https://www.kotn.com' + link;
        if (productUrl.includes('/products')) {
          productUrls.push(productUrl);
        }
      }
    }
  
    // Push unique URLs to the dataset
    const uniqueProductUrls = Array.from(new Set(productUrls));
    console.log(uniqueProductUrls);
    await Dataset.pushData({
      urls: uniqueProductUrls,
    });
  
    await Promise.all(
      uniqueProductUrls.map((link) => kotnCrawler.addRequests([{ url: link, label: 'KOTN_PRODUCT' }]))
    );
  
    linksCount += uniqueProductUrls.length;
  
    console.log(uniqueProductUrls);
    console.log(`Total product links scraped so far: ${linksCount}`);
  
});
z

infinite scrolling of pages

Similar Threads