infinite scrolling of pages

i have a crawler that goes through collection pages of stores

and scrapes their product links

and goes through those product page links to get product data

when getting the product links in the collection pages,

many sites utilize an infinite scrolling to render in all the products

how do i implement infinite scrolling into this specific crawler route handler here below

while scraping the product page urls to render in all the products to make sure i scraped all the products on the page:

kotnRouter.addHandler('KOTN_DETAIL', async ({ page, log }) => {
    log.info('Scraping product URLs');
    
    await page.goto(page.url(), { waitUntil: 'domcontentloaded' })

  
    const productUrls: string[] = [];
  
    const links = await page.$$eval('a', (elements) =>
      elements.map((el) => el.getAttribute('href'))
    );
  
    for (const link of links) {
      if (link && !link.startsWith('https://')) {
        const productUrl = 'https://www.kotn.com' + link;
        if (productUrl.includes('/products')) {
          productUrls.push(productUrl);
        }
      }
    }
  
    // Push unique URLs to the dataset
    const uniqueProductUrls = Array.from(new Set(productUrls));
    console.log(uniqueProductUrls);
    await Dataset.pushData({
      urls: uniqueProductUrls,
    });
  
    await Promise.all(
      uniqueProductUrls.map((link) => kotnCrawler.addRequests([{ url: link, label: 'KOTN_PRODUCT' }]))
    );
  
    linksCount += uniqueProductUrls.length;
  
    console.log(uniqueProductUrls);
    console.log(`Total product links scraped so far: ${linksCount}`);
  
});
z

kotnRouter.addHandler('KOTN_DETAIL', async ({ page, log }) => {
    log.info('Scraping product URLs');
    
    await page.goto(page.url(), { waitUntil: 'domcontentloaded' })

  
    const productUrls: string[] = [];
  
    const links = await page.$$eval('a', (elements) =>
      elements.map((el) => el.getAttribute('href'))
    );
  
    for (const link of links) {
      if (link && !link.startsWith('https://')) {
        const productUrl = 'https://www.kotn.com' + link;
        if (productUrl.includes('/products')) {
          productUrls.push(productUrl);
        }
      }
    }
  
    // Push unique URLs to the dataset
    const uniqueProductUrls = Array.from(new Set(productUrls));
    console.log(uniqueProductUrls);
    await Dataset.pushData({
      urls: uniqueProductUrls,
    });
  
    await Promise.all(
      uniqueProductUrls.map((link) => kotnCrawler.addRequests([{ url: link, label: 'KOTN_PRODUCT' }]))
    );
  
    linksCount += uniqueProductUrls.length;
  
    console.log(uniqueProductUrls);
    console.log(`Total product links scraped so far: ${linksCount}`);
  
});
z

Apify & Crawlee•3y ago•

7 replies

embarrassing-maroon

infinite scrolling of pages

kotnRouter.addHandler('KOTN_DETAIL', async ({ page, log }) => {
    log.info('Scraping product URLs');
    
    await page.goto(page.url(), { waitUntil: 'domcontentloaded' })

  
    const productUrls: string[] = [];
  
    const links = await page.$$eval('a', (elements) =>
      elements.map((el) => el.getAttribute('href'))
    );
  
    for (const link of links) {
      if (link && !link.startsWith('https://')) {
        const productUrl = 'https://www.kotn.com' + link;
        if (productUrl.includes('/products')) {
          productUrls.push(productUrl);
        }
      }
    }
  
    // Push unique URLs to the dataset
    const uniqueProductUrls = Array.from(new Set(productUrls));
    console.log(uniqueProductUrls);
    await Dataset.pushData({
      urls: uniqueProductUrls,
    });
  
    await Promise.all(
      uniqueProductUrls.map((link) => kotnCrawler.addRequests([{ url: link, label: 'KOTN_PRODUCT' }]))
    );
  
    linksCount += uniqueProductUrls.length;
  
    console.log(uniqueProductUrls);
    console.log(`Total product links scraped so far: ${linksCount}`);
  
});
z

kotnRouter.addHandler('KOTN_DETAIL', async ({ page, log }) => {
    log.info('Scraping product URLs');
    
    await page.goto(page.url(), { waitUntil: 'domcontentloaded' })

  
    const productUrls: string[] = [];
  
    const links = await page.$$eval('a', (elements) =>
      elements.map((el) => el.getAttribute('href'))
    );
  
    for (const link of links) {
      if (link && !link.startsWith('https://')) {
        const productUrl = 'https://www.kotn.com' + link;
        if (productUrl.includes('/products')) {
          productUrls.push(productUrl);
        }
      }
    }
  
    // Push unique URLs to the dataset
    const uniqueProductUrls = Array.from(new Set(productUrls));
    console.log(uniqueProductUrls);
    await Dataset.pushData({
      urls: uniqueProductUrls,
    });
  
    await Promise.all(
      uniqueProductUrls.map((link) => kotnCrawler.addRequests([{ url: link, label: 'KOTN_PRODUCT' }]))
    );
  
    linksCount += uniqueProductUrls.length;
  
    console.log(uniqueProductUrls);
    console.log(`Total product links scraped so far: ${linksCount}`);
  
});
z

Continue the conversation

Join the Discord to ask follow-up questions and connect with the community

A&C

Apify & Crawlee

This is the official developer community of Apify and Crawlee.

14,851 Members

Join

infinite scrolling of pages

infinite scrolling of pages

Continue the conversation

Apify & Crawlee

Continue the conversation

Apify & Crawlee

Similar Threads