5 replies

Navigation timed out after 60 seconds.

I'm scraping a website, if I run it in headless mode I get this error, if I run it headed I see the webpage completely loaded (the loading wheel still spins somehow).
this is my routes:

router.addDefaultHandler(async ({ enqueueLinks, log, page, request }) => {
    if (request.url.includes('/ayuda/')){
        await page.close()
        return;
    }
    if (request.url.includes('sitemaps.org')) {
        await page.close()
        return;
    }
    try {
        await page.waitForSelector('#cookies-agree');
        await page.click('#cookies-agree');
        log.info('cookies button clicked')
    } catch (error) {
        log.error('cookies button not available');
    }
    await page.setViewport({
        width: 1920,
        height: 6000,
    });

    let content = await page.content()
    let securityCheck = await checkForChallengeValidation(page, request, content, log)
    if (!securityCheck){
        throw RetryRequestError
    }

    await enqueueLinks({
        // regexps: [/(\/supermercado\/[a-zA-Z]+)/],
        selector: 'li._next > a._pagination_link',
        forefront: true,
    });
    await enqueueLinks({
        // regexps: [/(\/supermercado\/[0-9]+)/],
        selector: 'a.product_link',
        label: 'product',
        forefront: true,
    });

    await page.close()
    page = null;
});

router.addDefaultHandler(async ({ enqueueLinks, log, page, request }) => {
    if (request.url.includes('/ayuda/')){
        await page.close()
        return;
    }
    if (request.url.includes('sitemaps.org')) {
        await page.close()
        return;
    }
    try {
        await page.waitForSelector('#cookies-agree');
        await page.click('#cookies-agree');
        log.info('cookies button clicked')
    } catch (error) {
        log.error('cookies button not available');
    }
    await page.setViewport({
        width: 1920,
        height: 6000,
    });

    let content = await page.content()
    let securityCheck = await checkForChallengeValidation(page, request, content, log)
    if (!securityCheck){
        throw RetryRequestError
    }

    await enqueueLinks({
        // regexps: [/(\/supermercado\/[a-zA-Z]+)/],
        selector: 'li._next > a._pagination_link',
        forefront: true,
    });
    await enqueueLinks({
        // regexps: [/(\/supermercado\/[0-9]+)/],
        selector: 'a.product_link',
        label: 'product',
        forefront: true,
    });

    await page.close()
    page = null;
});

I'm scraping a website, if I run it in headless mode I get this error, if I run it headed I see the webpage completely loaded (the loading wheel still spins somehow).
this is my routes:

router.addDefaultHandler(async ({ enqueueLinks, log, page, request }) => {
    if (request.url.includes('/ayuda/')){
        await page.close()
        return;
    }
    if (request.url.includes('sitemaps.org')) {
        await page.close()
        return;
    }
    try {
        await page.waitForSelector('#cookies-agree');
        await page.click('#cookies-agree');
        log.info('cookies button clicked')
    } catch (error) {
        log.error('cookies button not available');
    }
    await page.setViewport({
        width: 1920,
        height: 6000,
    });

    let content = await page.content()
    let securityCheck = await checkForChallengeValidation(page, request, content, log)
    if (!securityCheck){
        throw RetryRequestError
    }

    await enqueueLinks({
        // regexps: [/(\/supermercado\/[a-zA-Z]+)/],
        selector: 'li._next > a._pagination_link',
        forefront: true,
    });
    await enqueueLinks({
        // regexps: [/(\/supermercado\/[0-9]+)/],
        selector: 'a.product_link',
        label: 'product',
        forefront: true,
    });

    await page.close()
    page = null;
});

router.addDefaultHandler(async ({ enqueueLinks, log, page, request }) => {
    if (request.url.includes('/ayuda/')){
        await page.close()
        return;
    }
    if (request.url.includes('sitemaps.org')) {
        await page.close()
        return;
    }
    try {
        await page.waitForSelector('#cookies-agree');
        await page.click('#cookies-agree');
        log.info('cookies button clicked')
    } catch (error) {
        log.error('cookies button not available');
    }
    await page.setViewport({
        width: 1920,
        height: 6000,
    });

    let content = await page.content()
    let securityCheck = await checkForChallengeValidation(page, request, content, log)
    if (!securityCheck){
        throw RetryRequestError
    }

    await enqueueLinks({
        // regexps: [/(\/supermercado\/[a-zA-Z]+)/],
        selector: 'li._next > a._pagination_link',
        forefront: true,
    });
    await enqueueLinks({
        // regexps: [/(\/supermercado\/[0-9]+)/],
        selector: 'a.product_link',
        label: 'product',
        forefront: true,
    });

    await page.close()
    page = null;
});

Navigation timed out after 60 seconds.

Similar Threads

Navigation timed out after 60 seconds.

Similar Threads

Similar Threads

Similar Threads