Navigation timed out after 60 seconds.
I'm scraping a website, if I run it in headless mode I get this error, if I run it headed I see the webpage completely loaded (the loading wheel still spins somehow).
this is my routes:
this is my routes:
router.addDefaultHandler(async ({ enqueueLinks, log, page, request }) => {
if (request.url.includes('/ayuda/')){
await page.close()
return;
}
if (request.url.includes('sitemaps.org')) {
await page.close()
return;
}
try {
await page.waitForSelector('#cookies-agree');
await page.click('#cookies-agree');
log.info('cookies button clicked')
} catch (error) {
log.error('cookies button not available');
}
await page.setViewport({
width: 1920,
height: 6000,
});
let content = await page.content()
let securityCheck = await checkForChallengeValidation(page, request, content, log)
if (!securityCheck){
throw RetryRequestError
}
await enqueueLinks({
// regexps: [/(\/supermercado\/[a-zA-Z]+)/],
selector: 'li._next > a._pagination_link',
forefront: true,
});
await enqueueLinks({
// regexps: [/(\/supermercado\/[0-9]+)/],
selector: 'a.product_link',
label: 'product',
forefront: true,
});
await page.close()
page = null;
});router.addDefaultHandler(async ({ enqueueLinks, log, page, request }) => {
if (request.url.includes('/ayuda/')){
await page.close()
return;
}
if (request.url.includes('sitemaps.org')) {
await page.close()
return;
}
try {
await page.waitForSelector('#cookies-agree');
await page.click('#cookies-agree');
log.info('cookies button clicked')
} catch (error) {
log.error('cookies button not available');
}
await page.setViewport({
width: 1920,
height: 6000,
});
let content = await page.content()
let securityCheck = await checkForChallengeValidation(page, request, content, log)
if (!securityCheck){
throw RetryRequestError
}
await enqueueLinks({
// regexps: [/(\/supermercado\/[a-zA-Z]+)/],
selector: 'li._next > a._pagination_link',
forefront: true,
});
await enqueueLinks({
// regexps: [/(\/supermercado\/[0-9]+)/],
selector: 'a.product_link',
label: 'product',
forefront: true,
});
await page.close()
page = null;
});
