import { PuppeteerCrawler, Dataset } from "crawlee";
import { router } from "./routes.js";
const crawler = new PuppeteerCrawler({
launchContext: {
useChrome: true,
userDataDir: 'C:\Users\enric\AppData\Local\Google\Chrome\User Data\Default',
launchOptions: {
headless: false,
}
},
requestHandler: router,
async failedRequestHandler({ request }) {
// This function is called when the crawling of a request failed too many times
await Dataset.pushData({
url: request.url,
succeeded: false,
errors: request.errorMessages,
})
},
});
await crawler.run([
'mylink'
]);
import { PuppeteerCrawler, Dataset } from "crawlee";
import { router } from "./routes.js";
const crawler = new PuppeteerCrawler({
launchContext: {
useChrome: true,
userDataDir: 'C:\Users\enric\AppData\Local\Google\Chrome\User Data\Default',
launchOptions: {
headless: false,
}
},
requestHandler: router,
async failedRequestHandler({ request }) {
// This function is called when the crawling of a request failed too many times
await Dataset.pushData({
url: request.url,
succeeded: false,
errors: request.errorMessages,
})
},
});
await crawler.run([
'mylink'
]);