// .env
CRAWLEE_PURGE_ON_START=false
// main.ts
const startUrls = ["example.com"]
const crawler = new CheerioCrawler({
requestHandler: router,
maxRequestsPerCrawl: 5,
maxConcurrency: 1,
// maxRequestsPerMinute: 30,
});
const main = async () => {
await crawler.addRequests(startUrls);
await crawler.run();
};
main();
// router.ts
router.addDefaultHandler(async ({ enqueueLinks, log }) => {
log.info(`enqueueing new URLs`);
await enqueueLinks({
strategy: EnqueueStrategy.SameHostname,
globs: [
"https://example.com/news/*",
],
label: "detail"
})
});
router.addHandler("detail", async ({ request: req, $, log }) => {
const title = await page.title();
log.info(`Title of ${request.loadedUrl} is '${title}'`);
// Save to DB Code here
});
// .env
CRAWLEE_PURGE_ON_START=false
// main.ts
const startUrls = ["example.com"]
const crawler = new CheerioCrawler({
requestHandler: router,
maxRequestsPerCrawl: 5,
maxConcurrency: 1,
// maxRequestsPerMinute: 30,
});
const main = async () => {
await crawler.addRequests(startUrls);
await crawler.run();
};
main();
// router.ts
router.addDefaultHandler(async ({ enqueueLinks, log }) => {
log.info(`enqueueing new URLs`);
await enqueueLinks({
strategy: EnqueueStrategy.SameHostname,
globs: [
"https://example.com/news/*",
],
label: "detail"
})
});
router.addHandler("detail", async ({ request: req, $, log }) => {
const title = await page.title();
log.info(`Title of ${request.loadedUrl} is '${title}'`);
// Save to DB Code here
});