Ffaint-white
I am trying to reseting crawlee cache in nextjs what its note working can any one help me
this my nextjs code at initial request data is displayed but if i try to request again its displaying empty array or i have to restart the application
//with this
Crawl finished. Final request statistics: {"requestsFinished":0,"requestsFailed":0,"retryHistogram":[],"requestAvgFailedDurationMillis":null,"requestAvgFinishedDurationMillis":null,"requestsFinishedPerMinute":0,"requestsFailedPerMinute":0,"requestTotalDurationMillis":0,"requestsTotal":0,"crawlerRuntimeMillis":445}
// scraper code
const config = Configuration.getGlobalConfig();
config.set('purgeOnStart', false);
config.set('persistStorage', false);
export const ovaLatest = async () => {
const data: { title: string; }[] = [];
const crawler = new CheerioCrawler({
async requestHandler({ $, enqueueLinks,request }): Promise<void> {
console.log(request.url)
const latest = $('div#left-div')
latest.find('div.home-post-titles > h2').each((index, el) => {
data.push({ title: $(el).text().trim() });
});
// await enqueueLinks();
}
});
await crawler.run(['https://www.xyz.com/page/4']);
console.log(data)
return { data: data }
}
// nextjs api route code
export default async function handler(
req: NextApiRequest,
res: NextApiResponse<Data>
) {
const data = await xyzLatest()
res.status(200).json(data)
}
//with this
Crawl finished. Final request statistics: {"requestsFinished":0,"requestsFailed":0,"retryHistogram":[],"requestAvgFailedDurationMillis":null,"requestAvgFinishedDurationMillis":null,"requestsFinishedPerMinute":0,"requestsFailedPerMinute":0,"requestTotalDurationMillis":0,"requestsTotal":0,"crawlerRuntimeMillis":445}
// scraper code
const config = Configuration.getGlobalConfig();
config.set('purgeOnStart', false);
config.set('persistStorage', false);
export const ovaLatest = async () => {
const data: { title: string; }[] = [];
const crawler = new CheerioCrawler({
async requestHandler({ $, enqueueLinks,request }): Promise<void> {
console.log(request.url)
const latest = $('div#left-div')
latest.find('div.home-post-titles > h2').each((index, el) => {
data.push({ title: $(el).text().trim() });
});
// await enqueueLinks();
}
});
await crawler.run(['https://www.xyz.com/page/4']);
console.log(data)
return { data: data }
}
// nextjs api route code
export default async function handler(
req: NextApiRequest,
res: NextApiResponse<Data>
) {
const data = await xyzLatest()
res.status(200).json(data)
}
This is the official developer community of Apify and Crawlee.
13,923Members
Resources
Recent Announcements
Similar Threads
Was this page helpful?
