import { useState } from 'crawlee';
// initialization categoryRouter, defaultRouter, categoryRequestQueue, detailRequestQueue
categoryRouter.addHandler("CATEGORY", async ({ enqueueLinks, crawler }) => {
const categoryName = ...;
const productCategories = await Actor.useState('productCategories', {}); // shared state
const detailUrls = await page.$$eval('.product-detail a', (linkElements) => linkElements.map((linkEl) => linkEl.href)
for (const detailUrl of detailUrls) {
productCategories[detailUrl] ??= [];
productCategories[detailUrl].push(categoryName);
}
// ...add product detail urls to the detailRequestQueue
});
detailRouter.addHandler("DETAIL", async ({ request }) => {
const productCategories = await Actor.useState('productCategories', {});
const categories = productCategories[request.url];
// ... saving
});
import { useState } from 'crawlee';
// initialization categoryRouter, defaultRouter, categoryRequestQueue, detailRequestQueue
categoryRouter.addHandler("CATEGORY", async ({ enqueueLinks, crawler }) => {
const categoryName = ...;
const productCategories = await Actor.useState('productCategories', {}); // shared state
const detailUrls = await page.$$eval('.product-detail a', (linkElements) => linkElements.map((linkEl) => linkEl.href)
for (const detailUrl of detailUrls) {
productCategories[detailUrl] ??= [];
productCategories[detailUrl].push(categoryName);
}
// ...add product detail urls to the detailRequestQueue
});
detailRouter.addHandler("DETAIL", async ({ request }) => {
const productCategories = await Actor.useState('productCategories', {});
const categories = productCategories[request.url];
// ... saving
});