const enqueued = await enqueueLinks({
selector: 'a',
transformRequestFunction: (request) => {
const url = new URL(request.url);
// Use just the domain and path as the unique key, ignoring query parameters and hash fragments
request.uniqueKey = url.origin + url.pathname;
if (request.uniqueKey.includes(startUrls[0].url)) {
if (globalStore.capturedLinks.hasOwnProperty(request.uniqueKey)) {
return;
}
globalStore.capturedLinks[request.uniqueKey] = false;
return request;
}
},
});
const enqueued = await enqueueLinks({
selector: 'a',
transformRequestFunction: (request) => {
const url = new URL(request.url);
// Use just the domain and path as the unique key, ignoring query parameters and hash fragments
request.uniqueKey = url.origin + url.pathname;
if (request.uniqueKey.includes(startUrls[0].url)) {
if (globalStore.capturedLinks.hasOwnProperty(request.uniqueKey)) {
return;
}
globalStore.capturedLinks[request.uniqueKey] = false;
return request;
}
},
});