Extract data from a json variable
I can't find out how to extract data from a variable on a page I'm crawling.
It looks like this:
My crawler is currently like this:
What am I doing wrong?
It looks like this:
<script>
window.addEventListener("load", function() {
var jsondata = [{"id":"JOB_POSTING-3-865832","jobreqid":"107457WD"}];
var jobsTable = new JobsTable({
element: document.getElementById("wdresults"),
data: jsondata
});
});
</script><script>
window.addEventListener("load", function() {
var jsondata = [{"id":"JOB_POSTING-3-865832","jobreqid":"107457WD"}];
var jobsTable = new JobsTable({
element: document.getElementById("wdresults"),
data: jsondata
});
});
</script>My crawler is currently like this:
const crawler = new PlaywrightCrawler({
async requestHandler({ request, page, log }) {
log.info(`Processing ${request.url}...`);
// Wait for the page to fully load
await page.waitForLoadState('networkidle');
// Extract the jsondata variable
const jsondata = await page.evaluate(() => {
// Check if the variable is defined and return it
if (typeof jsondata !== 'undefined') {
return jsondata;
}
return [];
});
// Log the extracted data for debugging
log.info(`Extracted data: ${JSON.stringify(jsondata)}`);
// Save the jsondata to a dataset
await Dataset.pushData({ jsondata });
},
});const crawler = new PlaywrightCrawler({
async requestHandler({ request, page, log }) {
log.info(`Processing ${request.url}...`);
// Wait for the page to fully load
await page.waitForLoadState('networkidle');
// Extract the jsondata variable
const jsondata = await page.evaluate(() => {
// Check if the variable is defined and return it
if (typeof jsondata !== 'undefined') {
return jsondata;
}
return [];
});
// Log the extracted data for debugging
log.info(`Extracted data: ${JSON.stringify(jsondata)}`);
// Save the jsondata to a dataset
await Dataset.pushData({ jsondata });
},
});What am I doing wrong?
