puppeteer/replay

Hi, all, nice to see you.

I have a question about how I would use puppeteer/replaypuppeteer/replay in PuppeteerCrawler for executing scenario generated on Chrome Recorder.

What I want to do is that, before executing crawling, function is called on each scenario step.

I tried it, not using PuppeteerCrawler.
But I'm not sure how it is integrated with PuppeteerCrawler.
Below is a current simple example.

Is there anyone who has any ideas?
Thank you.

import { BrowserPool, PuppeteerPlugin } from "@crawlee/browser-pool";
import ScenarioReplay from "./ScenarioReplay";
import fs from "fs";

// Read chrome recording json.
const recordingText = fs.readFileSync("path/to/scenario/json", "utf8");
const recording = parse(JSON.parse(recordingText));

const browserPool = new BrowserPool({
  prePageCloseHooks: [
    async (page, browserController) => {
      // do scraping and operate something on browser
    },
  ],
});

const page = await browserPool.newPage();

// create runner
const runner = await createRunner(
  recording,
  new ScenarioReplay(browserPool, page)
);

await runner.run();
await page.close();
await browser.newPage();
await browserPool.destroy();

import { BrowserPool, PuppeteerPlugin } from "@crawlee/browser-pool";
import ScenarioReplay from "./ScenarioReplay";
import fs from "fs";

// Read chrome recording json.
const recordingText = fs.readFileSync("path/to/scenario/json", "utf8");
const recording = parse(JSON.parse(recordingText));

const browserPool = new BrowserPool({
  prePageCloseHooks: [
    async (page, browserController) => {
      // do scraping and operate something on browser
    },
  ],
});

const page = await browserPool.newPage();

// create runner
const runner = await createRunner(
  recording,
  new ScenarioReplay(browserPool, page)
);

await runner.run();
await page.close();
await browser.newPage();
await browserPool.destroy();

import { PuppeteerRunnerExtension, Step, UserFlow } from "@puppeteer/replay";

class ScenarioReplay extends PuppeteerRunnerExtension {
  constructor(browser: Browser, page: Page) {
    super(browser, page);
  }

  override async beforeAllSteps(flow: UserFlow | undefined) {
    // doing something beforeAllSteps
  }

  override async beforeEachStep(step: Step, flow: UserFlow | undefined) {
    // doing something beforeEachStep
  }

  override async afterEachStep(step: Step, flow: UserFlow | undefined) {
    // doing something afterEachStep
  }

  override async afterAllSteps(flow: UserFlow | undefined) {
    // doing something afterAllSteps
  }
}

export default ScenarioReplay;

import { PuppeteerRunnerExtension, Step, UserFlow } from "@puppeteer/replay";

class ScenarioReplay extends PuppeteerRunnerExtension {
  constructor(browser: Browser, page: Page) {
    super(browser, page);
  }

  override async beforeAllSteps(flow: UserFlow | undefined) {
    // doing something beforeAllSteps
  }

  override async beforeEachStep(step: Step, flow: UserFlow | undefined) {
    // doing something beforeEachStep
  }

  override async afterEachStep(step: Step, flow: UserFlow | undefined) {
    // doing something afterEachStep
  }

  override async afterAllSteps(flow: UserFlow | undefined) {
    // doing something afterAllSteps
  }
}

export default ScenarioReplay;

Apify & Crawlee•4y ago•

5 replies

crooked-brown

puppeteer/replay

import { BrowserPool, PuppeteerPlugin } from "@crawlee/browser-pool";
import ScenarioReplay from "./ScenarioReplay";
import fs from "fs";

// Read chrome recording json.
const recordingText = fs.readFileSync("path/to/scenario/json", "utf8");
const recording = parse(JSON.parse(recordingText));

const browserPool = new BrowserPool({
  prePageCloseHooks: [
    async (page, browserController) => {
      // do scraping and operate something on browser
    },
  ],
});

const page = await browserPool.newPage();

// create runner
const runner = await createRunner(
  recording,
  new ScenarioReplay(browserPool, page)
);

await runner.run();
await page.close();
await browser.newPage();
await browserPool.destroy();

import { BrowserPool, PuppeteerPlugin } from "@crawlee/browser-pool";
import ScenarioReplay from "./ScenarioReplay";
import fs from "fs";

// Read chrome recording json.
const recordingText = fs.readFileSync("path/to/scenario/json", "utf8");
const recording = parse(JSON.parse(recordingText));

const browserPool = new BrowserPool({
  prePageCloseHooks: [
    async (page, browserController) => {
      // do scraping and operate something on browser
    },
  ],
});

const page = await browserPool.newPage();

// create runner
const runner = await createRunner(
  recording,
  new ScenarioReplay(browserPool, page)
);

await runner.run();
await page.close();
await browser.newPage();
await browserPool.destroy();

import { PuppeteerRunnerExtension, Step, UserFlow } from "@puppeteer/replay";

class ScenarioReplay extends PuppeteerRunnerExtension {
  constructor(browser: Browser, page: Page) {
    super(browser, page);
  }

  override async beforeAllSteps(flow: UserFlow | undefined) {
    // doing something beforeAllSteps
  }

  override async beforeEachStep(step: Step, flow: UserFlow | undefined) {
    // doing something beforeEachStep
  }

  override async afterEachStep(step: Step, flow: UserFlow | undefined) {
    // doing something afterEachStep
  }

  override async afterAllSteps(flow: UserFlow | undefined) {
    // doing something afterAllSteps
  }
}

export default ScenarioReplay;

import { PuppeteerRunnerExtension, Step, UserFlow } from "@puppeteer/replay";

class ScenarioReplay extends PuppeteerRunnerExtension {
  constructor(browser: Browser, page: Page) {
    super(browser, page);
  }

  override async beforeAllSteps(flow: UserFlow | undefined) {
    // doing something beforeAllSteps
  }

  override async beforeEachStep(step: Step, flow: UserFlow | undefined) {
    // doing something beforeEachStep
  }

  override async afterEachStep(step: Step, flow: UserFlow | undefined) {
    // doing something afterEachStep
  }

  override async afterAllSteps(flow: UserFlow | undefined) {
    // doing something afterAllSteps
  }
}

export default ScenarioReplay;

Continue the conversation

Join the Discord to ask follow-up questions and connect with the community

A&C

Apify & Crawlee

This is the official developer community of Apify and Crawlee.

15,508 Members

Join

puppeteer/replay

puppeteer/replay

Continue the conversation

Apify & Crawlee

Continue the conversation

Apify & Crawlee

Similar Threads