Crawl sitemap

 async crawlSitemap({
    url,
    containerId,
    channelId,
  }: {
    url: string;
    containerId: string;
    channelId: string;
  }): Promise<Result<ICrawlerInfo>> {
    const infos: ICrawlerExtraxtedInfo[] = [];
    const crawler = new PlaywrightCrawler({
      async requestHandler({ request, page, log }) {
        const title = await page.title();
        log.info(`Title of ${request.loadedUrl} is '${title}'`);

        infos.push({ url: request.loadedUrl, title: title });
      },
      // TODO: make this configurable
      maxRequestsPerCrawl: 50,
    });

    const result = await this.crawlerInfoService.createCrawlerInfo({
      containerId,
      channelId,
      url,
    });
    const { urls } = await Sitemap.load(url);
    await crawler.addRequests(urls);

    crawler.run().then(() => {
      this.logger.log(`Crawler finished ${url} with ${infos.length} links`);
      try {
        result.extractedInfo = infos;
        this.crawlerInfoService.updateCrawlerInfo(result.id, {
          status: CrawlerInfoStatus.COMPLETED,
          extractedInfo: infos,
        });
      } catch (error) {
        this.logger.error(`${error.message} ${error.stack}`);
        this.crawlerInfoService.updateCrawlerInfo(result.id, {
          status: CrawlerInfoStatus.FAILED,
        });
      }
    });
    return Ok(result);
  }