No matter which params i change my cpu is always around 50% and ram not used more than ~20%
๐ญPlaywrightCrawler๐จโ๐ปWeb-Scraping
Hi
, Here my code
const configuration = {
purgeOnStart: false,
availableMemoryRatio: 0.75,
maxMemoryOverloadedRatio: 0.75,
maxEventLoopOverloadedRatio: 0.75,
maxCpuOverloadedRatio: 0.75,
maxClientOverloadedRatio: 0.7,
desiredConcurrencyRatio: 0.5,
persistStorage: true,
persistStateIntervalMillis: 10000,
}
const crawler = new PlaywrightCrawler(
{
keepAlive: true,
minConcurrency: 126,
maxConcurrency: 252,
maxRequestRetries: 5,
requestHandlerTimeoutSecs: 10,
proxyConfiguration,
useSessionPool: false, // for performance and less detection must be false
sessionPoolOptions: { maxPoolSize: 426 },
persistCookiesPerSession: false,const configuration = {
purgeOnStart: false,
availableMemoryRatio: 0.75,
maxMemoryOverloadedRatio: 0.75,
maxEventLoopOverloadedRatio: 0.75,
maxCpuOverloadedRatio: 0.75,
maxClientOverloadedRatio: 0.7,
desiredConcurrencyRatio: 0.5,
persistStorage: true,
persistStateIntervalMillis: 10000,
}
const crawler = new PlaywrightCrawler(
{
keepAlive: true,
minConcurrency: 126,
maxConcurrency: 252,
maxRequestRetries: 5,
requestHandlerTimeoutSecs: 10,
proxyConfiguration,
useSessionPool: false, // for performance and less detection must be false
sessionPoolOptions: { maxPoolSize: 426 },
persistCookiesPerSession: false,