Uunexpected-ivory
maximum retries instance variable doesn't affect the crawler behavior
I am talking about crawler apify/website-content-crawler , and here is how i intialize the instance
client = ApifyClient(token="{}",max_retries=1,timeout_secs=50)
def crawler(url):
# Prepare the Actor input
run_input = {
"startUrls": [{ "url": url}],
"crawlerType": "playwright:firefox",
"includeUrlGlobs": [],
"excludeUrlGlobs": [],
"ignoreCanonicalUrl": False,
"maxCrawlDepth": 0,
"maxCrawlPages": 2,
"initialConcurrency": 1,
"maxConcurrency": 20,
"initialCookies": [],
"proxyConfiguration": { "useApifyProxy": True },
"requestTimeoutSecs": 40,
"dynamicContentWaitSecs": 10,
"maxScrollHeightPixels": 5000,
"removeElementsCssSelector": """nav, footer, script, style, noscript, svg,
[role="alert"],
[role="banner"],
[role="dialog"],
[role="alertdialog"],
[role="region"][aria-label*="skip" i],
[aria-modal="true"]""",
"removeCookieWarnings": True,
"clickElementsCssSelector": "[aria-expanded="false"]",
"htmlTransformer": "readableText",
"readableTextCharThreshold": 100,
"aggressivePrune": False,
"debugMode": False,
"debugLog": False,
"saveHtml": False,
"saveMarkdown": True,
"saveFiles": False,
"saveScreenshots": False,
"maxResults": 999,
}
# Run the Actor
run = client.actor("aYG0l9s7dbB7j3gbS").call(run_input=run_input)
client = ApifyClient(token="{}",max_retries=1,timeout_secs=50)
def crawler(url):
# Prepare the Actor input
run_input = {
"startUrls": [{ "url": url}],
"crawlerType": "playwright:firefox",
"includeUrlGlobs": [],
"excludeUrlGlobs": [],
"ignoreCanonicalUrl": False,
"maxCrawlDepth": 0,
"maxCrawlPages": 2,
"initialConcurrency": 1,
"maxConcurrency": 20,
"initialCookies": [],
"proxyConfiguration": { "useApifyProxy": True },
"requestTimeoutSecs": 40,
"dynamicContentWaitSecs": 10,
"maxScrollHeightPixels": 5000,
"removeElementsCssSelector": """nav, footer, script, style, noscript, svg,
[role="alert"],
[role="banner"],
[role="dialog"],
[role="alertdialog"],
[role="region"][aria-label*="skip" i],
[aria-modal="true"]""",
"removeCookieWarnings": True,
"clickElementsCssSelector": "[aria-expanded="false"]",
"htmlTransformer": "readableText",
"readableTextCharThreshold": 100,
"aggressivePrune": False,
"debugMode": False,
"debugLog": False,
"saveHtml": False,
"saveMarkdown": True,
"saveFiles": False,
"saveScreenshots": False,
"maxResults": 999,
}
# Run the Actor
run = client.actor("aYG0l9s7dbB7j3gbS").call(run_input=run_input)
This is the official developer community of Apify and Crawlee.
14,091Members
Resources
Recent Announcements
Similar Threads
Was this page helpful?
