rubber-blueR
Apify & Crawlee12mo ago
7 replies
rubber-blue

proxy_config.new_url() does not return new proxy

Here is my selenium python script, where i try to rotate proxies using the proxy_config.new_url():
# Standard libraries
import asyncio
import logging
import json

# Installed libraries
from selenium.common.exceptions import TimeoutException, WebDriverException
from selenium.webdriver.chrome.options import Options as ChromeOptions
from selenium.webdriver.common.proxy import ProxyType, Proxy
from selenium.webdriver.common.by import By
from selenium import webdriver
from apify import Actor

async def main() -> None:
    async with Actor:
        Actor.log.setLevel(logging.DEBUG)
        proxy_config = await Actor.create_proxy_configuration(groups=['RESIDENTIAL'])
        url = "https://api.ipify.org?format=json"
        for _ in range(10):
            proxy = await proxy_config.new_url()
            Actor.log.info(f'Using proxy: {proxy}')
            chrome_options = ChromeOptions()
            chrome_options.add_argument("--headless")
            chrome_options.add_argument("--no-sandbox")
            chrome_options.add_argument("--disable-dev-shm-usage")
            chrome_options.proxy = Proxy({'proxyType': ProxyType.MANUAL, 'httpProxy': proxy})
            try:
                with webdriver.Chrome(options=chrome_options) as driver:
                    driver.set_page_load_timeout(20)
                    driver.get(url)
                    content = driver.find_element(By.TAG_NAME, 'pre').text
                    ip = json.loads(content).get("ip")
                    Actor.log.info(f"IP = {ip}")
            except (TimeoutException, WebDriverException, json.JSONDecodeError):
                Actor.log.exception("An error occured")

Due to discord message size limitation i attach the log output of the above code in a new message below...
Was this page helpful?