Function isn't running when posted in api fy

Hello everyone,

I have the following code working perfectly when I am not using Apify however once I use Apify it doesn't run the second function. I am using the apify template for Scrapy .

Thanks for the help

from typing import Generator
from scrapy.responsetypes import Response
from apify import Actor
from urllib.parse import urljoin
import nest_asyncio
import scrapy
from itemadapter import ItemAdapter
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from scrapy.utils.reactor import install_reactor
from scrapy.http import Response, Request

class TitleSpider(scrapy.Spider):
    name = 'example'
    allowed_domains = ['apc.fr']
    start_urls = [
        "https://www.apc.fr/men/men-shirts.html",
    ]

    def parse(self, response: Response):
        Actor.log.info(f'TitleSpider is parsing {response}...')
        li_elements = response.css('li.product-item')
        product_links = []
        for li in li_elements:
            productlink_container = li.css('.product-link')
            product_link = productlink_container.css('a::attr(href)').get()
            if product_link:
                product_links.append(product_link)

        for link in product_links:
            yield scrapy.Request(url=link, callback=self.second_page)

    def second_page(self, response: Response):
        Actor.log.info(f'Second fonction is parsing {response}...')
        productname = response.css('h1.product-name::text').get()
        print(productname)

from typing import Generator
from scrapy.responsetypes import Response
from apify import Actor
from urllib.parse import urljoin
import nest_asyncio
import scrapy
from itemadapter import ItemAdapter
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from scrapy.utils.reactor import install_reactor
from scrapy.http import Response, Request

class TitleSpider(scrapy.Spider):
    name = 'example'
    allowed_domains = ['apc.fr']
    start_urls = [
        "https://www.apc.fr/men/men-shirts.html",
    ]

    def parse(self, response: Response):
        Actor.log.info(f'TitleSpider is parsing {response}...')
        li_elements = response.css('li.product-item')
        product_links = []
        for li in li_elements:
            productlink_container = li.css('.product-link')
            product_link = productlink_container.css('a::attr(href)').get()
            if product_link:
                product_links.append(product_link)

        for link in product_links:
            yield scrapy.Request(url=link, callback=self.second_page)

    def second_page(self, response: Response):
        Actor.log.info(f'Second fonction is parsing {response}...')
        productname = response.css('h1.product-name::text').get()
        print(productname)

Apify & Crawlee•3y ago•

4 replies

skinny-azure

Function isn't running when posted in api fy

from typing import Generator
from scrapy.responsetypes import Response
from apify import Actor
from urllib.parse import urljoin
import nest_asyncio
import scrapy
from itemadapter import ItemAdapter
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from scrapy.utils.reactor import install_reactor
from scrapy.http import Response, Request

class TitleSpider(scrapy.Spider):
    name = 'example'
    allowed_domains = ['apc.fr']
    start_urls = [
        "https://www.apc.fr/men/men-shirts.html",
    ]

    def parse(self, response: Response):
        Actor.log.info(f'TitleSpider is parsing {response}...')
        li_elements = response.css('li.product-item')
        product_links = []
        for li in li_elements:
            productlink_container = li.css('.product-link')
            product_link = productlink_container.css('a::attr(href)').get()
            if product_link:
                product_links.append(product_link)

        for link in product_links:
            yield scrapy.Request(url=link, callback=self.second_page)

    def second_page(self, response: Response):
        Actor.log.info(f'Second fonction is parsing {response}...')
        productname = response.css('h1.product-name::text').get()
        print(productname)

from typing import Generator
from scrapy.responsetypes import Response
from apify import Actor
from urllib.parse import urljoin
import nest_asyncio
import scrapy
from itemadapter import ItemAdapter
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from scrapy.utils.reactor import install_reactor
from scrapy.http import Response, Request

class TitleSpider(scrapy.Spider):
    name = 'example'
    allowed_domains = ['apc.fr']
    start_urls = [
        "https://www.apc.fr/men/men-shirts.html",
    ]

    def parse(self, response: Response):
        Actor.log.info(f'TitleSpider is parsing {response}...')
        li_elements = response.css('li.product-item')
        product_links = []
        for li in li_elements:
            productlink_container = li.css('.product-link')
            product_link = productlink_container.css('a::attr(href)').get()
            if product_link:
                product_links.append(product_link)

        for link in product_links:
            yield scrapy.Request(url=link, callback=self.second_page)

    def second_page(self, response: Response):
        Actor.log.info(f'Second fonction is parsing {response}...')
        productname = response.css('h1.product-name::text').get()
        print(productname)

Function isn't running when posted in api fy

Similar Threads

Function isn't running when posted in api fy

Similar Threads

Similar Threads

Similar Threads