如何在没有下一页信息的情况下无限循环使用scrapy进行抓取

如何解决如何在没有下一页信息的情况下无限循环使用scrapy进行抓取

我需要使用scrapy抓取网址，但我无法向下滚动网站以加载所有元素。

我尝试搜索下一页信息，但找不到

我的蜘蛛代码是：

import scrapy
from scrapy.spiders import CrawlSpider,Rule
from scrapy.linkextractors import LinkExtractor
from appinformatica.items import appinformaticaItem

import w3lib.html

class appinformaticaSpider (CrawlSpider):
    name = 'appinformatica'
    item_count=0
    start_urls =['https://www.appinformatica.com/telefonos/moviles/']
    rules = {
        Rule(LinkExtractor(allow=(),restrict_xpaths=('//*[@class="info-ficha"]/div[1]/a')),callback='parse_item',follow=False)
    }

    def parse_item(self,response):
        item = appinformaticaItem()
        self.item_count += 1
        item['Modelo'] = w3lib.html.remove_tags(response.xpath("//h1").get(default=''))
        item['Position'] = self.item_count
        item['Precio'] = w3lib.html.remove_tags(response.xpath('//*[@id="ficha-producto"]/div[2]/div[1]/div/div[1]').get(default=''))
        item['PrecioTienda'] = w3lib.html.remove_tags(response.xpath('//*[@id="ficha-producto"]/div[2]/div[1]/div/div[2]').get(default=''))
        item['Stock'] = w3lib.html.remove_tags(response.xpath('//*[@id="ficha-producto"]/div[2]/div[3]/p[3]').get(default=''))
        item['Submodelo'] = w3lib.html.remove_tags(response.xpath('//*[@id="ficha-producto"]/div[2]/div[3]/p[2]/strong[2]').get(default=''))
        item['Url'] = w3lib.html.remove_tags(response.url)
        yield item

有人可以帮我吗？

解决方法

将 allow 更改为 allow=(r'/moviles/.*.html'),follow=True 并放置您的 allowed_domains。试试这个。

import scrapy
from scrapy.spiders import CrawlSpider,Rule
from scrapy.linkextractors import LinkExtractor
# from appinformatica.items import appinformaticaItem

import w3lib.html

class appinformaticaSpider (CrawlSpider):
    name = 'appinformatica'
    allowed_domains = ["appinformatica.com"]
    item_count=0
    start_urls =['https://www.appinformatica.com/telefonos/moviles/']
    rules = {
        Rule(LinkExtractor(allow=(r'/moviles/.*\.html'),),callback='parse_item',follow=True)
    }

    def parse_item(self,response):
        item = {}
        self.item_count += 1
        item['Modelo'] = w3lib.html.remove_tags(response.xpath("//h1").get(default=''))
        item['Position'] = self.item_count
        item['Precio'] = w3lib.html.remove_tags(response.xpath('//*[@id="ficha-producto"]/div[2]/div[1]/div/div[1]').get(default=''))
        item['PrecioTienda'] = w3lib.html.remove_tags(response.xpath('//*[@id="ficha-producto"]/div[2]/div[1]/div/div[2]').get(default=''))
        item['Stock'] = w3lib.html.remove_tags(response.xpath('//*[@id="ficha-producto"]/div[2]/div[3]/p[3]').get(default=''))
        item['Submodelo'] = w3lib.html.remove_tags(response.xpath('//*[@id="ficha-producto"]/div[2]/div[3]/p[2]/strong[2]').get(default=''))
        item['Url'] = w3lib.html.remove_tags(response.url)
        yield item