scrapy - How to extract items that are paginated

[Python] Get links of product to every page of a retailor

Here is the code of a simple spider that would use Crawling scraped links & next pagination. You have two options to solve your problem. The general one is to use yield to generate new requests instead of return. That way you can issue more than one new request from a single callback. Check the second example at http://doc.scrapy.org/en/latest/topics/spiders.html#basespider-example.




#------------------------------------------------------------------------------- # Name: module1 # Purpose: # # Author: Ranvijay.Sachan # # Created: 31/10/2014 # Copyright: (c) Ranvijay.Sachan 2014 # Licence: <your licence> #------------------------------------------------------------------------------- from scrapy.spider import BaseSpider from scrapy.selector import HtmlXPathSelector from scrapy.http.request import Request from scraping.articles import ArticleItem from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor from scrapy import log class Tesco(BaseSpider): name = "tesco" allowed_domains = ['ezakupy.tesco.pl'] start_urls = ['http://ezakupy.tesco.pl/en-GB/Search/List?searchQuery=knorr'] article_extractor = SgmlLinkExtractor(allow=r'/en-GB/ProductDetail/ProductDetail/') next_page_extractor = SgmlLinkExtractor( allow='pageNo=\d+', restrict_xpaths="//li[@class='current'][1]/following-sibling::li/a") def parse(self, response): articles = self.article_extractor.extract_links(response) for link in articles: item = ArticleItem() item['title'] = link.text item['url'] = link.url yield item log.msg("111111111111"+str(self.next_page_extractor)) next_pages = self.next_page_extractor.extract_links(response) log.msg("222222222222"+str(next_pages)) if next_pages: yield Request(next_pages[0].url) #------------------------------------------------------------------------------- # Name: module1 # Purpose: # # Author: Ranvijay.Sachan # # Created: 31/10/2014 # Copyright: (c) Ranvijay.Sachan 2014 # Licence: <your licence> #------------------------------------------------------------------------------- from scrapy.item import Item, Field class ArticleItem(Item): title = Field() url = Field()