Scrapy - Crawls for Scraping AJAX Pages

Here is the code of a simple spider that would use Crawls for Scraping AJAX Pages. #------------------------------------------------------------------------------- # Name: module1 # Purpose: # # Author: Ranvijay.Sachan # # Created: 31/10/2014 # Copyright: (c) Ranvijay.Sachan 2014 # Licence: <your licence> #------------------------------------------------------------------------------- from scrapy.http import Request from scrapy.spider import BaseSpider import urllib import json from scraping.DoveItem import DoveItem class DoveAjaxspider(BaseSpider): name = "dove" allowed_domains = ["dove.in"] start_urls = ["http://www.mydove.com.au/en/"] def parse(self, response): # This receives the response from the start url. But we don't do anything with it. allProductType = ['Bar/Body Wash','Lotion','Deodorant','Face','Hair','Men+Care'] url = 'http://www.mydove.com.au/en/ProductService.svc/LoadCategoryLanding' requests = [] dataRow = {} count=0 for productCategory in allProductType: payload = '{"subCategoryName" : "'+productCategory+'"}' count = count + 1 requests = Request(url, dont_filter=True,body=payload,method='POST', headers={"content-type":"application/json"}, callback=self.parse_stores) yield requests def parse_stores(self, response): print "Start processw ---------------------- " data = json.loads(response.body) productData = json.loads(data['d']) for productcat in productData: productList = productcat['ProductList'] ##print productList for productItem in productList: yield McDonaldsItem(name=productItem['ProductBuyURL']) #------------------------------------------------------------------------------- # Name: module1 # Purpose: # # Author: Ranvijay.Sachan # # Created: 31/10/2014 # Copyright: (c) Ranvijay.Sachan 2014 # Licence: <your licence> #------------------------------------------------------------------------------- class DoveItem(Item): name = Field() url = Field()