Hi Everyone
EastAsiaEg has changed a little bit their payload therefore if you execute my code you will get repeated data.
Here is the new code:
import scrapy import json class DemoXHRPostSpider(scrapy.Spider): name = 'post_xhr' payload = ''' {"categoryId":"4","manufacturerId":"0","vendorId":"0","priceRangeFilterModel7Spikes":{"CategoryId":"4","ManufacturerId":"0","VendorId":"0","SelectedPriceRange":{},"MinPrice":"0","MaxPrice":"19000"},"specificationFiltersModel7Spikes":{"CategoryId":"4","ManufacturerId":"0","VendorId":"0","SpecificationFilterGroups":[{"Id":27,"FilterItems":[{"Id":"103","FilterItemState":"Unchecked"},{"Id":"104","FilterItemState":"Unchecked"},{"Id":"110","FilterItemState":"Unchecked"},{"Id":"108","FilterItemState":"Unchecked"},{"Id":"111","FilterItemState":"Unchecked"},{"Id":"853","FilterItemState":"Unchecked"},{"Id":"2626","FilterItemState":"Unchecked"}]},{"Id":11,"FilterItems":[{"Id":"302","FilterItemState":"Unchecked"},{"Id":"1071","FilterItemState":"Unchecked"},{"Id":"2494","FilterItemState":"Unchecked"}]},{"Id":6,"FilterItems":[{"Id":"2461","FilterItemState":"Unchecked"},{"Id":"21","FilterItemState":"Unchecked"},{"Id":"24","FilterItemState":"Unchecked"},{"Id":"25","FilterItemState":"Unchecked"},{"Id":"26","FilterItemState":"Unchecked"}]},{"Id":5,"FilterItems":[{"Id":"2742","FilterItemState":"Unchecked"},{"Id":"2743","FilterItemState":"Unchecked"},{"Id":"3493","FilterItemState":"Unchecked"},{"Id":"4330","FilterItemState":"Unchecked"}]},{"Id":2,"FilterItems":[{"Id":"8","FilterItemState":"Unchecked"},{"Id":"1451","FilterItemState":"Unchecked"}]},{"Id":8,"FilterItems":[{"Id":"61","FilterItemState":"Unchecked"},{"Id":"3458","FilterItemState":"Unchecked"},{"Id":"63","FilterItemState":"Unchecked"},{"Id":"4804","FilterItemState":"Unchecked"},{"Id":"3719","FilterItemState":"Unchecked"}]},{"Id":333,"FilterItems":[{"Id":"4034","FilterItemState":"Unchecked"},{"Id":"4035","FilterItemState":"Unchecked"}]},{"Id":334,"FilterItems":[{"Id":"2489","FilterItemState":"Unchecked"},{"Id":"2488","FilterItemState":"Unchecked"},{"Id":"2487","FilterItemState":"Unchecked"}]},{"Id":680,"FilterItems":[{"Id":"4787","FilterItemState":"Unchecked"},{"Id":"4794","FilterItemState":"Unchecked"},{"Id":"4791","FilterItemState":"Unchecked"},{"Id":"4795","FilterItemState":"Unchecked"},{"Id":"4935","FilterItemState":"Unchecked"}]}]},"manufacturerFiltersModel7Spikes":{"CategoryId":"4","ManufacturerFilterItems":[{"Id":"134","FilterItemState":"Unchecked"},{"Id":"137","FilterItemState":"Unchecked"},{"Id":"136","FilterItemState":"Unchecked"},{"Id":"162","FilterItemState":"Unchecked"},{"Id":"189","FilterItemState":"Unchecked"},{"Id":"141","FilterItemState":"Unchecked"},{"Id":"145","FilterItemState":"Unchecked"},{"Id":"142","FilterItemState":"Unchecked"},{"Id":"144","FilterItemState":"Unchecked"},{"Id":"2","FilterItemState":"Unchecked"},{"Id":"1","FilterItemState":"Unchecked"},{"Id":"6","FilterItemState":"Unchecked"},{"Id":"8","FilterItemState":"Unchecked"},{"Id":"28","FilterItemState":"Unchecked"},{"Id":"7","FilterItemState":"Unchecked"},{"Id":"84","FilterItemState":"Unchecked"},{"Id":"23","FilterItemState":"Unchecked"},{"Id":"122","FilterItemState":"Unchecked"},{"Id":"154","FilterItemState":"Unchecked"},{"Id":"108","FilterItemState":"Unchecked"}]},"pageNumber":null,"orderby":"10","viewmode":"grid","pagesize":0,"queryString":"#/pageSize=10&viewMode=grid&orderBy=10","shouldNotStartFromFirstPage":true,"keyword":"","searchCategoryId":"0","searchManufacturerId":"0","searchVendorId":"0","priceFrom":"","priceTo":"","includeSubcategories":"False","searchInProductDescriptions":"False","advancedSearch":"False","isOnSearchPage":"False"} ''' i = 2 def start_requests(self): yield scrapy.Request(url='http://eastasiaeg.com/25/getFilteredProducts', method='POST', body=self.payload, headers={'content-type': 'application/json'}) def parse(self, response): for product in response.xpath("//div[@class='product-item']"): yield { 'product_name': product.xpath(".//div[@class='details']/h2/a/text()").extract_first(), 'id': product.xpath(".//@data-productid").extract_first() } new_payload = ''' {"categoryId":"4","manufacturerId":"0","vendorId":"0","priceRangeFilterModel7Spikes":{"CategoryId":"4","ManufacturerId":"0","VendorId":"0","SelectedPriceRange":{},"MinPrice":"0","MaxPrice":"19000"},"specificationFiltersModel7Spikes":{"CategoryId":"4","ManufacturerId":"0","VendorId":"0","SpecificationFilterGroups":[{"Id":27,"FilterItems":[{"Id":"103","FilterItemState":"Unchecked"},{"Id":"104","FilterItemState":"Unchecked"},{"Id":"110","FilterItemState":"Unchecked"},{"Id":"108","FilterItemState":"Unchecked"},{"Id":"111","FilterItemState":"Unchecked"},{"Id":"853","FilterItemState":"Unchecked"},{"Id":"2626","FilterItemState":"Unchecked"}]},{"Id":11,"FilterItems":[{"Id":"302","FilterItemState":"Unchecked"},{"Id":"1071","FilterItemState":"Disabled"},{"Id":"2494","FilterItemState":"Unchecked"}]},{"Id":6,"FilterItems":[{"Id":"2461","FilterItemState":"Unchecked"},{"Id":"21","FilterItemState":"Unchecked"},{"Id":"24","FilterItemState":"Unchecked"},{"Id":"25","FilterItemState":"Unchecked"},{"Id":"26","FilterItemState":"Unchecked"}]},{"Id":5,"FilterItems":[{"Id":"2742","FilterItemState":"Unchecked"},{"Id":"2743","FilterItemState":"Unchecked"},{"Id":"3493","FilterItemState":"Unchecked"},{"Id":"4330","FilterItemState":"Unchecked"}]},{"Id":2,"FilterItems":[{"Id":"8","FilterItemState":"Unchecked"},{"Id":"1451","FilterItemState":"Unchecked"}]},{"Id":8,"FilterItems":[{"Id":"61","FilterItemState":"Unchecked"},{"Id":"3458","FilterItemState":"Unchecked"},{"Id":"63","FilterItemState":"Unchecked"},{"Id":"4804","FilterItemState":"Unchecked"},{"Id":"3719","FilterItemState":"Unchecked"}]},{"Id":333,"FilterItems":[{"Id":"4034","FilterItemState":"Unchecked"},{"Id":"4035","FilterItemState":"Unchecked"}]},{"Id":334,"FilterItems":[{"Id":"2489","FilterItemState":"Unchecked"},{"Id":"2488","FilterItemState":"Unchecked"},{"Id":"2487","FilterItemState":"Unchecked"}]},{"Id":680,"FilterItems":[{"Id":"4787","FilterItemState":"Unchecked"},{"Id":"4794","FilterItemState":"Unchecked"},{"Id":"4791","FilterItemState":"Unchecked"},{"Id":"4795","FilterItemState":"Unchecked"},{"Id":"4935","FilterItemState":"Unchecked"}]}]},"manufacturerFiltersModel7Spikes":{"CategoryId":"4","ManufacturerFilterItems":[{"Id":"134","FilterItemState":"Unchecked"},{"Id":"137","FilterItemState":"Unchecked"},{"Id":"136","FilterItemState":"Unchecked"},{"Id":"162","FilterItemState":"Unchecked"},{"Id":"189","FilterItemState":"Unchecked"},{"Id":"141","FilterItemState":"Unchecked"},{"Id":"145","FilterItemState":"Unchecked"},{"Id":"142","FilterItemState":"Unchecked"},{"Id":"144","FilterItemState":"Unchecked"},{"Id":"2","FilterItemState":"Unchecked"},{"Id":"1","FilterItemState":"Unchecked"},{"Id":"6","FilterItemState":"Unchecked"},{"Id":"8","FilterItemState":"Unchecked"},{"Id":"28","FilterItemState":"Unchecked"},{"Id":"7","FilterItemState":"Unchecked"},{"Id":"84","FilterItemState":"Unchecked"},{"Id":"23","FilterItemState":"Unchecked"},{"Id":"122","FilterItemState":"Unchecked"},{"Id":"154","FilterItemState":"Unchecked"},{"Id":"108","FilterItemState":"Unchecked"}]},"pageNumber":2,"orderby":"10","viewmode":"grid","pagesize":0,"queryString":"","shouldNotStartFromFirstPage":true,"keyword":"","searchCategoryId":"0","searchManufacturerId":"0","searchVendorId":"0","priceFrom":"","priceTo":"","includeSubcategories":"False","searchInProductDescriptions":"False","advancedSearch":"False","isOnSearchPage":"False"} ''' if self.i <= 8: new_payload = json.loads(new_payload) new_payload['pageNumber'] = self.i self.i += 1 yield scrapy.Request(url='http://eastasiaeg.com/25/getFilteredProducts', method='POST', body=json.dumps(new_payload), headers={'content-type': 'application/json'})