当前位置: 首页>>代码示例>>Python>>正文

Python Request.meta['proxy']方法代码示例

本文整理汇总了Python中scrapy.http.request.Request.meta['proxy']方法的典型用法代码示例。如果您正苦于以下问题：Python Request.meta['proxy']方法的具体用法？Python Request.meta['proxy']怎么用？Python Request.meta['proxy']使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类scrapy.http.request.Request的用法示例。


在下文中一共展示了Request.meta['proxy']方法的5个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。

示例1: snapdeal_scraper

# 需要导入模块: from scrapy.http.request import Request [as 别名]
# 或者: from scrapy.http.request.Request import meta['proxy'] [as 别名]
	def snapdeal_scraper(self,response):
		item = response.meta['item']
		sel = Selector(response)
		item['Snapdeal_URL']= response.url
		try:
			if sel.xpath("//div[@class='notifyMe-soldout']"):
				ProductName = sel.xpath("//h1[@itemprop='name']/text()").extract()[0].replace(",","")
				item['Snapdeal__ProductName'] =ProductName
				item['Snapdeal_MRP']=item['Snapdeal_SP'] = ''
				item['Snapdeal_Stock'] = 'Out of Stock'
				
				
			else:
				mrp = sel.xpath("//span[@id='original-price-id']/text()").extract()
				if mrp:
					item['Snapdeal_SP'] = sel.xpath("//span[@id='selling-price-id']/text()").extract()[0]
					item['Snapdeal_MRP'] = sel.xpath("//span[@id='original-price-id']/text()").extract()[0]
				else:
					item['Snapdeal_MRP'] = sel.xpath("//span[@id='selling-price-id']/text()").extract()[0]
					item['Snapdeal_SP'] = ''
					
				item['Snapdeal__ProductName'] = sel.xpath("//h1[@itemprop='name']/text()").extract()[0].replace(",","")
				stock = sel.xpath("//div[@class='notifyMe-soldout']").extract()
				discntnd = sel.xpath("//div[@class='noLongerProduct']").extract()
				if stock or discntnd:
					item['Snapdeal_Stock'] = "Out Of Stock"
				else:
					item['Snapdeal_Stock'] = "In Stock"				
				
		except:			
			item['Snapdeal__ProductName'] =	item['Snapdeal_MRP'] = item['Snapdeal_SP'] = ''
			item['Snapdeal_Stock'] = 'Not Found'
		
		
		try:
			amazon_url = amazon_urls[item['index']]
			request = Request(amazon_url,
								headers={'Referer':'http://amazon.in'},
								callback = self.amazon_scraper)
			request.meta['item'] = item
			request.meta['proxy'] = "http://111.161.126.100:80"
			yield request
			
		except:				
			try:
				flipkart_url = flipkart_urls[item['index']]
				request = Request(flipkart_url,callback = self.flipkart_scraper)
				request.meta['item'] = item
				# request.meta['proxy'] = "http://111.161.126.100:80"
				yield request
		
			except:			
				try:
					paytm_url = paytm_urls[item['index']]
					request = Request(paytm_url,callback = self.paytm_scraper)
					request.meta['item'] = item
					request.meta['proxy'] = "http://111.161.126.100:80"
					yield request
				except:
					self.to_csv(item)
开发者ID:Diwahars，项目名称:scrapers，代码行数:62，代码来源:PopularProductsScraper.py

示例2: amazon_marketplace

# 需要导入模块: from scrapy.http.request import Request [as 别名]
# 或者: from scrapy.http.request.Request import meta['proxy'] [as 别名]
	def amazon_marketplace(self,response):
		
		sel = Selector(response)
		item = response.meta['item']
		try:
			sp = sel.xpath("//span[@style='text-decoration: inherit; white-space: nowrap;']/text()").extract()[0].replace(",","")
			shippingcost = sel.xpath("//span[@class='olpShippingPrice']/span/text()").extract()
			if shippingcost:
				sp = str(float(sp) + float(sel.xpath("//span[@class='olpShippingPrice']/span/text()").extract()[0].replace(",","")))	
			
			if sp>item['SP']:
				sp = item['SP']
		except:			
			try:
				flipkart_url = flipkart_urls[item['index']]
				request = Request(flipkart_url,callback = self.flipkart_scraper)
				request.meta['item'] = item
				# request.meta['proxy'] = "http://111.161.126.100:80"
				yield request
		
			except:				
				try:
					paytm_url = paytm_urls[item['index']]
					request = Request(paytm_url,callback = self.paytm_scraper)
					request.meta['item'] = item
					request.meta['proxy'] = "http://111.161.126.100:80"
					yield request
				except:
					self.to_csv(item)
开发者ID:Diwahars，项目名称:scrapers，代码行数:31，代码来源:PopularProductsScraper.py

示例3: flipkart_scraper

# 需要导入模块: from scrapy.http.request import Request [as 别名]
# 或者: from scrapy.http.request.Request import meta['proxy'] [as 别名]
	def flipkart_scraper(self,response):
		sel = Selector(response)
		item = response.meta['item']		
		item['Flipkart_URL'] = response.url
		
		
		
		if sel.xpath("//h1[@class='title']/text()").extract():
			item['Flipkart_ProductName'] = sel.xpath("//h1[@class='title']/text()").extract()[0].replace(",","")      
			
			
			mrp_xpath = sel.xpath("//span[@class='price']/text()").extract()			
			sp_xpath = sel.xpath("//span[@class='selling-price omniture-field']/text()").extract()
			
			if mrp_xpath and sp_xpath:
				item['Flipkart_MRP'] = mrp_xpath[0].replace(",","").replace("Rs.","")
				item['Flipkart_SP'] = sp_xpath[0].replace(",","").replace("Rs.","")
			elif sp_xpath:
				item['Flipkart_MRP'] = sp_xpath[0].replace(",","").replace("Rs.","")
				item['Flipkart_SP'] = ''
			elif mrp_xpath:
				item['Flipkart_MRP'] = mrp_xpath[0].replace(",","").replace("Rs.","")
				item['Flipkart_SP'] = ''
				
			
			stock = sel.xpath("//div[@class='out-of-stock-status'] | //div[@class='no-sellers-available omniture-field']").extract()					
			
			if not stock :
				item['Flipkart_Stock'] = "In Stock"
			else:
				item['Flipkart_Stock'] = 'Out Of Stock'			
		
		else:
			item['Flipkart_ProductName'] = item['Flipkart_SP'] = item['Flipkart_MRP'] = ''
			item['Flipkart_Stock'] = "Not Found"
		
		print item
		try:			
			paytm_url = paytm_urls[item['index']]
			paytm_url = paytm_url.replace('//paytm.com/shop', '//catalog.paytm.com/v1')
			request = Request(paytm_url,callback = self.paytm_scraper)
			request.meta['item'] = item
			request.meta['proxy'] = "http://111.161.126.100:80"			
			yield request
		
		except:
			self.to_csv(item)
开发者ID:Diwahars，项目名称:scrapers，代码行数:49，代码来源:PopularProductsScraper.py

示例4: parse

# 需要导入模块: from scrapy.http.request import Request [as 别名]
# 或者: from scrapy.http.request.Request import meta['proxy'] [as 别名]
	def parse(self,response):
		item = BigCItem()
		sel = Selector(response)
		
		item['Category']=(sel.xpath("//div[@id='ProductBreadcrumb']/ul/li/a/text()").extract()[1]+
								"/"+sel.xpath("//div[@id='ProductBreadcrumb']/ul/li/a/text()").extract()[2]+
								"/"+sel.xpath("//div[@id='ProductBreadcrumb']/ul/li/a/text()").extract()[3])
		
		item['Product_Name'] = sel.xpath("//h1/text()").extract()[0].replace(",","")
		
		sp = sel.xpath("//span[@class='RetailPrice']/strike/text()").extract()
		if sp:
			item['SP'] = sel.xpath("//em[@class='ProductPrice VariationProductPrice']/text()").extract()[0].replace(",","").split("Rs ")[-1]
			item['MRP'] = sel.xpath("//span[@class='RetailPrice']/strike/text()").extract()[0].replace(",","").split("Rs ")[-1]
		else:
			item['SP'] = ""
			item['MRP'] = sel.xpath("//em[@class='ProductPrice VariationProductPrice']/text()").extract()[0].replace(",","").split("Rs ")[-1]
    
		stock = sel.xpath("//div[@class='stockIcon Out Of Stock']").extract()
		if stock:
			item['Stock'] = "Out of Stock"
		else:
			item['Stock'] = "In Stock"
		
		item['sku'] = sel.xpath("//span[@class='VariationProductSKU']/text()").extract()[0].strip()
		item['index'] = lys_urls.index(response.url)
		item['URL'] = response.url
		
			
		try:
			snapdeal_url = snapdeal_urls[item['index']]
			request = Request(snapdeal_url,callback = self.snapdeal_scraper)
			request.meta['item'] = item			
			yield request
		except:	
			
			try:
				amazon_url = amazon_urls[item['index']]
				request = Request(amazon_url,
									headers={'Referer':'http://amazon.in'},
									callback = self.amazon_scraper)
				request.meta['item'] = item
				request.meta['proxy'] = "http://111.161.126.100:80"
				yield request
			
			except:				
				try:
					flipkart_url = flipkart_urls[item['index']]
					request = Request(flipkart_url,callback = self.flipkart_scraper)
					request.meta['item'] = item
					# request.meta['proxy'] = "http://111.161.126.100:80"
					yield request
			
				except:			
					try:
						paytm_url = paytm_urls[item['index']]
						request = Request(paytm_url,callback = self.paytm_scraper)
						request.meta['item'] = item
						request.meta['proxy'] = "http://111.161.126.100:80"
						yield request
					except:
						pass
开发者ID:Diwahars，项目名称:scrapers，代码行数:64，代码来源:PopularProductsScraper.py

示例5: amazon_scraper

# 需要导入模块: from scrapy.http.request import Request [as 别名]
# 或者: from scrapy.http.request.Request import meta['proxy'] [as 别名]
	def amazon_scraper(self,response):		
		print "Amazon"
		sel = Selector(response)
		item = response.meta['item']
		item['Amazon_URL']= response.url		
		try:				
			item['Amazon_ProductName'] = sel.xpath("//h1/span[@id='productTitle']/text()").extract()[0].replace(",","")					
			mrp = sel.xpath("//td[@class='a-span12 a-color-secondary a-size-base a-text-strike']/text()").extract()
			saleprice = sel.xpath("//span[@id='priceblock_saleprice']/text()").extract()
			ourprice = sel.xpath("//span[@id='priceblock_ourprice']/text()").extract()
			saleshipping = sel.xpath("//span[@class='a-size-base a-color-secondary']/text()").extract()
			
			item['Amazon_Stock'] = 'In Stock' 
			if mrp and saleprice:
				item['Amazon_SP'] = saleprice[0].replace(",","")
				item['Amazon_MRP'] = mrp[0].replace(",","")
			elif mrp and ourprice:
				item['Amazon_SP'] = ourprice[0].replace(",","")
				item['Amazon_MRP'] = mrp[0].replace(",","")
			elif saleshipping and mrp:
				item['Amazon_SP'] = sel.xpath("//span[@class='a-size-base a-color-secondary']/text()").extract()[0].replace(",","")
				item['Amazon_MRP'] = mrp[0].replace(",","")
			elif saleprice:
				item['Amazon_SP'] = ''
				item['Amazon_MRP'] = saleprice[0].replace(",","")
			elif ourprice:
				item['Amazon_MRP'] = ourprice[0].replace(",","")
				item['Amazon_SP'] =''
			elif mrp:
				item['Amazon_MRP'] = mrp[0].replace(",","")				
			else:
				item['Amazon_MRP'] =''			
				item['Amazon_SP'] =''
				item['Amazon_Stock'] = 'Out Of Stock' 
				
			marketplace_url = sel.xpath("//span[@class='a-size-medium a-color-success']//a/@href").extract()
			if marketplace_url:
				marketplace_url = 'http://www.amazon.in/'+marketplace_url[0]
				request = Request(marketplace_url,
									headers={'Referer':'http://amazon.in'},
									callback = self.amazon_marketplace)
				request.meta['item'] = item
				request.meta['proxy'] = "http://111.161.126.100:80"				
				yield request		
		except:			
			item['Amazon_ProductName'] = item['Amazon_MRP' ]= item['Amazon_SP'] = item['Amazon_Match'] = ''
			item['Amazon_Stock'] = 'Not Found'		
		
		
		try:
			flipkart_url = flipkart_urls[item['index']]
			request = Request(flipkart_url,callback = self.flipkart_scraper)
			request.meta['item'] = item
			# request.meta['proxy'] = "http://111.161.126.100:80"
			yield request
	
		except:				
			try:
				paytm_url = paytm_urls[item['index']]
				request = Request(paytm_url,callback = self.paytm_scraper)
				request.meta['item'] = item
				request.meta['proxy'] = "http://111.161.126.100:80"
				yield request
			except:
				self.to_csv(item)
开发者ID:Diwahars，项目名称:scrapers，代码行数:67，代码来源:PopularProductsScraper.py


注：本文中的scrapy.http.request.Request.meta['proxy']方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。