Python exporter.CsvItemExporter类代码示例

class CsdnPipeline(object):
    users={}
    def __init__(self):
        self.file=None
        self.exporter=None
    def set_file(self,filename):
        self.file=open(filename,'wb')
        self.exporter=CsvItemExporter(self.file)

    def process_item(self, item, spider):
        if spider.name=="csdn.user":
            if self.file is None:
                self.set_file("export_users.csv")
            else:
                self.exporter.export_item(item)
        if spider.name=="csdn.login":
            if item['username']:
                sql='update t_csdn_users set real_password=:password,real_email=:email where username=:username';
                username=item['username']
                password=item['password']
                email=item['email']
                param={'username':username,'password':password,'email':email}
                spider.oracle_db.execute_sql(sql,param,False)
                log.msg("username:"+username+"\tpassword:"+password,level=log.INFO)
        return item

    def spider_open(self, spider):
        self.csv_exporter_item   = CsvItemExporter(open("carepack.csv", "w"),quoting=csv.QUOTE_ALL )
        self.csv_exporter_detail = CsvItemExporter(open("model.csv"   ,"w" ),quoting=csv.QUOTE_ALL )

        # Make a quick copy of the list
        self.csv_exporter_item.start_exporting()
        self.csv_exporter_detail.start_exporting()

 def spider_opened(self, spider):
     file = open('%s_%s.csv' % (spider.name, int(time.time())), 'w+b')
     self.files[spider] = file
     if 'yopt' in spider.name:
         self.exporter = CsvItemExporter(file,fields_to_export = ['date','instrument','option_symbol','symbol','expiration','type','strike','last','change','bid','ask','volume','open_int'],dialect='excel')
     elif 'prices' in spider.name:
         self.exporter = CsvItemExporter(file,fields_to_export = ['date','open','high','low','close','volume','adj_close'],dialect='excel')
     self.exporter.start_exporting()

class ExportCSV(ExportData):
    """
    Exporting to export/csv/spider-name.csv file
    """
    def spider_opened(self, spider):
        file_to_save = open('exports/csv/%s.csv' % spider.name, 'w+b')
        self.files[spider] = file_to_save
        self.exporter = CsvItemExporter(file_to_save)
        self.exporter.start_exporting()

class WriteToCsv(object):
	@classmethod
	
	def process_item(self, item, spider):
		self.file = open(settings.csv_file_path, 'ab+')
		self.exporter = CsvItemExporter(self.file, include_headers_line=False)
		self.exporter.fields_to_export = settings.csv_export_fields
		self.exporter.export_item(item)
		return item

    def __init__(self):
        self.keystat_file = open('yahoo_keystat.csv', 'w+b')
        self.summary_file = open('yahoo_summary.csv', 'w+b')
        self.summary_exporter = CsvItemExporter(self.summary_file)
        self.keystat_exporter = CsvItemExporter(self.keystat_file)
        self.summary_exporter.start_exporting()
        self.keystat_exporter.start_exporting()

        dispatcher.connect(self.spider_closed, signals.spider_closed)

    def __init__(self):
        self.files = {}
        self.exporter1 = CsvItemExporter(fields_to_export=ProfRatingItem.fields.keys(),file=open("profRating.csv",'wb'))
        self.exporter2 = CsvItemExporter(fields_to_export=ProfSummaryItem.fields.keys(),file=open("profSummary.csv",'wb'))

	@classmethod
	def from_crawler(cls, crawler):
		pipeline = cls()
		crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
		crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
		return pipeline

 def _exporter(self, item, spider):
     if spider.name not in self.files:
         self.files[spider.name] = {}
     name = item.__class__.__name__.lower()
     if name not in self.files[spider.name]:
         f = open('%s_%s.csv' % (spider.name, name), 'w+b')
         exporter = CsvItemExporter(f)
         exporter.start_exporting()
         self.files[spider.name][name] = (exporter, f)
     else:
         exporter, f = self.files[spider.name][name]
     return exporter

 def spider_opened(self, spider):
   file = open("mediabase.csv", 'w+b')
   self.files[spider] = file
   self.exporter = CsvItemExporter(file)
   #self.exporter.fields_to_export = ["Name","Address","City","Neighborhood","State","Zip","Phone","Website","Image_url","Hours_Mon","Hours_Tue","Hours_Wed","Hours_Thu","Hours_Fri","Hours_Sat","Hours_Sun","Price","TakesReservation","Delivery","TakeOut","AcceptsCreditCards","GoodFor","Parking","WheelChairAccessible","BikeParking","GoodForKids","GoodForGroups","Attire","Ambience","NoiseLevel","Alcohol","OutDoorSeating","Wifi","HasTV","WaiterService","Caters","Url"]
   self.exporter.fields_to_export = ["Type","Area","PlaceName","Web","Tel","Address","Zip","Town","Hours","CompanyName","OrganizationNo","Turnover","Employed","LastName","FirstName","Telephone","AllabolagUrl","EniroUrl"]
   self.exporter.start_exporting()

 def spider_opened(self, spider):
     file = open('%s_items.csv' % spider.name, 'w+b')
     self.files[spider] = file
     self.exporter = CsvItemExporter(file)
     list = ['id','title', 'time', 'director', 'year', 'star','cost']
     self.exporter.fields_to_export = list
     self.exporter.start_exporting()

 def spider_opened(self, spider):
     file = open("%s_FL_Orlando_20150716.csv" % spider.name, "w+b")
     self.files[spider] = file
     self.exporter = CsvItemExporter(file)
     self.exporter.fields_to_export = [
         "state",
         "city",
         "neighborhood",
         "zip_code",
         "listing_type",
         "property_type",
         "construction",
         "lot",
         "mls_number",
         "parcel",
         "price",
         "zestimate",
         "zestimate_rent",
         "built_in",
         "bedrooms",
         "baths",
         "address",
         "description",
         "listing_provided_by",
         "url",
         "timestamp",
     ]
     self.exporter.start_exporting()

	def spider_opened(self, spider):
		if spider.name in 'realestate':
			self.file = open('current_listing.csv', 'w+b')
		else:
			self.file = open('past_listing.csv', 'w+b')
		self.exporter = CsvItemExporter(self.file)
		self.exporter.start_exporting()

 def spider_opened(self, spider):
     file = open('%s_%s_items.csv' % (spider.name, spider.category), 'w+b')
     self.files[spider] = file
     self.exporter = CsvItemExporter(file)
     self.exporter.fields_to_export = ['vendor', 'product', 'default']
     #self.exporter.fields_to_export = ['default']
     self.exporter.start_exporting()

 def spider_opened(self, spider):
     file = open('/tmp/%s_log.csv' % spider.name, 'w+b')
     self.files[spider] = file
     self.exporter = CsvItemExporter(file)
     self.exporter.fields_to_export = ['_id', 'name', 'count', 'free', 'timestamp', 'lat', 'lon', 'url']
     self.exporter.include_headers_line = 'false';
     self.exporter.start_exporting()

	def open_spider(self,spider):
		self.file=open('wechat.csv','w+b')
		self.file.write('\xEF\xBB\xBF')
		self.exporter=CsvItemExporter(self.file)
		self.exporter.fields_to_export = ['page','wID','wtitle','wsub','Link']
		# self.exporter.fields_to_export = ['jobTitle','salary','education','ex','age','num','contact','location','updatetime','url','Requirement','comInfo','delivery_num']
		self.exporter.start_exporting()