当前位置: 首页>>代码示例>>Python>>正文


Python exporters.CsvItemExporter方法代码示例

本文整理汇总了Python中scrapy.exporters.CsvItemExporter方法的典型用法代码示例。如果您正苦于以下问题:Python exporters.CsvItemExporter方法的具体用法?Python exporters.CsvItemExporter怎么用?Python exporters.CsvItemExporter使用的例子?那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在scrapy.exporters的用法示例。


在下文中一共展示了exporters.CsvItemExporter方法的10个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。

示例1: process_item

# 需要导入模块: from scrapy import exporters [as 别名]
# 或者: from scrapy.exporters import CsvItemExporter [as 别名]
def process_item(self, item, spider):
        if spider.name != '':
            asin = item.get('asin', None)
            if asin:
                self.counter += 1
                if self.counter == 10000:
                    self.exporter.finish_exporting()
                    file = self.files.pop(spider)
                    file.close()

                    self.file_count += 1

                    file = open('amazon_products_%s.csv' % (self.file_count),'w+b')
                    self.files[spider] = file
                    self.export_fields = ['title', 'asin', 'category', 'rank', 'new_price1', 'new_price2']
                    self.exporter = CsvItemExporter(file, fields_to_export=self.export_fields)
                    self.exporter.start_exporting()

                    self.counter = 0
                print self.counter, '*'*50
                self.exporter.export_item(item)
        return item 
开发者ID:parul1931,项目名称:amazon,代码行数:24,代码来源:pipelines.py

示例2: spider_opened

# 需要导入模块: from scrapy import exporters [as 别名]
# 或者: from scrapy.exporters import CsvItemExporter [as 别名]
def spider_opened(self, spider):
        fields_to_export = ['mailingList', 'emailId',
                            'senderName', 'senderEmail',
                            'timestampSent', 'timestampReceived',
                            'subject', 'url', 'replyto']

        fields_to_export = [f for f in fields_to_export if f not in spider.drop_fields]

        if not os.path.exists('data'):
            os.makedirs('data')

        if len(spider.scraping_lists) == 1:
            dest_file_path = 'data/{}ByEmail.csv'.format(spider.scraping_lists[0])
            fields_to_export.remove('mailingList')
        else:
            dest_file_path = 'data/{}ByEmail.csv'.format(spider.name)

        dest_file = open(dest_file_path, 'wb')

        self.exporter = CsvItemExporter(dest_file)
        self.files[spider] = dest_file
        self.exporter.fields_to_export = fields_to_export

        self.exporter.start_exporting() 
开发者ID:gaalcaras,项目名称:mailingListScraper,代码行数:26,代码来源:pipelines.py

示例3: spider_opened

# 需要导入模块: from scrapy import exporters [as 别名]
# 或者: from scrapy.exporters import CsvItemExporter [as 别名]
def spider_opened(self, spider):
        files = [
            ('bos', 'bos.csv'),
            ('vitimas', 'vitimas.csv'),
            ('naturezas', 'naturezas-envolvidas.csv'),
        ]
        for key, fname in files:
            file = open(join(spider.target_dir, fname), 'w+b')
            self.files.append(file)
            self.exporters[key] = CsvItemExporter(file)
            self.exporters[key].start_exporting() 
开发者ID:eltermann,项目名称:ssp-transparencia,代码行数:13,代码来源:pipelines.py

示例4: spider_opened

# 需要导入模块: from scrapy import exporters [as 别名]
# 或者: from scrapy.exporters import CsvItemExporter [as 别名]
def spider_opened(self, spider):
        """
        ????????????
        :param spider:
        :return:
        """
        print time.strftime("%Y-%m-%d %H:%M:%S"), 'Pipeline   Signals: spider_opened'
        file_csv = open('%s_items.csv' % spider.name, 'w+b')
        self.files[spider] = file_csv
        self.exporter = CsvItemExporter(file_csv)
        self.exporter.start_exporting() 
开发者ID:zhanghe06,项目名称:scrapy_project,代码行数:13,代码来源:signals.py

示例5: spider_opened

# 需要导入模块: from scrapy import exporters [as 别名]
# 或者: from scrapy.exporters import CsvItemExporter [as 别名]
def spider_opened(self, spider):
        file_csv = open('%s_items.csv' % spider.name, 'w+b')
        self.files[spider] = file_csv
        self.exporter = CsvItemExporter(file_csv)
        self.exporter.start_exporting() 
开发者ID:zhanghe06,项目名称:scrapy_project,代码行数:7,代码来源:exporter_csv.py

示例6: __init__

# 需要导入模块: from scrapy import exporters [as 别名]
# 或者: from scrapy.exporters import CsvItemExporter [as 别名]
def __init__(self):
        self.file = open("quotedata.csv", 'wb')
        self.exporter = CsvItemExporter(self.file, unicode)
        self.exporter.start_exporting() 
开发者ID:zseta,项目名称:scrapyfundamentals,代码行数:6,代码来源:pipelines.py

示例7: __init__

# 需要导入模块: from scrapy import exporters [as 别名]
# 或者: from scrapy.exporters import CsvItemExporter [as 别名]
def __init__(self):
        self.file = open("booksdata.csv", 'wb')
        self.exporter = CsvItemExporter(self.file, unicode)
        self.exporter.start_exporting() 
开发者ID:zseta,项目名称:scrapyfundamentals,代码行数:6,代码来源:pipelines.py

示例8: spider_opened

# 需要导入模块: from scrapy import exporters [as 别名]
# 或者: from scrapy.exporters import CsvItemExporter [as 别名]
def spider_opened(self, spider):
        file = open('jd_{}.csv'.format(self.file_count), 'w+b')
        self.files[spider] = file
        self.exporter = CsvItemExporter(file)
        self.exporter.start_exporting() 
开发者ID:poluo,项目名称:Spider,代码行数:7,代码来源:pipelines.py

示例9: spider_opened

# 需要导入模块: from scrapy import exporters [as 别名]
# 或者: from scrapy.exporters import CsvItemExporter [as 别名]
def spider_opened(self, spider):
        if spider.name != '':
            file = open('amazon_products_%s.csv' % (self.file_count),'w+b')
            self.files[spider] = file
            self.export_fields = ['title', 'asin', 'category', 'rank', 'new_price1', 'new_price2']
            self.exporter = CsvItemExporter(file, fields_to_export=self.export_fields)
            self.exporter.start_exporting() 
开发者ID:parul1931,项目名称:amazon,代码行数:9,代码来源:pipelines.py

示例10: spider_opened

# 需要导入模块: from scrapy import exporters [as 别名]
# 或者: from scrapy.exporters import CsvItemExporter [as 别名]
def spider_opened(self, spider):
        
        spider_name = re.sub('-.+?$', '', spider.name)
        self.filename = spider_name + ".csv"
        
        file = open('output/' + self.filename, 'w+b', 0)
        exporter = CsvItemExporter(file)
        
        EXPORT_FIELDS = [
            'BrandTitle',
            'BrandPictureLink',
            'BrandDescription',
            'CategoryTitle',
            'CategoryPictureLink',
            'CategoryDescription',
            'ProductTitle',
            'ProductPictureLink',
            'ProductThcContent',
            'ProductCbdContent',
            'ProductDescription',
            'ProductURL',      
        ]
                
        exporter.fields_to_export = EXPORT_FIELDS
        exporter.start_exporting()

        self.exporters['Result'] = exporter 
开发者ID:gangabass,项目名称:weedmaps,代码行数:29,代码来源:pipelines.py


注:本文中的scrapy.exporters.CsvItemExporter方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。