当前位置: 首页>>代码示例>>Python>>正文

Python ScrapyCommand.process_options方法代码示例

本文整理汇总了Python中scrapy.command.ScrapyCommand.process_options方法的典型用法代码示例。如果您正苦于以下问题：Python ScrapyCommand.process_options方法的具体用法？Python ScrapyCommand.process_options怎么用？Python ScrapyCommand.process_options使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类scrapy.command.ScrapyCommand的用法示例。


在下文中一共展示了ScrapyCommand.process_options方法的10个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。

示例1: process_options

# 需要导入模块: from scrapy.command import ScrapyCommand [as 别名]
# 或者: from scrapy.command.ScrapyCommand import process_options [as 别名]
 def process_options(self, args, opts):
     ScrapyCommand.process_options(self, args, opts)
     try:
         opts.spargs = arglist_to_dict(opts.spargs)
     except ValueError:
         raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
     if opts.overwrite_output:
         if opts.output:
             raise UsageError("Please use only one of --output and --overwrite-output")
         opts.output = opts.overwrite_output
         self.settings.overrides['FEED_OVERWRITE'] = True
     if opts.output:
         if opts.output == '-':
             self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
         else:
             self.settings.set('FEED_URI', opts.output, priority='cmdline')
         valid_output_formats = (
             list(self.settings.getdict('FEED_EXPORTERS').keys()) +
             list(self.settings.getdict('FEED_EXPORTERS_BASE').keys())
         )
         if not opts.output_format:
             opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
         if opts.output_format not in valid_output_formats:
             raise UsageError("Unrecognized output format '%s', set one"
                              " using the '-t' switch or as a file extension"
                              " from the supported list %s" % (opts.output_format,
                                                               tuple(valid_output_formats)))
         self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
开发者ID:yhager，项目名称:scrapy，代码行数:30，代码来源:crawl.py

示例2: process_options

# 需要导入模块: from scrapy.command import ScrapyCommand [as 别名]
# 或者: from scrapy.command.ScrapyCommand import process_options [as 别名]
 def process_options(self, args, opts):
     ScrapyCommand.process_options(self, args, opts)
     try:
         opts.spargs = arglist_to_dict(opts.spargs)
     except ValueError:
         raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
     if opts.nofollow:
         settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
开发者ID:bihicheng，项目名称:scrapy，代码行数:10，代码来源:crawl.py

示例3: process_options

# 需要导入模块: from scrapy.command import ScrapyCommand [as 别名]
# 或者: from scrapy.command.ScrapyCommand import process_options [as 别名]
 def process_options(self, args, opts):
     ScrapyCommand.process_options(self, args, opts)
     try:
         opts.spargs = arglist_to_dict(opts.spargs)
     except ValueError:
         raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
     if opts.output:
         if opts.output == '-':
             self.settings.overrides['FEED_URI'] = 'stdout:'
         else:
             self.settings.overrides['FEED_URI'] = opts.output
         self.settings.overrides['FEED_FORMAT'] = opts.output_format
开发者ID:Aaron1011，项目名称:oh-mainline，代码行数:14，代码来源:crawl.py

示例4: process_options

# 需要导入模块: from scrapy.command import ScrapyCommand [as 别名]
# 或者: from scrapy.command.ScrapyCommand import process_options [as 别名]
 def process_options(self, args, opts):
     ScrapyCommand.process_options(self, args, opts)
     try:
         opts.spargs = arglist_to_dict(opts.spargs)
     except ValueError:
         raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
     if opts.output:
         if opts.output == '-':
             self.settings.overrides['FEED_URI'] = 'stdout:'
         else:
             self.settings.overrides['FEED_URI'] = opts.output
         valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys()
         if opts.output_format not in valid_output_formats:
             raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format, valid_output_formats))
         self.settings.overrides['FEED_FORMAT'] = opts.output_format
开发者ID:1012，项目名称:scrapy，代码行数:17，代码来源:crawl.py

示例5: execute

# 需要导入模块: from scrapy.command import ScrapyCommand [as 别名]
# 或者: from scrapy.command.ScrapyCommand import process_options [as 别名]
def execute(argv=None):
    if argv is None:
        argv = sys.argv

    cmds = _get_commands_dict()

    cmdname = _get_command_name(argv)
    _update_default_settings("scrapy.conf.commands", cmdname)
    _update_default_settings(settings["COMMANDS_SETTINGS_MODULE"], cmdname)

    parser = optparse.OptionParser(
        formatter=optparse.TitledHelpFormatter(), conflict_handler="resolve", add_help_option=False
    )

    if cmdname in cmds:
        cmd = cmds[cmdname]
        cmd.add_options(parser)
        opts, args = parser.parse_args(args=argv[1:])
        cmd.process_options(args, opts)
        parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax())
        parser.description = cmd.long_desc()
        if cmd.requires_project and not settings.settings_module:
            print "Error running: scrapy-ctl.py %s\n" % cmdname
            print "Cannot find project settings module in python path: %s" % settings.settings_module_path
            sys.exit(1)
        if opts.help:
            parser.print_help()
            sys.exit()
    elif not cmdname:
        cmd = ScrapyCommand()
        cmd.add_options(parser)
        opts, args = parser.parse_args(args=argv)
        cmd.process_options(args, opts)
        _print_usage(settings.settings_module)
        sys.exit(2)
    else:
        print "Unknown command: %s\n" % cmdname
        print 'Use "scrapy-ctl.py -h" for help'
        sys.exit(2)

    del args[0]  # remove command name from args
    send_catch_log(signal=command_executed, cmdname=cmdname, cmdobj=cmd, args=args, opts=opts)
    from scrapy.core.manager import scrapymanager

    scrapymanager.configure(control_reactor=True)
    ret = _run_command(cmd, args, opts)
    if ret is False:
        parser.print_help()
开发者ID:kenzouyeh，项目名称:scrapy，代码行数:50，代码来源:cmdline.py

示例6: process_options

# 需要导入模块: from scrapy.command import ScrapyCommand [as 别名]
# 或者: from scrapy.command.ScrapyCommand import process_options [as 别名]
 def process_options(self, args, opts):
     ScrapyCommand.process_options(self, args, opts)
     try:
         opts.spargs = arglist_to_dict(opts.spargs)
     except ValueError:
         raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
     if opts.output:
         if opts.output == '-':
             self.settings.overrides['FEED_URI'] = 'stdout:'
         else:
             self.settings.overrides['FEED_URI'] = opts.output
         valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys()
         if not opts.output_format:
             opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
         if opts.output_format not in valid_output_formats:
             raise UsageError("Unrecognized output format '%s', set one"
                              " using the '-t' switch or as a file extension"
                              " from the supported list %s" % (opts.output_format,
                                                               tuple(valid_output_formats)))
         self.settings.overrides['FEED_FORMAT'] = opts.output_format
开发者ID:0xfab，项目名称:scrapy，代码行数:22，代码来源:runspider.py

示例7: process_options

# 需要导入模块: from scrapy.command import ScrapyCommand [as 别名]
# 或者: from scrapy.command.ScrapyCommand import process_options [as 别名]
 def process_options(self, args, opts):
     ScrapyCommand.process_options(self, args, opts)
     try:
         opts.spargs = arglist_to_dict(opts.spargs)
     except ValueError:
         raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
     if opts.output:
         if opts.output == "-":
             self.settings.set("FEED_URI", "stdout:", priority="cmdline")
         else:
             self.settings.set("FEED_URI", opts.output, priority="cmdline")
         valid_output_formats = list(self.settings.getdict("FEED_EXPORTERS").keys()) + list(
             self.settings.getdict("FEED_EXPORTERS_BASE").keys()
         )
         if not opts.output_format:
             opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
         if opts.output_format not in valid_output_formats:
             raise UsageError(
                 "Unrecognized output format '%s', set one"
                 " using the '-t' switch or as a file extension"
                 " from the supported list %s" % (opts.output_format, tuple(valid_output_formats))
             )
         self.settings.set("FEED_FORMAT", opts.output_format, priority="cmdline")
开发者ID:miminus，项目名称:Spider_demo，代码行数:25，代码来源:crawlall.py

示例8: process_options

# 需要导入模块: from scrapy.command import ScrapyCommand [as 别名]
# 或者: from scrapy.command.ScrapyCommand import process_options [as 别名]
 def process_options(self, args, opts):
     ScrapyCommand.process_options(self, args, opts)
     if opts.nofollow:
         settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
开发者ID:serkanh，项目名称:scrapy，代码行数:6，代码来源:crawl.py

示例9: process_options

# 需要导入模块: from scrapy.command import ScrapyCommand [as 别名]
# 或者: from scrapy.command.ScrapyCommand import process_options [as 别名]
 def process_options(self, args, opts):
     ScrapyCommand.process_options(self, args, opts)
     try:
         opts.spargs = arglist_to_dict(opts.spargs)
     except ValueError:
         raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
开发者ID:AlekseyEf，项目名称:scrapy，代码行数:8，代码来源:parse.py

示例10: process_options

# 需要导入模块: from scrapy.command import ScrapyCommand [as 别名]
# 或者: from scrapy.command.ScrapyCommand import process_options [as 别名]
 def process_options(self, args, opts):
     ScrapyCommand.process_options(self, args, opts)
开发者ID:dangra，项目名称:scrapy-sci，代码行数:4，代码来源:review.py


注：本文中的scrapy.command.ScrapyCommand.process_options方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。