Python command.ScrapyCommand类代码示例

 def process_options(self, args, opts):
     ScrapyCommand.process_options(self, args, opts)
     try:
         opts.spargs = arglist_to_dict(opts.spargs)
     except ValueError:
         raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
     if opts.overwrite_output:
         if opts.output:
             raise UsageError("Please use only one of --output and --overwrite-output")
         opts.output = opts.overwrite_output
         self.settings.overrides['FEED_OVERWRITE'] = True
     if opts.output:
         if opts.output == '-':
             self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
         else:
             self.settings.set('FEED_URI', opts.output, priority='cmdline')
         valid_output_formats = (
             list(self.settings.getdict('FEED_EXPORTERS').keys()) +
             list(self.settings.getdict('FEED_EXPORTERS_BASE').keys())
         )
         if not opts.output_format:
             opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
         if opts.output_format not in valid_output_formats:
             raise UsageError("Unrecognized output format '%s', set one"
                              " using the '-t' switch or as a file extension"
                              " from the supported list %s" % (opts.output_format,
                                                               tuple(valid_output_formats)))
         self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')

 def add_options(self, parser):
     ScrapyCommand.add_options(self, parser)
     parser.add_option("-p", "--project", help="the project name in the target")
     parser.add_option("-v", "--version", help="the version to deploy. Defaults to current timestamp")
     parser.add_option("-l", "--list-targets", action="store_true", help="list available targets")
     parser.add_option("-L", "--list-projects", metavar="TARGET", help="list available projects on TARGET")
     parser.add_option("--egg", metavar="FILE", help="use the given egg, instead of building it")

 def add_options(self, parser):
     ScrapyCommand.add_options(self, parser)
     parser.add_option("--spider", dest="spider", default=None, help="use this spider without looking for one")
     parser.add_option(
         "--nolinks", dest="nolinks", action="store_true", help="don't show links to follow (extracted requests)"
     )
     parser.add_option("--noitems", dest="noitems", action="store_true", help="don't show scraped items")
     parser.add_option(
         "--nocolour", dest="nocolour", action="store_true", help="avoid using pygments to colorize the output"
     )
     parser.add_option(
         "-r", "--rules", dest="rules", action="store_true", help="use CrawlSpider rules to discover the callback"
     )
     parser.add_option(
         "-c", "--callback", dest="callback", help="use this callback for parsing, instead looking for a callback"
     )
     parser.add_option(
         "-d",
         "--depth",
         dest="depth",
         type="int",
         default=1,
         help="maximum depth for parsing requests [default: %default]",
     )
     parser.add_option(
         "-v", "--verbose", dest="verbose", action="store_true", help="print each depth level one by one"
     )

 def add_options(self, parser):
     ScrapyCommand.add_options(self, parser)
     parser.add_option("--list", dest="list", action="store_true")
     parser.add_option("--dump", dest="dump", action="store_true")
     parser.add_option("-t", "--template", dest="template", default="crawl",
         help="Uses a custom template.")
     parser.add_option("--force", dest="force", action="store_true",
         help="If the spider already exists, overwrite it with the template")

 def add_options(self, parser):
     ScrapyCommand.add_options(self, parser)
     parser.add_option("--spider", dest="spider", default=None, \
         help="always use this spider when arguments are urls")
     parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \
         help="set spider argument (may be repeated)")
     parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", \
         help="don't follow links (for use with URLs only)")

 def process_options(self, args, opts):
     ScrapyCommand.process_options(self, args, opts)
     try:
         opts.spargs = arglist_to_dict(opts.spargs)
     except ValueError:
         raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
     if opts.nofollow:
         settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False

 def add_options(self, parser):
     ScrapyCommand.add_options(self, parser)
     parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
                       help="set spider argument (may be repeated)")
     parser.add_option("-o", "--output", metavar="FILE",
                       help="dump scraped items into FILE (use - for stdout)")
     parser.add_option("-t", "--output-format", metavar="FORMAT", default="jsonlines",
                       help="format to use for dumping items with -o (default: %default)")

 def add_options(self, parser):
     ScrapyCommand.add_options(self, parser)
     parser.add_option(
         "--verbose",
         "-v",
         dest="verbose",
         action="store_true",
         help="also display twisted/python/platform info (useful for bug reports)",
     )

 def add_options(self, parser):
     ScrapyCommand.add_options(self, parser)
     parser.add_option("-d", "--date", help="Print data for given date (in format yyyy-mm-dd, by default is last stored data)")
     parser.add_option("-c", "--compare", metavar="DATE", help="Compare with data on given date (in format yyyy-mm-dd)")
     parser.add_option("-D", "--day", action="store_true", help="Compare with previous day")
     parser.add_option("-W", "--week", action="store_true", help="Compare with previous week")
     parser.add_option("-M", "--month", action="store_true", help="Compare with previous month")
     parser.add_option("-4", "--four_month", action="store_true", help="Compare with previous 4 month")
     parser.add_option("-Y", "--year", action="store_true", help="Compare with previous year")
     parser.add_option("--show_year", action="store_true", help="Show year of comparing date in report")

 def add_options(self, parser):
     ScrapyCommand.add_options(self, parser)
     parser.add_option("-l", "--list", dest="list", action="store_true",
         help="List available templates")
     parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
         help="Dump template to standard output")
     parser.add_option("-t", "--template", dest="template", default="crawl",
         help="Uses a custom template.")
     parser.add_option("--force", dest="force", action="store_true",
         help="If the spider already exists, overwrite it with the template")

 def add_options(self, parser):
     ScrapyCommand.add_options(self, parser)
     parser.add_option("--server", dest="server", action="store_true", help="Run scraper server that polls for tasks and runs them")
     parser.add_option("--demonize", dest="demonize", action="store_true", help="Run scrapy as a demon process continiously processing the spider's tasks.")
     parser.add_option("--all", dest="all", action="store_true", help="Run all available tasks for the spider.")
     parser.add_option('--interval', dest='interval', help="Polling interval for executing the spider in demon mode.")
     parser.add_option('--priority', dest='priority', help="Priority of task that you want to execute." )
     parser.add_option('--task-name', dest='task_name', help="Name of task that you would like to execute.", default=None )
     parser.add_option('--task-id', dest='task_id', help="Id of task that you would like to execute.", default=None )
     parser.add_option('--child-logfile', dest='child_logfile', help="Pass this parameter if you want to log output of child processes.", default=None)
     parser.add_option("--child", dest="child", action="store_true", help="Make this process a child.")

 def add_options(self, parser):
     ScrapyCommand.add_options(self, parser)
     parser.add_option("-s", action='store_true', dest='skip_start_urls', default=False,
                       help="Skip start URLs and work on queue")
     parser.add_option("-w", action='store_true', dest='dont_exit', default=False,
                       help="Don't exit when idle, wait around for more work")
     parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \
         help="set spider argument (may be repeated)")
     parser.add_option("-o", "--output", metavar="FILE", \
         help="dump scraped items into FILE (use - for stdout)")
     parser.add_option("-t", "--output-format", metavar="FORMAT", default="jsonlines", \
         help="format to use for dumping items with -o (default: %default)")

 def add_options(self, parser):
     ScrapyCommand.add_options(self, parser)
     parser.add_option("--get", dest="get", metavar="SETTING", \
         help="print raw setting value")
     parser.add_option("--getbool", dest="getbool", metavar="SETTING", \
         help="print setting value, intepreted as a boolean")
     parser.add_option("--getint", dest="getint", metavar="SETTING", \
         help="print setting value, intepreted as an integer")
     parser.add_option("--getfloat", dest="getfloat", metavar="SETTING", \
         help="print setting value, intepreted as an float")
     parser.add_option("--getlist", dest="getlist", metavar="SETTING", \
         help="print setting value, intepreted as an float")

 def process_options(self, args, opts):
     ScrapyCommand.process_options(self, args, opts)
     try:
         opts.spargs = arglist_to_dict(opts.spargs)
     except ValueError:
         raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
     if opts.output:
         if opts.output == '-':
             self.settings.overrides['FEED_URI'] = 'stdout:'
         else:
             self.settings.overrides['FEED_URI'] = opts.output
         self.settings.overrides['FEED_FORMAT'] = opts.output_format

 def add_options(self, parser):
     ScrapyCommand.add_options(self, parser)
     parser.add_option("-u", "--unreviewed", dest="unreviewed", action="store_true", default=False,
                       help="train and test with unreviewed files")
     parser.add_option("-r", "--reviewed", dest="reviewed", action="store_true", default=True,
                       help="train and test with reviewed files")
                     
     parser.add_option("--report", action="store_true", dest="print_report", default=False,
                       help="Print a detailed classification report.")
     parser.add_option("--confusion_matrix", action="store_true", dest="print_cm", default=False,
                   help="Print the confusion matrix.")
     #Not supported:
     parser.add_option("-t", "--top-n-features", dest="topn", type="int", default=0,
                       help="number of top features to reveal")