Python MRJob.set_up_logging方法代码示例

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
 def test_verbose(self):
     with patch.object(sys, 'stderr', StringIO()) as stderr:
         MRJob.set_up_logging(verbose=True)
         log = logging.getLogger('__main__')
         log.info('INFO')
         log.debug('DEBUG')
         self.assertEqual(stderr.getvalue(), 'INFO\nDEBUG\n')

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
def main(args):
    # parser command-line args
    usage = '%prog [options]'
    description = "Collect EMR stats from active jobflows. "
    description += "Active jobflows are those in states of: "
    description += "BOOTSTRAPPING, RUNNING, STARTING, and WAITING. "
    description += "Collected stats include total number of active jobflows"
    description += "and total number of Amazon EC2 instances used to execute"
    description += "these jobflows. The instance counts are not separated by"
    description += "instance type."
    option_parser = OptionParser(usage=usage, description=description)
    option_parser.add_option(
        "-p", "--pretty-print",
        action="store_true", dest="pretty_print", default=False,
        help=('Pretty print the collected stats'))
    add_basic_opts(option_parser)

    options, args = option_parser.parse_args(args)
    if args:
        option_parser.error('takes no arguments')

    MRJob.set_up_logging(quiet=options.quiet, verbose=options.verbose)
    log.info('collecting EMR active jobflows...')
    job_flows = collect_active_job_flows(options.conf_paths)
    log.info('compiling stats from collected jobflows...')
    stats = job_flows_to_stats(job_flows)

    if options.pretty_print:
        pretty_print(stats)
    else:
        print(json.dumps(stats))

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
def main(args=None):
    now = _boto3_now()

    arg_parser = _make_arg_parser()
    options = arg_parser.parse_args(args)

    MRJob.set_up_logging(quiet=options.quiet, verbose=options.verbose)

    log.info('getting information about running jobs')

    min_time = timedelta(hours=options.min_hours)

    emr_client = EMRJobRunner(**_runner_kwargs(options)).make_emr_client()
    cluster_summaries = _boto3_paginate(
        'Clusters', emr_client, 'list_clusters',
        ClusterStates=['STARTING', 'BOOTSTRAPPING', 'RUNNING'])

    if not options.exclude:
        filtered_cluster_summaries = cluster_summaries
    else:
        filtered_cluster_summaries = _filter_clusters(
            cluster_summaries, emr_client, options.exclude)

    job_info = _find_long_running_jobs(
        emr_client, filtered_cluster_summaries, min_time, now=now)

    _print_report(job_info)

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
def main(cl_args=None):
    arg_parser = _make_arg_parser()
    options = arg_parser.parse_args(cl_args)

    MRJob.set_up_logging(quiet=options.quiet,
                         verbose=options.verbose)

    # max_hours_idle -> max_mins_idle
    max_mins_idle = options.max_mins_idle
    if max_mins_idle is None and options.max_hours_idle is not None:
        log.warning('--max-hours-idle is deprecated and will be removed'
                    ' in v0.7.0. Please use --max-mins-idle instead.')
        max_mins_idle = options.max_hours_idle * 60

    if options.mins_to_end_of_hour is not None:
        log.warning('--mins-to-end-of-hour is deprecated as of v0.6.0'
                    ' and does nothing')

    _maybe_terminate_clusters(
        dry_run=options.dry_run,
        max_mins_idle=max_mins_idle,
        unpooled_only=options.unpooled_only,
        now=_boto3_now(),
        pool_name=options.pool_name,
        pooled_only=options.pooled_only,
        max_mins_locked=options.max_mins_locked,
        quiet=options.quiet,
        **_runner_kwargs(options)
    )

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
def main(cl_args=None):
    parser = _make_arg_parser()
    options = parser.parse_args(cl_args)

    runner_alias = options.runner or _DEFAULT_RUNNER
    runner_class = _runner_class(runner_alias)

    if options.help or not options.script_or_jar:
        _print_help(options, runner_class)
        sys.exit(0)

    MRJob.set_up_logging(
        quiet=options.quiet,
        verbose=options.verbose,
    )

    kwargs = _get_runner_opt_kwargs(options, runner_class)
    kwargs.update(_HARD_CODED_OPTS)

    kwargs['input_paths'] = [os.devnull]

    step = _get_step(options, parser, cl_args)
    kwargs['steps'] = [step.description()]

    runner = runner_class(**kwargs)

    try:
        runner.run()
    finally:
        runner.cleanup()

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
def main(args=None):
    option_parser = make_option_parser()
    options = parse_args(option_parser, args)

    MRJob.set_up_logging(quiet=options.quiet, verbose=options.verbose)

    with EMRJobRunner(**runner_kwargs(options)) as runner:
        perform_actions(options, runner)

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
 def test_default_options(self):
     with no_handlers_for_logger('__main__'):
         with patch.object(sys, 'stderr', StringIO()) as stderr:
             MRJob.set_up_logging()
             log = logging.getLogger('__main__')
             log.info('INFO')
             log.debug('DEBUG')
             self.assertEqual(stderr.getvalue(), 'INFO\n')

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
def main():
    option_parser = make_option_parser()
    try:
        options = parse_args(option_parser)
    except OptionError:
        option_parser.error('This tool takes exactly one argument.')

    MRJob.set_up_logging(quiet=options.quiet, verbose=options.verbose)

    with EMRJobRunner(**runner_kwargs(options)) as runner:
        perform_actions(options, runner)

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
def main(args):
    # parser command-line args
    option_parser = make_option_parser()
    options, args = option_parser.parse_args(args)

    if args:
        option_parser.error('takes no arguments')

    MRJob.set_up_logging(quiet=options.quiet, verbose=options.verbose)

    print_report(options)

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
def main(cl_args=None):
    arg_parser = _make_arg_parser()
    options = arg_parser.parse_args(cl_args)

    MRJob.set_up_logging(quiet=options.quiet, verbose=options.verbose)

    time_old = _process_time(options.time_untouched)

    for path in options.uris:
        _s3_cleanup(path, time_old,
                    dry_run=options.test,
                    **_runner_kwargs(options))

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
def main(cl_args=None):
    # parser command-line args
    arg_parser = _make_arg_parser()
    options = arg_parser.parse_args(cl_args)

    MRJob.set_up_logging(quiet=options.quiet, verbose=options.verbose)

    # create the persistent job
    runner = EMRJobRunner(**_runner_kwargs(options))
    log.debug('Terminating cluster %s' % options.cluster_id)
    runner.make_emr_client().terminate_job_flows(
        JobFlowIds=[options.cluster_id])
    log.info('Terminated cluster %s' % options.cluster_id)

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
def main():
    option_parser = make_option_parser()
    options, args = option_parser.parse_args()

    if args:
        option_parser.error('takes no arguments')

    MRJob.set_up_logging(quiet=options.quiet, verbose=options.verbose)

    inspect_and_maybe_terminate_job_flows(
        conf_path=options.conf_path,
        max_hours_idle=options.max_hours_idle,
        now=datetime.utcnow(),
        dry_run=options.dry_run)

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
def main():
    option_parser = make_option_parser()
    options, args = option_parser.parse_args()

    # make sure time and uris are given
    if not args or len(args) < 2:
        option_parser.error("Please specify time and one or more URIs")

    MRJob.set_up_logging(quiet=options.quiet, verbose=options.verbose)

    time_old = process_time(args[0])

    for path in args[1:]:
        s3_cleanup(path, time_old, conf_path=options.conf_path, dry_run=options.test)

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
def main(cl_args=None):
    option_parser = make_option_parser()
    options, args = option_parser.parse_args(cl_args)

    MRJob.set_up_logging(quiet=options.quiet, verbose=options.verbose)

    # make sure time and uris are given
    if not args or len(args) < 2:
        option_parser.error('Please specify time and one or more URIs')

    time_old = process_time(args[0])

    for path in args[1:]:
        s3_cleanup(path, time_old,
                   dry_run=options.text,
                   **runner_kwargs(options))

# 需要导入模块: from mrjob.job import MRJob [as 别名]
# 或者: from mrjob.job.MRJob import set_up_logging [as 别名]
def main(cl_args=None):
    # parser command-line args
    option_parser = make_option_parser()
    options, args = option_parser.parse_args(cl_args)

    if len(args) != 1:
        option_parser.error('This tool takes exactly one argument.')
    emr_job_flow_id = args[0]

    MRJob.set_up_logging(quiet=options.quiet, verbose=options.verbose)

    # create the persistent job
    runner = EMRJobRunner(conf_paths=options.conf_paths)
    log.debug('Terminating job flow %s' % emr_job_flow_id)
    runner.make_emr_conn().terminate_jobflow(emr_job_flow_id)
    log.info('Terminated job flow %s' % emr_job_flow_id)