Python Log.start方法代码示例

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def main():

    try:
        settings = startup.read_settings()
        constants.set(settings.constants)
        Log.start(settings.debug)

        some_failures = http.post_json("http://activedata.allizom.org/query", data={
            "from": "unittest",
            "select": [
                {"name": "branch", "value": "build.branch"},
                {"name": "revision", "value": "build.revision12"},
                {"name": "suite", "value": "run.suite"},
                {"name": "chunk", "value": "run.chunk"},
                {"name": "test", "value": "result.test"}
            ],
            "where": {"and": [
                {"eq": {"result.ok": False}},
                {"gt": {"run.timestamp": Date.today() - WEEK}},
                {"missing": "treeherder.job.note"}
            ]},
            "format": "list",
            "limit": 10
        })


        th = TreeHerder(settings={})

        # th.get_job_classification("mozilla-inbound", "7380457b8ba0")
        for f in some_failures.data:
            th.get_job_classification(f.branch, f.revision)

    except Exception, e:
        Log.error("Problem with etl", e)

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def main():
    try:
        settings = startup.read_settings()
        Log.start(settings.debug)
        constants.set(settings.constants)

        with startup.SingleInstance(flavor_id=settings.args.filename):
            with aws.s3.Bucket(settings.destination) as bucket:

                if settings.param.debug:
                    if settings.source.durable:
                        Log.error("Can not run in debug mode with a durable queue")
                    synch = SynchState(bucket.get_key(SYNCHRONIZATION_KEY, must_exist=False))
                else:
                    synch = SynchState(bucket.get_key(SYNCHRONIZATION_KEY, must_exist=False))
                    if settings.source.durable:
                        synch.startup()

                queue = PersistentQueue(settings.param.queue_file)
                if queue:
                    last_item = queue[len(queue) - 1]
                    synch.source_key = last_item._meta.count + 1

                with pulse.Consumer(settings=settings.source, target=None, target_queue=queue, start=synch.source_key):
                    Thread.run("pulse log loop", log_loop, settings, synch, queue, bucket)
                    Thread.wait_for_shutdown_signal(allow_exit=True)
                    Log.warning("starting shutdown")

                queue.close()
                Log.note("write shutdown state to S3")
                synch.shutdown()

    except Exception, e:
        Log.error("Problem with etl", e)

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def main():
    try:
        settings = startup.read_settings()
        Log.start(settings.debug)
        backfill(settings)
    except Exception, e:
        Log.error("Problem with backfill", e)

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def main():
    """
    CLEAR OUT KEYS FROM BUCKET BY RANGE, OR BY FILE
    """
    settings = startup.read_settings(defs=[
        {
            "name": ["--bucket"],
            "help": "bucket to scan",
            "type": str,
            "dest": "bucket",
            "required": True
        }
    ])
    Log.start(settings.debug)

    source = Connection(settings.aws).get_bucket(settings.args.bucket)

    for k in qb.sort(source.keys()):
        try:
            data = source.read_bytes(k)
            if convert.ascii2unicode(data).find("2e2834fa7ecd8d3bb1ad49ec981fdb89eb4df95e18") >= 0:
                Log.note("Found at {{key}}", key=k)
        except Exception, e:
            Log.warning("Problem with {{key}}", key=k, cause=e)
        finally:

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def main():

    try:
        settings = startup.read_settings(defs=[{
            "name": ["--id"],
            "help": "id(s) to process.  Use \"..\" for a range.",
            "type": str,
            "dest": "id",
            "required": False
        }])
        constants.set(settings.constants)
        Log.start(settings.debug)

        if settings.args.id:
            etl_one(settings)
            return

        hg = HgMozillaOrg(settings=settings.hg)
        resources = Dict(hg=dictwrap(hg))
        stopper = Signal()
        for i in range(coalesce(settings.param.threads, 1)):
            ETL(
                name="ETL Loop " + unicode(i),
                work_queue=settings.work_queue,
                resources=resources,
                workers=settings.workers,
                settings=settings.param,
                please_stop=stopper
            )

        Thread.wait_for_shutdown_signal(stopper, allow_exit=True)
    except Exception, e:
        Log.error("Problem with etl", e)

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def start():
    try:
        settings = startup.read_settings()
        Log.start(settings.debug)
        main(settings)
    except Exception, e:
        Log.error("Problems exist", e)

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def main():
    settings = startup.read_settings()
    Log.start(settings.debug)
    try:
        with Multithread(update_repo, threads=10, outbound=False) as multi:
            for repo in Random.combination(settings.param.repos):
                multi.execute([{"repos": repo, "settings": settings}])
    finally:
        Log.stop()

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def main():
    settings = startup.read_settings()
    constants.set(settings.constants)
    Log.start(settings.debug)
    with startup.SingleInstance(flavor_id=settings.args.filename):
        try:
            full_etl(settings)
        finally:
            Log.stop()

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def main():

    try:
        settings = startup.read_settings()
        constants.set(settings.constants)
        Log.start(settings.debug)

        hg = HgMozillaOrg(settings.hg)
        th = TreeHerder(settings=settings)
        find_some_work(th)
    except Exception, e:
        Log.error("Problem with etl", e)

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def main():
    settings = startup.read_settings(defs={
       "name": ["--restart", "--reset", "--redo"],
       "help": "force a reprocessing of all data",
       "action": "store_true",
       "dest": "restart"
    })
    Log.start(settings.debug)

    try:
        with startup.SingleInstance(flavor_id=settings.args.filename):
            if settings.args.restart:
                reviews = Cluster(settings.destination).create_index(settings.destination)
            else:
                reviews = Cluster(settings.destination).get_proto(settings.destination)

            bugs = Cluster(settings.source).get_index(settings.source)

            with FromES(bugs) as esq:
                es_max_bug = esq.query({
                    "from": "private_bugs",
                    "select": {"name": "max_bug", "value": "bug_id", "aggregate": "maximum"}
                })

            #PROBE WHAT RANGE OF BUGS IS LEFT TO DO (IN EVENT OF FAILURE)
            with FromES(reviews) as esq:
                es_min_bug = esq.query({
                    "from": "reviews",
                    "select": {"name": "min_bug", "value": "bug_id", "aggregate": "minimum"}
                })

            batch_size = coalesce(bugs.settings.batch_size, settings.size, 1000)
            threads = coalesce(settings.threads, 4)
            Log.note(str(settings.min_bug))
            min_bug = int(coalesce(settings.min_bug, 0))
            max_bug = int(coalesce(settings.max_bug, Math.min(es_min_bug + batch_size * threads, es_max_bug)))

            with ThreadedQueue(reviews, batch_size=coalesce(reviews.settings.batch_size, 100)) as sink:
                func = functools.partial(full_etl, settings, sink)
                with Multithread(func, threads=threads) as m:
                    m.inbound.silent = True
                    Log.note("bugs from {{min}} to {{max}}, step {{step}}", {
                        "min": min_bug,
                        "max": max_bug,
                        "step": batch_size
                    })
                    m.execute(reversed([{"bugs": range(s, e)} for s, e in qb.intervals(min_bug, max_bug, size=1000)]))

            if settings.args.restart:
                reviews.add_alias()
                reviews.delete_all_but_self()
    finally:
        Log.stop()

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def main():
    try:
        settings = startup.read_settings()
        Log.start(settings.debug)

        source = get_container(settings.source)
        destination = get_container(settings.destination)

        work_queue = aws.Queue(settings.work_queue)
        backfill(source, destination, work_queue, settings)
    except Exception, e:
        Log.error("Problem with backfill", e)

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def main():
    try:
        settings = startup.read_settings()
        constants.set(settings.constants)
        Log.start(settings.debug)

        big_data.MAX_STRING_SIZE = 100 * 1000 * 1000

        # get_active_data(settings)
        get_bugs(settings)
    except Exception, e:
        Log.error("Problem with etl", e)

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def main():
    global all_creds
    global config

    try:
        config = startup.read_settings()
        constants.set(config.constants)
        Log.start(config.debug)

        all_creds = config.users

        app.run(**config.flask)
    except Exception, e:
        Log.error("Serious problem with MoDataSubmission service!  Shutdown completed!", cause=e)

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def start():
    global hg
    global config
    _ = wrap

    try:
        config = startup.read_settings()
        constants.set(config.constants)
        Log.start(config.debug)
        if config.hg:
            hg = HgMozillaOrg(config.hg)
        main()
    except Exception, e:
        Log.error("Problems exist", e)

# 需要导入模块: from pyLibrary.debugs.logs import Log [as 别名]
# 或者: from pyLibrary.debugs.logs.Log import start [as 别名]
def main():
    try:
        settings = startup.read_settings(defs={
            "name": ["--num"],
            "help": "number to show",
            "type": int,
            "dest": "num",
            "default": '10',
            "required": False
        })
        Log.start(settings.debug)
        list_queue(settings.source, settings.args.num)
    except Exception, e:
        Log.error("Problem with etl", e)