Python exceptions.SoftTimeLimitExceeded方法代码示例

# 需要导入模块: from celery import exceptions [as 别名]
# 或者: from celery.exceptions import SoftTimeLimitExceeded [as 别名]
def execute_command(job):
    output = ""
    # Disable duplicate celery log messages
    if logger.propagate:
        logger.propagate = False
    logger.info("Running cmd for " + str(job))
    try:
        cmd_result = subprocess.run(
            job['command'],
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT
        )
        output = cmd_result.stdout.decode("utf-8")
        job['errored_out'] = False
    except SoftTimeLimitExceeded:
        job['errored_out'] = True
    job['output'] = output
    return job 

# 需要导入模块: from celery import exceptions [as 别名]
# 或者: from celery.exceptions import SoftTimeLimitExceeded [as 别名]
def crawl_comment_by_page(mid, page_num):
    try:
        cur_url = BASE_URL.format(mid, page_num)
        html = get_page(cur_url, auth_level=1, is_ajax=True)
        comment_datas = comment.get_comment_list(html, mid)
    except SoftTimeLimitExceeded:
        crawler.error(
            "comment SoftTimeLimitExceeded    mid={mid} page_num={page_num}".
            format(mid=mid, page_num=page_num))
        app.send_task(
            'tasks.comment.crawl_comment_by_page',
            args=(mid, page_num),
            queue='comment_page_crawler',
            routing_key='comment_page_info')
    CommentOper.add_all(comment_datas)
    if page_num == 1:
        WbDataOper.set_weibo_comment_crawled(mid)
    return html, comment_datas 

# 需要导入模块: from celery import exceptions [as 别名]
# 或者: from celery.exceptions import SoftTimeLimitExceeded [as 别名]
def __get(session, relative_path='/', headers=None, cookies=None):
    if not headers:
        headers = {}

    if not cookies:
        cookies = {}

    try:
        # TODO: limit the maximum size of the response, to keep malicious site operators from killing us
        # TODO: Perhaps we can naively do it for now by simply setting a timeout?
        # TODO: catch TLS errors instead of just setting it to None?
        return session.get(session.url.scheme + '://' + session.url.netloc + relative_path,
                           headers=headers,
                           cookies=cookies,
                           timeout=TIMEOUT)
    # Let celery exceptions percolate upward
    except (SoftTimeLimitExceeded, TimeLimitExceeded):
        raise
    except (KeyboardInterrupt, SystemExit):
        raise
    except:
        return None 

# 需要导入模块: from celery import exceptions [as 别名]
# 或者: from celery.exceptions import SoftTimeLimitExceeded [as 别名]
def start(**kwargs):
    """
    添加扫描任务
    :param kwargs: 任务配置
    :return:
    """
    pro = None
    try:
        init()
        result_file = '{0}.json'.format(kwargs.get("project_name"))
        kwargs['result_file'] = result_file
        pro = start_scan(**kwargs)
    except SoftTimeLimitExceeded as ex:
        logger.critical(ex)
        if pro:
            pro.update_scan_failed_status(ex)
    except Exception as ex:
        import traceback;traceback.print_exc()
        logger.critical(ex)
    except KeyboardInterrupt:
        logger.error("user aborted")
    except SystemExit:
        raise

    return False 

# 需要导入模块: from celery import exceptions [as 别名]
# 或者: from celery.exceptions import SoftTimeLimitExceeded [as 别名]
def test_timed_out(self):
        import subprocess
        subprocess.run = mock.Mock(side_effect=SoftTimeLimitExceeded)

        job = Job(environment_id="12345", command="echo 'HELLO'")
        task = execute_command.run(job)
        assert task['errored_out'] is True 

# 需要导入模块: from celery import exceptions [as 别名]
# 或者: from celery.exceptions import SoftTimeLimitExceeded [as 别名]
def crawl_person_infos(uid):
    """
    Crawl user info and their fans and followers
    For the limit of weibo's backend, we can only crawl 5 pages of the fans and followers.
    We also have no permissions to view enterprise's followers and fans info
    :param uid: current user id
    :return: None
    """
    if not uid:
        return

    try:
        user, is_crawled = get_profile(uid)
        # If it's enterprise user, just skip it
        if user and user.verify_type == 2:
            SeedidsOper.set_seed_other_crawled(uid)
            return

        # Crawl fans and followers
        if not is_crawled:
            app.send_task('tasks.user.crawl_follower_fans', args=(uid,), queue='fans_followers',
                          routing_key='for_fans_followers')

    # By adding '--soft-time-limit secs' when you start celery, this will resend task to broker
    # e.g. celery -A tasks.workers -Q user_crawler worker -l info -c 1 --soft-time-limit 10
    except SoftTimeLimitExceeded:
        crawler.error("user SoftTimeLimitExceeded    uid={uid}".format(uid=uid))
        app.send_task('tasks.user.crawl_person_infos', args=(uid, ), queue='user_crawler',
                      routing_key='for_user_info') 

# 需要导入模块: from celery import exceptions [as 别名]
# 或者: from celery.exceptions import SoftTimeLimitExceeded [as 别名]
def clean_source(source):
    """
    This celery task will clean the specified source. This is a destructive operation that will delete unused
    certificates from each source.

    :param source:
    :return:
    """
    function = f"{__name__}.{sys._getframe().f_code.co_name}"
    task_id = None
    if celery.current_task:
        task_id = celery.current_task.request.id

    log_data = {
        "function": function,
        "message": "Cleaning source",
        "source": source,
        "task_id": task_id,
    }

    if task_id and is_task_active(function, task_id, (source,)):
        log_data["message"] = "Skipping task: Task is already active"
        current_app.logger.debug(log_data)
        return

    current_app.logger.debug(log_data)
    try:
        clean([source], True)
    except SoftTimeLimitExceeded:
        log_data["message"] = "Clean source: Time limit exceeded."
        current_app.logger.error(log_data)
        sentry.captureException()
        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
    return log_data 

# 需要导入模块: from celery import exceptions [as 别名]
# 或者: from celery.exceptions import SoftTimeLimitExceeded [as 别名]
def certificate_reissue():
    """
    This celery task reissues certificates which are pending reissue
    :return:
    """
    function = f"{__name__}.{sys._getframe().f_code.co_name}"
    task_id = None
    if celery.current_task:
        task_id = celery.current_task.request.id

    log_data = {
        "function": function,
        "message": "reissuing certificates",
        "task_id": task_id,
    }

    if task_id and is_task_active(function, task_id, None):
        log_data["message"] = "Skipping task: Task is already active"
        current_app.logger.debug(log_data)
        return

    current_app.logger.debug(log_data)
    try:
        cli_certificate.reissue(None, True)
    except SoftTimeLimitExceeded:
        log_data["message"] = "Certificate reissue: Time limit exceeded."
        current_app.logger.error(log_data)
        sentry.captureException()
        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
        return

    log_data["message"] = "reissuance completed"
    current_app.logger.debug(log_data)
    metrics.send(f"{function}.success", "counter", 1)
    return log_data 

# 需要导入模块: from celery import exceptions [as 别名]
# 或者: from celery.exceptions import SoftTimeLimitExceeded [as 别名]
def certificate_rotate(**kwargs):

    """
    This celery task rotates certificates which are reissued but having endpoints attached to the replaced cert
    :return:
    """
    function = f"{__name__}.{sys._getframe().f_code.co_name}"
    task_id = None
    if celery.current_task:
        task_id = celery.current_task.request.id

    region = kwargs.get("region")
    log_data = {
        "function": function,
        "message": "rotating certificates",
        "task_id": task_id,
    }

    if task_id and is_task_active(function, task_id, None):
        log_data["message"] = "Skipping task: Task is already active"
        current_app.logger.debug(log_data)
        return

    current_app.logger.debug(log_data)
    try:
        if region:
            log_data["region"] = region
            cli_certificate.rotate_region(None, None, None, None, True, region)
        else:
            cli_certificate.rotate(None, None, None, None, True)
    except SoftTimeLimitExceeded:
        log_data["message"] = "Certificate rotate: Time limit exceeded."
        current_app.logger.error(log_data)
        sentry.captureException()
        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
        return

    log_data["message"] = "rotation completed"
    current_app.logger.debug(log_data)
    metrics.send(f"{function}.success", "counter", 1)
    return log_data 

# 需要导入模块: from celery import exceptions [as 别名]
# 或者: from celery.exceptions import SoftTimeLimitExceeded [as 别名]
def endpoints_expire():
    """
    This celery task removes all endpoints that have not been recently updated
    :return:
    """
    function = f"{__name__}.{sys._getframe().f_code.co_name}"
    task_id = None
    if celery.current_task:
        task_id = celery.current_task.request.id

    log_data = {
        "function": function,
        "message": "endpoints expire",
        "task_id": task_id,
    }

    if task_id and is_task_active(function, task_id, None):
        log_data["message"] = "Skipping task: Task is already active"
        current_app.logger.debug(log_data)
        return

    current_app.logger.debug(log_data)
    try:
        cli_endpoints.expire(2)  # Time in hours
    except SoftTimeLimitExceeded:
        log_data["message"] = "endpoint expire: Time limit exceeded."
        current_app.logger.error(log_data)
        sentry.captureException()
        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
        return

    metrics.send(f"{function}.success", "counter", 1)
    return log_data 

# 需要导入模块: from celery import exceptions [as 别名]
# 或者: from celery.exceptions import SoftTimeLimitExceeded [as 别名]
def check_revoked():
    """
    This celery task attempts to check if any certs are expired
    :return:
    """
    function = f"{__name__}.{sys._getframe().f_code.co_name}"
    task_id = None
    if celery.current_task:
        task_id = celery.current_task.request.id

    log_data = {
        "function": function,
        "message": "check if any certificates are revoked revoked",
        "task_id": task_id,
    }

    if task_id and is_task_active(function, task_id, None):
        log_data["message"] = "Skipping task: Task is already active"
        current_app.logger.debug(log_data)
        return

    current_app.logger.debug(log_data)
    try:
        cli_certificate.check_revoked()
    except SoftTimeLimitExceeded:
        log_data["message"] = "Checking revoked: Time limit exceeded."
        current_app.logger.error(log_data)
        sentry.captureException()
        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
        return

    metrics.send(f"{function}.success", "counter", 1)
    return log_data 

# 需要导入模块: from celery import exceptions [as 别名]
# 或者: from celery.exceptions import SoftTimeLimitExceeded [as 别名]
def notify_expirations():
    """
    This celery task notifies about expiring certs
    :return:
    """
    function = f"{__name__}.{sys._getframe().f_code.co_name}"
    task_id = None
    if celery.current_task:
        task_id = celery.current_task.request.id

    log_data = {
        "function": function,
        "message": "notify for cert expiration",
        "task_id": task_id,
    }

    if task_id and is_task_active(function, task_id, None):
        log_data["message"] = "Skipping task: Task is already active"
        current_app.logger.debug(log_data)
        return

    current_app.logger.debug(log_data)
    try:
        cli_notification.expirations(
            current_app.config.get("EXCLUDE_CN_FROM_NOTIFICATION", [])
        )
    except SoftTimeLimitExceeded:
        log_data["message"] = "Notify expiring Time limit exceeded."
        current_app.logger.error(log_data)
        sentry.captureException()
        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
        return

    metrics.send(f"{function}.success", "counter", 1)
    return log_data 

# 需要导入模块: from celery import exceptions [as 别名]
# 或者: from celery.exceptions import SoftTimeLimitExceeded [as 别名]
def sync_source(source):
    """
    This celery task will sync the specified source.

    :param source:
    :return:
    """

    function = f"{__name__}.{sys._getframe().f_code.co_name}"
    task_id = None
    if celery.current_task:
        task_id = celery.current_task.request.id

    log_data = {
        "function": function,
        "message": "Syncing source",
        "source": source,
        "task_id": task_id,
    }

    if task_id and is_task_active(function, task_id, (source,)):
        log_data["message"] = "Skipping task: Task is already active"
        current_app.logger.debug(log_data)
        return

    current_app.logger.debug(log_data)
    try:
        sync([source])
        metrics.send(
            f"{function}.success", "counter", 1, metric_tags={"source": source}
        )
    except SoftTimeLimitExceeded:
        log_data["message"] = "Error syncing source: Time limit exceeded."
        current_app.logger.error(log_data)
        sentry.captureException()
        metrics.send(
            "sync_source_timeout", "counter", 1, metric_tags={"source": source}
        )
        metrics.send("celery.timeout", "counter", 1, metric_tags={"function": function})
        return

    log_data["message"] = "Done syncing source"
    current_app.logger.debug(log_data)
    metrics.send(f"{function}.success", "counter", 1, metric_tags={"source": source})
    return log_data 

# 需要导入模块: from celery import exceptions [as 别名]
# 或者: from celery.exceptions import SoftTimeLimitExceeded [as 别名]
def __create_session(url: str, **kwargs) -> dict:
    s = requests.Session()

    # Allow certificate verification to be disabled on the initial request, which means that sites won't get
    # penalized on things like HSTS, even for self-signed certificates
    s.verify = kwargs['verify']

    # Add the headers to the session
    if kwargs['headers']:
        s.headers.update(kwargs['headers'])

    # Set all the cookies and force them to be sent only over HTTPS; this might change in the future
    if kwargs['cookies']:
        s.cookies.update(kwargs['cookies'])

        for cookie in s.cookies:
            cookie.secure = True

    # Override the User-Agent; some sites (like twitter) don't send the CSP header unless you have a modern
    # user agent
    s.headers.update({
        'User-Agent': RETRIEVER_USER_AGENT,
    })

    try:
        r = s.get(url, timeout=TIMEOUT)

        # No tls errors
        r.verified = True
    # Let celery exceptions percolate upward
    except (SoftTimeLimitExceeded, TimeLimitExceeded):
        raise
    # We can try again if there's an SSL error, making sure to note it in the session
    except requests.exceptions.SSLError:
        try:
            r = s.get(url, timeout=TIMEOUT, verify=False)
            r.verified = False
        except (KeyboardInterrupt, SystemExit):
            raise
        except:
            r = None
            s = None
    except (KeyboardInterrupt, SystemExit):
        raise
    except:
        r = None
        s = None

    # Store the domain name and scheme in the session
    if r is not None and s is not None:
        s.url = urlparse(r.url)

    return {'session': s, 'response': r}