Python logging.error函数代码示例

def get_data(url, hash_result=True):
    """
        Extract data from the global hash given a request object.  If an item
        is successfully recovered data is returned
    """

    request_obj = build_request_obj(url)
    hash_table_ref = read_pickle_data()

    # Traverse the hash key structure to find data
    # @TODO rather than iterate through REQUEST_META_BASE &
    #   REQUEST_META_QUERY_STR look only at existing attributes

    logging.debug(__name__ + " - Attempting to pull data for request " \
                             "COHORT {0}, METRIC {1}".
                  format(request_obj.cohort_expr, request_obj.metric))

    key_sig = build_key_signature(request_obj, hash_result=hash_result)
    item = find_item(hash_table_ref, key_sig)

    if item:
        # item[0] will be a stringified structure that
        # is initialized, see set_data.
        try:
            return eval(item[0])
        except SyntaxError:
            logging.error(__name__ + ' :: Failed to retrieve {0}'.
                          format(key_sig))
            return None
    else:
        return None

 def set_password(self, password):
     try:
         password = escape(unicode(password))
         self.pw_hash = generate_password_hash(password)
     except (TypeError, NameError) as e:
         logging.error(__name__ + ' :: Hash set error - ' + e.message)
         self.pw_hash = None

def build_key_signature(request_meta, hash_result=False):
    """
        Given a RequestMeta object contruct a hashkey.

        Parameters
        ~~~~~~~~~~

            request_meta : RequestMeta
                Stores request data.
    """
    key_sig = list()

    # Build the key signature -- These keys must exist
    for key_name in REQUEST_META_BASE:
        key = getattr(request_meta, key_name)
        if key:
            key_sig.append(key_name + HASH_KEY_DELIMETER + key)
        else:
            logging.error(__name__ + ' :: Request must include %s. '
                                     'Cannot set data %s.' %
                                     (key_name, str(request_meta)))
            return ''
    # These keys may optionally exist
    for key_name in REQUEST_META_QUERY_STR:
        if hasattr(request_meta, key_name):
            key = getattr(request_meta, key_name)
            if key:
                key_sig.append(key_name + HASH_KEY_DELIMETER + str(key))

    if hash_result:
        return sha1(str(key_sig).encode('utf-8')).hexdigest()
    else:
        return key_sig

def format_namespace(namespace, col='page_namespace'):
    """ Format the namespace condition in queries and returns the string.

        Expects a list of numeric namespace keys.  Otherwise returns
        an empty condition string.

        ** THIS METHOD ONLY EMITS SQL SAFE STRINGS **
    """
    ns_cond = ''

    # Copy so as not to affect mutable ref
    namespace = deepcopy(namespace)

    if hasattr(namespace, '__iter__'):
        if len(namespace) == 1:
            ns_cond = '{0} = '.format(col) \
                + escape_var(str(namespace.pop()))
        else:
            ns_cond = '{0} in ('.format(col) + \
                ",".join(DataLoader()
                .cast_elems_to_string(escape_var(list(namespace)))) + ')'
    else:
        try:
            ns_cond = '{0} = '.format(col) + escape_var(int(namespace))
        except ValueError:
            # No namespace condition
            logging.error(__name__ + ' :: Could not apply namespace '
                                     'condition on {0}'.format(str(namespace)))
            pass

    return ns_cond

def _get_revisions(args):
    """ Retrieve total set of revision records for users within timeframe """
    um.log_pool_worker_start(__name__, _get_revisions.__name__, args[0], args[1])

    users = args[0]
    state = args[1]

    metric_params = um.UserMetric._unpack_params(state)
    query_args_type = namedtuple('QueryArgs', 'date_start date_end namespace')

    revs = list()
    umpd_obj = UMP_MAP[metric_params.group](users, metric_params)
    try:
        for t in umpd_obj:
            revs += \
                list(query_mod.rev_query(t.user, metric_params.project,
                                         query_args_type(t.start, t.end,
                                                         metric_params.namespace)))
    except query_mod.UMQueryCallError as e:
        logging.error('{0}:: {1}. PID={2}'.format(__name__,
                                                  e.message, os.getpid()))
        return []

    um.log_pool_worker_end(__name__, _process_help.__name__)
    return revs

def get_cohort_refresh_datetime(utm_id):
    """
        Get the latest refresh datetime of a cohort.  Returns current time
        formatted as a string if the field is not found.
    """

    # @TODO MOVE DB REFS INTO QUERY MODULE
    conn = dl.Connector(instance=settings.__cohort_data_instance__)
    query = """ SELECT utm_touched FROM usertags_meta WHERE utm_id = %s """
    conn._cur_.execute(query, int(utm_id))

    utm_touched = None
    try:
        utm_touched = conn._cur_.fetchone()[0]
    except ValueError:
        pass

    # Ensure the field was retrieved
    if not utm_touched:
        logging.error(__name__ + '::Missing utm_touched for cohort %s.' %
                                 str(utm_id))
        utm_touched = datetime.now()

    del conn
    return utm_touched.strftime(DATETIME_STR_FORMAT)

def _process_help(args):
    """ Used by Threshold::process() for forking.
        Should not be called externally. """

    state = args[1]
    thread_args = RevertRateArgsClass(state[0], state[1], state[2],
                                      state[3], state[4], state[6],
                                      state[7], state[8])
    users = args[0]

    if thread_args.log_progress:
        logging.info(__name__ +
                    ' :: Computing reverts on %s users (PID %s)'
                    % (len(users), str(os.getpid())))
    results_agg = list()
    dropped_users = 0

    umpd_obj = UMP_MAP[thread_args.group](users, thread_args)
    for user_data in umpd_obj:

        total_revisions = 0.0
        total_reverts = 0.0

        # Call query on revert rate for each user
        #
        # 1. Obtain user registration date
        # 2. Compute end date based on 't'
        # 3. Get user revisions in time period
        query_args = namedtuple('QueryArgs', 'date_start date_end')\
            (format_mediawiki_timestamp(user_data.start),
             format_mediawiki_timestamp(user_data.end))

        try:
            revisions = query_mod.\
                revert_rate_user_revs_query(user_data.user,
                                            thread_args.project,
                                            query_args)
        except query_mod.UMQueryCallError as e:
            logging.error(__name__ + ' :: Failed to '
                                     'get revisions: {0}'.format(e.message))
            dropped_users += 1
            continue

        results_thread = mpw.build_thread_pool(revisions, _revision_proc,
                                               thread_args.rev_threads, state)

        for r in results_thread:
            total_revisions += r[0]
            total_reverts += r[1]
        if not total_revisions:
            results_agg.append([user_data.user, 0.0, total_revisions])
        else:
            results_agg.append([user_data.user, total_reverts / total_revisions,
                                total_revisions])

    if thread_args.log_progress:
        logging.debug(__name__ + ' :: PID {0} complete. Dropped users = {1}'.
            format(str(os.getpid()), dropped_users))

    return results_agg

def all_urls():
    """ View for listing all requests.  Retrieves from cache """

    # @TODO - this reads the entire cache into memory, filters will be needed
    # This extracts ALL data from the cache, the data is assumed to be in the
    # form of <hash key -> (data, key signature)> pairs.  The key signature is
    # extracted to reconstruct the url.

    all_data = read_pickle_data()
    key_sigs = list()

    for key, val in all_data.iteritems():
        if hasattr(val, '__iter__'):
            try:
                key_sigs.append(val[1])
            except (KeyError, IndexError):
                logging.error(__name__ + ' :: Could not render key signature '
                                         'from data, key = {0}'.format(key))

    # Compose urls from key sigs
    url_list = list()
    for key_sig in key_sigs:

        url = get_url_from_keys(key_sig, 'cohorts')
        url_list.append("".join(['<a href="',
                                 request.url_root, url + '">',
                                 url,
                                 '</a>']))
    return render_template('all_urls.html', urls=url_list)

    def __init__(self, **kwargs):
        super(TimeToThreshold, self).__init__(**kwargs)

        try:
            self._threshold_obj_ = self.__threshold_types[self.threshold_type_class](**kwargs)
        except NameError:
            logging.error(__name__ + "::Invalid threshold class. " "Using default (EditCountThreshold).")
            self._threshold_obj_ = self.EditCountThreshold(**kwargs)

def teardown():
    """ When the instance is deleted store the pickled data and shutdown
        the job controller """

    # Shutdown API handlers gracefully
    try:
        terminate_process_with_checks(job_controller_proc)
        terminate_process_with_checks(response_controller_proc)
    except Exception:
        logging.error(__name__ + ' :: Could not shut down callbacks.')

def req_cb_get_is_running(key, lock):
    lock.acquire()
    req_notification_queue_in.put([2, key], True)
    try:
        val = req_notification_queue_out.get(block=True, timeout=BLOCK_TIMEOUT)[0]
    except Empty:
        logging.error(__name__ + " :: req_cb_get_is_running -" " Block time expired.")
        return False
    lock.release()
    return val

def req_cb_get_cache_keys(lock):
    lock.acquire()
    req_notification_queue_in.put([3], block=True)
    try:
        val = req_notification_queue_out.get(block=True, timeout=BLOCK_TIMEOUT)
    except Empty:
        logging.error(__name__ + " :: req_cb_get_cache_keys -" " Block time expired.")
        return []
    lock.release()
    return val

def process_responses(response_queue, msg_in):
    """ Pulls responses off of the queue. """

    log_name = '{0} :: {1}'.format(__name__, process_responses.__name__)
    logging.debug(log_name  + ' - STARTING...')

    while 1:
        stream = ''

        # Block on the response queue
        try:
            res = response_queue.get(True)
            request_meta = rebuild_unpacked_request(res)
        except Exception:
            logging.error(log_name + ' - Could not get request meta')
            continue

        data = response_queue.get(True)
        while data:
            stream += data
            try:
                data = response_queue.get(True, timeout=1)
            except Empty:
                break

        try:
            data = eval(stream)
        except Exception as e:

            # Report a fraction of the failed response data directly in the
            # logger
            if len(unicode(stream)) > 2000:
                excerpt = stream[:1000] + ' ... ' + stream[-1000:]
            else:
                excerpt = stream

            logging.error(log_name + ' - Request failed. {0}\n\n' \
                                     'data excerpt: {1}'.format(e.message, excerpt))

            # Format a response that will report on the failed request
            stream = "OrderedDict([('status', 'Request failed.'), " \
                     "('exception', '" + escape(unicode(e.message)) + "')," \
                     "('request', '" + escape(unicode(request_meta)) + "'), " \
                     "('data', '" + escape(unicode(stream)) + "')])"

        key_sig = build_key_signature(request_meta, hash_result=True)

        # Set request in list to "not alive"
        req_cb_flag_job_complete(key_sig, REQ_NCB_LOCK)

        logging.debug(log_name + ' - Setting data for {0}'.format(
            str(request_meta)))
        set_data(stream, request_meta)

    logging.debug(log_name + ' - SHUTTING DOWN...')

def req_cb_get_url(key, lock):
    lock.acquire()
    req_notification_queue_in.put([4, key], block=True)
    try:
        val = req_notification_queue_out.get(True, timeout=BLOCK_TIMEOUT)[0]
    except Empty:
        logging.error(__name__ + ' :: req_cb_get_url -'
                                 ' Block time expired.')
        return ''
    lock.release()
    return val

def __future(rev_id, page_id, n, project, namespace):
    """ Produce the n revisions on a page after a given revision
            Returns a generator of revision objects """
    try:
        future = query_mod.page_rev_hist_query(rev_id, page_id, n, project,
                                                namespace, look_ahead=True)
    except query_mod.UMQueryCallError as e:
        logging.error(__name__ + ' :: Failed to '
                                 'get revision future: {0}'.format(e.message))
        future = list()
    return future