Python logger.Logger类代码示例

class BatchCrawler():
    
    MAX_DOCS_NUM = 100
    
    def __init__(self, database_config_path, source_name, domain, encode, request_interval):
        self.logger = Logger("crawler", domain)
        self.adapter = DocRawAdapter(database_config_path, source_name, self.logger)
        self.domain = domain
        self.encode = encode 
        self.request_interval = request_interval
    
    def run(self):
        while True:
            count = 0
            try:
                for url_hash, url in self.adapter.load_uncrawled_docs(BatchCrawler.MAX_DOCS_NUM):
                    count += 1
                    self.logger.log("crawling url %s"%url, 2)
                    page = common_utils.page_crawl(url)
                    if page == None:
                        self.adapter.update_doc_raw_as_crawled_failed(url_hash)
                        continue
                    if self.encode != "utf-8":
                        page = unicode(page, self.encode).encode("utf-8")

                    self.adapter.update_doc_raw_with_crawled_page(url_hash, "utf-8", page)
                    time.sleep(float(self.request_interval))
                if count < BatchCrawler.MAX_DOCS_NUM:
                    break
            except:
                self.logger.log("mongo error")

class DataQueue:
    def __init__(self):
        self.dataQueue = queue.Queue()
        self.dv = datavalidator.DataValidator()
        self.log = Logger().get('database.dataqueue.DataQueue')
    """we want to check the data here and fail early
        if the data is good then we want to put it in the data queue
        we will want another python script for the validations (datavalidator.py)
        we need to enforce type constraints because the database will not
        see datavalidator.py"""
    def insert_into_data_queue(self, value):
        if not self.dv.run_all_checks(value):
            self.log.error('--> Validation failed! Unable to add data '
                           'into data queue: ' +
                           str(value))
            return False
        try:
            self.dataQueue.put(value)
        except queue.Full as e:
            self.log.critical('Data queue is full!')
        finally:
            return True

    def get_next_item(self):
        item = self.dataQueue.get()
        self.dataQueue.task_done()
        return item

    def check_empty(self):
        result = self.dataQueue.empty()
        return result

def validate_time_period(query_tokens):
    log = Logger().get('reportserver.manager.utilities')
    log.debug("given query_tokens:" + str(query_tokens))

    uom = None
    units = None

    for token in query_tokens:
        if '=' in token:
            uom,units = token.split('=')
            if uom in UnitOfMeasure.get_values(UnitOfMeasure):
                units = int(units)
                break
            else:
                uom  = None
                units = None

    # default if we aren't given valid uom and units
    #TODO:  get this from a config file.
    if uom is None or units is None:
        uom = "days"
        units = 1

    log.debug("validate_time_period: " + str(uom) + ": " + str(units))
    return (uom, units)

class Handler(object):

    def __init__(self,section,filename=None,data_results=None):
        self.logger = Logger()
        self.key=ConfigurationManager.readCuckooResultsConfig(variable='key',section=section)
        self.encapsulation = literal_eval(ConfigurationManager.readCuckooResultsConfig(variable='encapsulation',section=section))
        self.keys = list(ConfigurationManager.readCuckooResultsConfig(variable='keys',section=section).split(','))
        #Check if there are not any keys
        if self.keys==['']:
            self.keys=None
        self.subsectionskeys={}
        if self.encapsulation:
            self.subsections = returnsubsections(self.encapsulation,section=section,subsections=[])
            for subsection in self.subsections:
                self.subsectionskeys[ConfigurationManager.readCuckooResultsConfig(variable='key',section='subsection_'+subsection)] = list(ConfigurationManager.readCuckooResultsConfig(variable='keys',section='subsection_'+subsection).split(','))
        results=None
        try:
            if data_results is not None:
                results=data_results[self.key]
            elif filename is not  None:
                results = load_results(filename)[self.key]
        except Exception, e:
            self.logger.errorLogging(str(e))

        if isinstance(results,dict):
            self.dictionary = results
            self.list = None
        elif isinstance(results,list):
            self.list= results
            self.dictionary = None
        else:
            self.list = None
            self.dictionary = None

def MatchFiles(checkerFile, c1File, targetArch, debuggableMode):
    for testCase in checkerFile.testCases:
        if testCase.testArch not in [None, targetArch]:
            continue
        if testCase.forDebuggable != debuggableMode:
            continue

        # TODO: Currently does not handle multiple occurrences of the same group
        # name, e.g. when a pass is run multiple times. It will always try to
        # match a check group against the first output group of the same name.
        c1Pass = c1File.findPass(testCase.name)
        if c1Pass is None:
            Logger.fail(
                'Test case "{}" not found in the CFG file'.format(testCase.name),
                testCase.fileName,
                testCase.startLineNo,
            )

        Logger.startTest(testCase.name)
        try:
            MatchTestCase(testCase, c1Pass)
            Logger.testPassed()
        except MatchFailedException as e:
            lineNo = c1Pass.startLineNo + e.lineNo
            if e.assertion.variant == TestAssertion.Variant.Not:
                Logger.testFailed(
                    "NOT assertion matched line {}".format(lineNo), e.assertion.fileName, e.assertion.lineNo
                )
            else:
                Logger.testFailed(
                    "Assertion could not be matched starting from line {}".format(lineNo),
                    e.assertion.fileName,
                    e.assertion.lineNo,
                )

 def process_do_maintenance(self):
     """ メンテナンスの設定を行う
     """
     if self.operation.maintenance_mode:
         self.operation.render_maintenance_config()
         self.operation.cp_maintenance_config()
         self.operation.nginx.restart()
         Logger.put('メンテナンスを{}にしたよ'.format(self.operation.maintenance_mode))

 def render_vhost_conf(self):
     template = self.env.get_template('vhost.conf.j2')
     for virtual_host in self.virtual_hosts:
         nginx_config = template.render(virtual_host=virtual_host)
         virtual_host_conf = os.path.join(self.nginx.files_dir, virtual_host.conf)
         with open(virtual_host_conf, 'w+') as dest:
             Logger.put("{}を生成したよー".format(virtual_host_conf))
             dest.write(nginx_config.encode('utf-8'))

 def addAssertion(self, new_assertion):
   if new_assertion.variant == TestAssertion.Variant.NextLine:
     if not self.assertions or \
        (self.assertions[-1].variant != TestAssertion.Variant.InOrder and \
         self.assertions[-1].variant != TestAssertion.Variant.NextLine):
       Logger.fail("A next-line assertion can only be placed after an "
                   "in-order assertion or another next-line assertion.",
                   new_assertion.fileName, new_assertion.lineNo)
   self.assertions.append(new_assertion)

 def parse_json(self):
     init_data = {}
     if not isinstance(self.item['data'], str):
         return False
     try:
         init_data = json.loads(self.item['data'])
     except:
         Logger.put("JSONにできないやつ送られてきた!. {}".format(self.item['data']))
         return False
     self.operation = Operation(init_data)
     return True

 def cp(self, _from, _to):
     """ cp _from _to
     @param _from str file path
     @param _to str file path
     @return Popen#returncode, res, err
     """
     cp = 'cp -rf {} {}'.format(_from, _to),
     # DEBUG
     Logger.put(cp)
     ret_code, res, err = shell_command(cp)
     return ret_code, res, err

def get_date_delta(iso_date_from, iso_date_to):
    try:
        date_from = dateutil.parser.parse(iso_date_from)
        date_to = dateutil.parser.parse(iso_date_to)
        delta = date_to - date_from
    except Exception as e:
        log = Logger().get('reportserver.manager.utilities')
        log.error("Error: " + str(e))
        delta = 0

    return str(delta)

def DumpPass(outputFilename, passName):
  c1File = ParseC1visualizerStream(os.path.basename(outputFilename), open(outputFilename, "r"))
  compiler_pass = c1File.findPass(passName)
  if compiler_pass:
    maxLineNo = compiler_pass.startLineNo + len(compiler_pass.body)
    lenLineNo = len(str(maxLineNo)) + 2
    curLineNo = compiler_pass.startLineNo
    for line in compiler_pass.body:
      Logger.log((str(curLineNo) + ":").ljust(lenLineNo) + line)
      curLineNo += 1
  else:
    Logger.fail("Pass \"" + passName + "\" not found in the output")

    def __init__(self, parent, name, body, startLineNo):
        self.parent = parent
        self.name = name
        self.body = body
        self.startLineNo = startLineNo

        if not self.name:
            Logger.fail("C1visualizer pass does not have a name", self.fileName, self.startLineNo)
        if not self.body:
            Logger.fail("C1visualizer pass does not have a body", self.fileName, self.startLineNo)

        self.parent.addPass(self)

def update_analyzers_pool():
    logger = Logger()
    try:
        global analyzers_pool
        analyzers = open("analyzers", "r")
        fcntl.fcntl(analyzers, fcntl.LOCK_EX)
        data = analyzers.read()
        fcntl.fcntl(analyzers, fcntl.LOCK_UN)
        analyzers.close()
        analyzers_pool = literal_eval(data)
    except Exception, e:
        info = str(e)
        logger.errorLogging(info)

  def __init__(self, parent, name, startLineNo, testArch = None):
    assert isinstance(parent, CheckerFile)

    self.parent = parent
    self.name = name
    self.assertions = []
    self.startLineNo = startLineNo
    self.testArch = testArch

    if not self.name:
      Logger.fail("Test case does not have a name", self.fileName, self.startLineNo)

    self.parent.addTestCase(self)