Python Logger.info方法代码示例

# 需要导入模块: from Utils import Logger [as 别名]
# 或者: from Utils.Logger import info [as 别名]
def diff_task(linkdb, output, pagelist):
    page_filenames = []
    #trick :)
    if os.path.exists(pagelist):
        Logger.info('Use pagelist indead of page files!')
        with open(pagelist, 'r') as f:
            for line in f:
                line = line.strip()
                if not line:
                    break
                filename = line
                page_filenames.append(filename)
    else:
        Logger.info('CANNOT find pagelist file: %s' % pagelist)

    tasks = {}
    sources = []
    with open(linkdb, 'r') as f:
        for line in f:
            line = line.strip()
            if line:
                try:
                    hashurl, url, reply_time, source = line.split('\t')[:4]
                except: continue
                filename = '%s' % (hashurl)
                tasks.update({
                    filename:{
                        'url':url,
                        'source':source
                    }
                })
                sources.append(source)

    if not pagelist:
        for source in set(sources):
            source = os.path.join(output, source)
            if os.path.exists(source):
                filenames = os.listdir(source)
                for filename in filenames:
                    tasks.pop(filename) 
            else:
                os.mkdir(source)
    else:
        for source in set(sources):
            source = os.path.join(output, source)
            if not os.path.exists(source):
                os.mkdir(source)
        for filename in page_filenames:
            try:
                tasks.pop(filename) 
            except:
                print 'Skip', filename
    return tasks

# 需要导入模块: from Utils import Logger [as 别名]
# 或者: from Utils.Logger import info [as 别名]
 def open(self, url, delay=0.1):
     response = None
     try:
         response = self.br.open(url, timeout=20.0)
     except urllib2.HTTPError, e:
         while e.code != 404:
             interval = Interval.val()
             time.sleep(interval)
             Logger.info('sleep %ds error %d %s' % (interval, e.code, url))
             try:
                 response = self.br.open(url, timeout=20.0)
                 Logger.info('skip 403 ' + url)
                 break
             except urllib2.HTTPError, e:
                 if e.code != 404:
                     continue
             except:

# 需要导入模块: from Utils import Logger [as 别名]
# 或者: from Utils.Logger import info [as 别名]
 def parse(self, **args):
     hashurl = args['hashurl']
     title = args['title']
     text = args['text']
     ret = TextParser.parse(title+' '+text)
     if ret.has_key('error'):
         Logger.info(hashurl+' '+ret['error'])
         return
     record = '\t'.join([
         hashurl,
         title,
         text,
         ret['jushi'],
         ret['shouji'],
         ret['zujin'],
         ret['dizhi'],
         ret['ditie'],
     ])
     self._db.insert(record)

# 需要导入模块: from Utils import Logger [as 别名]
# 或者: from Utils.Logger import info [as 别名]
 def parse(self, **args):
     page = args['page']
     source = args['source']
     hashurl = args['hashurl']
     ret = PageParser.parse(page, source)
     if ret.has_key('error'):
         Logger.info(hashurl+' '+ret['error'])
         return
     record = '\t'.join([
         hashurl,
         ret['title2'] if ret['title2']\
             else ret['title'],
         json.dumps(ret['author']),
         json.dumps(ret['images']),
         json.dumps(ret['links']),
         ret['text'],
         ret['pub_time'],
     ]).encode('utf-8')
     self._db.insert(record)

# 需要导入模块: from Utils import Logger [as 别名]
# 或者: from Utils.Logger import info [as 别名]
 def crawl(self, **args):
     source = args['source']
     ext = args['ext']
     reply_time = args['reply_time']
     br = Browser()
     page = br.open(self.baseurl)
     new_reply_time = reply_time
     while True:
         links = PageParser.parse(page, source)
         for i, link in enumerate(links):
             if reply_time < link.reply_time:
                 if i is 0:
                     new_reply_time = link.reply_time
                 self.db.insert('\t'.join([str(link), source, json.dumps(ext)]))
             else:
                 return new_reply_time
         try:
             page = br.follow_link(text='后页>')
         except:
             Logger.info('finished!')
             break
     return new_reply_time

# 需要导入模块: from Utils import Logger [as 别名]
# 或者: from Utils.Logger import info [as 别名]
def run_linkspider(db, meta):
    source = 'douban'
    baseurls = [
        'http://www.douban.com/group/beijingzufang/discussion',
        'http://www.douban.com/group/fangzi/discussion',
        'http://www.douban.com/group/262626/discussion',
        'http://www.douban.com/group/276176/discussion',
        'http://www.douban.com/group/26926/discussion',
        'http://www.douban.com/group/sweethome/discussion',
        'http://www.douban.com/group/242806/discussion',
        'http://www.douban.com/group/257523/discussion',
        'http://www.douban.com/group/279962/discussion',
        'http://www.douban.com/group/334449/discussion',
    ]

    for baseurl in baseurls:
        Logger.info('start '+baseurl)        
        groupid = baseurl\
            .replace('http://www.douban.com/group/', '')\
            .replace('/discussion', '')
        reply_time = 0
        if meta.has(source, groupid)\
            and meta.get(source, groupid).has_key('reply_time'):
            reply_time = meta.get(source, groupid)['reply_time']
        linkspider = LinkSpider(
            baseurl=baseurl,
            db=db,
        )
        reply_time = linkspider.crawl(
            source=source,
            reply_time=reply_time,
            ext={
                'groupid':groupid
            }
        )
        meta.set(source, groupid, {
            'reply_time':reply_time
        })
    meta.write()

# 需要导入模块: from Utils import Logger [as 别名]
# 或者: from Utils.Logger import info [as 别名]
        # Fork - create daemon process
        try:
            pid = os.fork()
            if pid > 0:
                # I am the parent
                self.isDaemon = False
                pidFile.close()
                sys.exit(0)
            else:
                os.setsid()
                pid = os.fork()
                if pid > 0:
                    # write a pid to a file and leave the method
                    pidFile.write(str(pid) + " ")
                    LOGGER.info("A daemon with pidfile " + self.pidFile + " launched successfully, pid:" + str(pid))
                    self.isDaemon = False
                    pidFile.close()
                    return
                else:
                    # i'm child of a child - the daemon process, so continue
                    self.isDaemon = True
                    pidFile.close()
        except OSError, e:
            raise RuntimeError("Fork for runnable with pidfile " + self.pidFile + " failed: " + str(e))

        LOGGER.info("Running process with pidfile: " + self.pidFile + " [" + str(os.getpid()) + "]")
        sys.stdout.flush()
        # run the daemon tasks
        runnable.run()

# 需要导入模块: from Utils import Logger [as 别名]
# 或者: from Utils.Logger import info [as 别名]
                    merged[h] = (title2, url2, pub_time2)
                else:
                    title2, url2, pub_time2 = merged[h]
            else:
                title2, url2, pub_time2 = title, url, pub_time
            buf_list.append((h, title2, url2, pub_time2))
        if len(buf_list) > 1:
            buf_list = sorted(buf_list, key=lambda i:i[3], reverse=True)
            simdb.insert('\t'.join(
                [buf_list[0][0], json.dumps(buf_list[1:])]
            ))
     
def main(**args):
    modulepath = args['modulepath']
    finaldb_cut = os.path.join(modulepath, '../output/final.db.2')
    simdb = os.path.join(modulepath, '../output/sim.db')
    if os.path.exists(simdb):
        backup(simdb)
        os.remove(simdb)
    sim_merge(finaldb_cut, DB(simdb))

if __name__ == '__main__':
    starttime = datetime.datetime.now()   
    run = sys.argv[0]
    modulepath = os.path.dirname(run)
    main(
        modulepath=modulepath
    )
    endtime = datetime.datetime.now()   
    Logger.info('done! %lds' % (endtime-starttime).seconds)

# 需要导入模块: from Utils import Logger [as 别名]
# 或者: from Utils.Logger import info [as 别名]
        try:
            f = open(dip, "r")
            cacheImgFile = open(cacheImg, "w")
            # buffsize = 52428800 #read 50 MB at a time
            buffsize = (1024 ** 3) / 2  # read/write 512 MB at a time
            buff = f.read(buffsize)
            while buff:
                cacheImgFile.write(buff)
                buff = f.read(buffsize)
            f.close()
            cacheImgFile.close()
        except IOError, e:
            msg = "Can't open %s: %s" % (dip, e)
            raise ClusterManagerException(msg)
        else:
            LOGGER.info(("Disk image %s  (for %s) copied to cache file %s") % (dip, hostObj.name, cacheImg))

    def defineHost(self, host):
        """
        Defines virtual host in a cluster using given host object,
        not starting it. Host with the given name may be defined once
        in the system. Stores hosts objects in class property self.hosts.
        
        @param host: ClusterManager.Host object
        @raise ClusterManagerException: when fails
        @return: host object from libvirt lib
        """
        if self.hosts.has_key(host.uname):
            return self.hosts[host.uname][0]

        for h in self.hosts.itervalues():

# 需要导入模块: from Utils import Logger [as 别名]
# 或者: from Utils.Logger import info [as 别名]
        try:
            response = self.br.open(url, timeout=20.0)
        except urllib2.HTTPError, e:
            while e.code != 404:
                interval = Interval.val()
                time.sleep(interval)
                Logger.info('sleep %ds error %d %s' % (interval, e.code, url))
                try:
                    response = self.br.open(url, timeout=20.0)
                    Logger.info('skip 403 ' + url)
                    break
                except urllib2.HTTPError, e:
                    if e.code != 404:
                        continue
                except:
                    Logger.info('cannot handle browser error!')
                    break
            Interval.reset()
        except:
            pass
        time.sleep(delay)
        if response:
            page = response.read()
        else:
            page = ''
        return page

    def close(self):
        return self.br.close()

    def follow_link(self, **args):