Python resource.getrusage函数代码示例

def memory_usage():

    """
    This function ...
    :return: 
    """

    # If we are on linux
    if platform == "linux" or platform == "linux2":

        kilobytes = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss # peak memory usage (bytes on OS X, kilobytes on Linux)
        gigabytes = kilobytes * 1e-6

        return gigabytes

    # If we are on Mac OS X
    elif platform == "darwin":

        kilobytes = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss # peak memory usage (bytes on OS X, kilobytes on Linux)
        gigabytes = kilobytes * 1e-9

        return gigabytes

    # We don't support Windows
    elif platform == "win32": raise EnvironmentError("The Windows operating system is not supported")

    # Unrecognized platform
    else: raise EnvironmentError("Unrecognized platform")

    def execute(self, verbose=False):
        if verbose:
            note('executing: %s' % ' '.join("'%s'" % arg
                                            for arg in self.command))

        start_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
        start_time = time.time()

        p = subprocess.Popen(self.command,
                             stdout=open(self.stdout_path, 'w'),
                             stderr=open(self.stderr_path, 'w'),
                             env=self.env)
        self.result = p.wait() == 0

        end_time = time.time()
        end_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
        self.metrics["user_time"] = end_rusage.ru_utime - start_rusage.ru_utime
        self.metrics["sys_time"] = end_rusage.ru_stime - start_rusage.ru_stime
        self.metrics["wall_time"] = end_time - start_time

        if verbose:
            note("command executed in -- "
                 "user: %.4fs, wall: %.4fs, sys: %.4fs" % (
                    self.metrics["user_time"], self.metrics["wall_time"],
                    self.metrics["sys_time"]))

def conv_insts(fam, fam_io_manager, sp, sp_io_manager, 
               ninst=1, update_freq=100, verbose=False):
    n = 0
    for point in sp.points():
        param_uuid = uuid.uuid4()
        sp.record_point(point, param_uuid, sp_io_manager)
        for i in range(ninst):
            inst_uuid = uuid.uuid4()
            inst = sp.gen_inst(point, inst_uuid, sp_io_manager)
            fam.record_inst(inst, inst_uuid, param_uuid, sp.name, 
                            fam_io_manager)
            if n % update_freq == 0:
                if verbose:
                    # print('Total writes: {0}'.format(
                    #         sum([tbl.n_writes for tbl in fam_tables.values() + sp_tables.values()])))
                    print('Memusg before collect: {0}'.format(
                            resource.getrusage(resource.RUSAGE_SELF).ru_maxrss))
                gc.collect()
                if verbose:
                    print('Memusg after collect: {0}'.format(
                            resource.getrusage(resource.RUSAGE_SELF).ru_maxrss))
                    print('{0} instances have been converted'.format(n))
            n += 1
    
    if verbose:
        print('{0} instances have been converted'.format(n))

def resources(bot, user, chan, realtarget, *args):
	if chan is not None: replyto = chan
	else: replyto = user

	uptime = time.time() - bot.parent.starttime
	m, s = divmod(uptime, 60)
	h, m = divmod(m, 60)
	d, h = divmod(h, 24)
	try:
		res = resource.getrusage(resource.RUSAGE_BOTH)
	except:
		res = resource.getrusage(resource.RUSAGE_SELF)

	bot.slowmsg(replyto, "Resource usage:")
	for i, v in (
		('uptime (s)', "%d (%d days %02d:%02d:%02d)" % (uptime, d, h, m, s)),
		('utime (s)', res.ru_utime),
		('stime (s)', res.ru_stime),
		('memory (MiB)', (res.ru_maxrss/1024.0)),
		('I/O (blocks)', res.ru_inblock+res.ru_oublock),
		('page faults', res.ru_majflt),
		('signals', res.ru_nsignals),
		('context switches (voluntary)', res.ru_nvcsw),
		('context switches (involuntary)', res.ru_nivcsw),
	):
		bot.slowmsg(replyto, "- %s: %s" % (i, v))
	bot.slowmsg(replyto, "EOL.")

def LogVariableElimination(Factors, Ordering, verbose=True):    
    """ Variable Elimination algorithm """
    start_mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
    tw = 0 # elimination width
    wtw = 0 # weigthed width
    delta_mem = 0
    max_memory = 0
    for var in Ordering:
        if verbose:
        	print "-%s" % var.label, 
        	sys.stdout.flush()
        B = []
        for f in Factors:
        	if var in f.scope:
        		B.append(f)
        for f in B:
        	Factors.remove(f)
    	f = LogFactorSumBProduct(B,[var])
    	Factors.append(f)
        delta_mem = (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) - start_mem
        max_memory = max(delta_mem, max_memory)
        if verbose:
            print "[tw: %d,\tdim: %d,\tmem: %d MB]" % (len(f.scope),f.dimension,delta_mem / 1000000.0)
            tw = max(tw, len(f.scope))
            wtw = max(wtw, f.dimension)
            sys.stdout.flush()
    f = Factors.pop()
    while len(Factors) > 0:
        fp = Factors.pop()
        f = LogFactorProduct(f,fp)
    if verbose:
        print
    return f, tw, wtw, max_memory

def run(input_filename, output_filename):
    articles = defaultdict(set)

    without_identifiers = set()

    reader = csv.reader(open(input_filename, 'r'))

    try:
        biggest = 0

        for i, article in enumerate(reader):
            article = Article(*article)
            identifiers = [(k,v) for k,v in article._asdict().items() if k in IDENTIFIERS and v]
            data = None # dict(identifiers)
            if not identifiers:
                without_identifiers.add(article.id)
                continue
            articles[identifiers[0]].add(article.id)
            for identifier in identifiers[1:]:
                if articles[identifiers[0]] is not articles[identifier]:
                    articles[identifiers[0]] |= articles[identifier]
                    articles[identifier] = articles[identifiers[0]]
                    if len(articles[identifier]) > biggest:
                        biggest = len(articles[identifier])

            if i % 10000 == 0:
                print "%7d" % i, resource.getrusage(resource.RUSAGE_SELF)[2], biggest
                if resource.getrusage(resource.RUSAGE_SELF)[2] > 1e7:
                    print "Using too much memory"
                    raise Exception
    except Exception, e:
        print e

def get_kmer_counts(input, output, k, ns, nprocs, verbose):
    """Analyse kmers. Multiprocessing enabled"""
    #define base2digit dict for 4-char seq
    base2digit = {"A": "0", "C": "1", "G": "2", "T": "3"}    
    if ns:
        #change to 5-char seq if Ns in seq
        base2digit = {"A": "0", "C": "1", "G": "2", "N": "3", "T": "4"}
    #init mer counts
    #255 for uint8 #65,535 for uint16 or #4,294,967,295 for uint32 
    merCounts = np.zeros(len(base2digit)**k/2, dtype='uint16')
    #start pool #maxtasksperchild=1000)
    p = Pool(nprocs, initializer=init_args, initargs=(k, ns, base2digit)) 
    #process reads
    for i, ids in enumerate(p.imap_unordered(seq2mers, SeqIO.parse(input, 'fastq'), \
                                             chunksize=100), 1):
        if not i%1e4:
            sys.stderr.write(" %s [%s Mb]\r"%(i, resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024))
        for mid in ids:
            merCounts[mid] += 1
    sys.stderr.write(" %s [%s Mb]\n"%(i, resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024))
    #get mer freq
    maxCount    = merCounts.max()
    if maxCount < 100:
        maxCount = 100
    occurencies = [0]*maxCount
    for c in merCounts:
        occurencies[c-1] += 1
    #write to file
    output.write("\n".join("%s\t%s"%xy for xy in enumerate(occurencies,1))+"\n")
    return occurencies

    def moveFilesToFinalLocation(self):
        success = True
        # free some memory for file copy command
        if self.debug:
            print('DEBUG: max mem used A:', resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
        self.deleteSampleTree()
        if self.debug:
            print('DEBUG: max mem used B:', resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)

        for tmpFileName in self.tmpFiles:
            outputFileName = self.outputFolder + '/' + self.tmpFolder.join(tmpFileName.split(self.tmpFolder)[1:])
            print ('copy ', tmpFileName, ' to ', outputFileName)
            if self.fileLocator.fileExists(outputFileName):
                self.deleteFile(outputFileName)
            #command = 'xrdcp -d 1 ' + self.fileLocator.getXrootdFileName(tmpFileName) + ' ' + self.fileLocator.getXrootdFileName(outputFileName)
            #print('the command is', command)
            #sys.stdout.flush()
            #returnCode = subprocess.call([command], shell=True)
            copySuccessful = self.fileLocator.cp(tmpFileName, outputFileName)
            if not copySuccessful:
                success = False
                print('\x1b[31mERROR: copy failed for {tmpfile}->{outputfile} !\x1b[0m'.format(tmpfile=tmpFileName,
                                                                                                outputfile=outputFileName))
            else:
                # delete temporary file if copy was successful
                self.deleteFile(tmpFileName)
        return success

    def test_dumps_usage(self):
        '''
        repeatedly serialize, check that usage doesn't go up
        '''
        if cdumps is None:
            logger.warn('no C dumps(), skipping test_dumps_usage')
            return
        start_usage = resource.getrusage(resource.RUSAGE_SELF)
        usage_history = [start_usage]
        for o in _range(_TEST_OUTER):
            for i in _range(_TEST_COUNT):
                ob = _randob()
                blob = cdumps(ob)
                # and silently drop the result. I hope the garbage collector works!
            t_usage = resource.getrusage(resource.RUSAGE_SELF)
            usage_history.append(t_usage)
        end_usage = usage_history[-1]
        dmaxrss = end_usage.ru_maxrss - start_usage.ru_maxrss
        didrss = end_usage.ru_idrss - start_usage.ru_idrss
        dmaxrsspct = ((end_usage.ru_maxrss != 0) and (dmaxrss / end_usage.ru_maxrss)) or 0
        didrsspct = ((end_usage.ru_idrss != 0) and (didrss / end_usage.ru_idrss)) or 0

        sys.stderr.write('maxrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_maxrss, end_usage.ru_maxrss, dmaxrss, dmaxrsspct * 100.0))
        sys.stderr.write('idrss: {} - {}, d={} ({:.2f}%)\n'.format(start_usage.ru_idrss, end_usage.ru_idrss, didrss, didrsspct * 100.0))

        assert (dmaxrsspct) < 0.05, [x.ru_maxrss for x in usage_history]
        assert (didrsspct) < 0.05, [x.ru_idrss for x in usage_history]

 def update(self):
     import time
     self.current += 1
     self.percentage = int(round(100*float(self.current)/self.total))
     if self.percentage % self.printint == 0 and self.percentage != self.lpercentage:
         self.stf=int(round((self.total-self.current)/((self.current-self.lcurrent)/(time.time()-self.ltime))))
         if self.type == 'full' and self.logfile: self.logfile.write(
             '#Progress => '+str(self.percentage)+'%, '+
             str( round((self.current-self.lcurrent)/(time.time()-self.ltime),2) )+' '+self.unit+'/second, '+
             time.strftime("%A, %d %b %Y %H:%M:%S",time.localtime())+
             ', left: '+str(self.stf/60/60)+'h '+str(self.stf/60%60)+'min '+str(self.stf%60)+'s')
         if self.mem:
             import resource
             total_memory_used = (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss)
             this_process_memory_used = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
             if total_memory_used/1024/1024 > 1024:
                 self.logfile.write(', using '+str(round(float(total_memory_used)/1024/1024/1024,2))+' ('+str(round(float(this_process_memory_used)/1024/1024/1024,2))+') GB.\n')
             elif total_memory_used/1024 > 1024:
                 self.logfile.write(', using '+str(round(float(total_memory_used)/1024/1024,2))+' ('+str(round(float(this_process_memory_used)/1024/1024,2))+') MB.\n')
             else:
                 self.logfile.write(', using '+str(round(float(total_memory_used)/1024,2))+' ('+str(round(float(this_process_memory_used)/1024,2))+') KB.\n')
         else:    self.logfile.write('\n')
         if self.type == 'minimal': self.logfile.write('..')
         self.ltime = time.time()
         self.lcurrent = self.current
         self.lpercentage = self.percentage

    def run(self):
        self.comm.start()       # Start the "Comm" thread
        while not self.running:
            self.handle_message(True)

        rudata_start = resource.getrusage(resource.RUSAGE_SELF)
        count = 0
        while(self.running and count < self.iterations):
            # Non CS:
            self.label()        # labels mark the points where we can "break"
            self.work()
            self.label()
            self.enter_critical_section()
            print("Process %d(clock:%d) has entered critical section."%
                  (self.peerid,self.clock))
            self.label()
            self.work()
            self.label()
            print("Process %d is leaving critical section."%self.peerid)
            self.leave_critical_section()
            count += 1

        rudata_end = resource.getrusage(resource.RUSAGE_SELF)
        utime = rudata_end.ru_utime - rudata_start.ru_utime
        stime = rudata_end.ru_stime - rudata_start.ru_stime
        send(self.net, LOG, (utime, stime, rudata_end.ru_maxrss))
        while self.running:
            self.handle_message(True)

def generic_batch_processor_v2(harvester, bman_list):

    error_map = {}
    next_bman_list = []
    failed_list = []
    max_step_size = 50
    step_size = max_step_size #full throttle!
    fail_ratio = 0.15
    step_factor = 1.66
    lap_start = time.time()
    bman_total = 1
    error_sum = 0

    while bman_list:
        usage = resource.getrusage(resource.RUSAGE_SELF)
        logger.info(u"New batch. Size:%d for %s Mem:%s MB" % (len(bman_list), harvester,unicode(getattr(usage, "ru_maxrss")/(1024.0))))

        if (E_UNEX in error_map and error_map[E_UNEX] / float(bman_total) > fail_ratio) or error_sum > 4:
            step_size = int(step_size / step_factor) if int(step_size / step_factor) > 1 else 1
            del error_map[E_UNEX]
        else:
            step_size = step_size * 2 if step_size * 2 < max_step_size else max_step_size

        split_bman = [bman_list[i:i+step_size] for i  in range(0, len(bman_list), step_size)]
        bman_total = len(split_bman)

        for (counter, bman_chunk) in enumerate(split_bman,1):
            
            if not(E_UNEX in error_map and error_map[E_UNEX] / float(bman_total) > fail_ratio) or not (E_USER_QUOTA in error_map):
                actual_fail_ratio = error_map[E_UNEX] / float(bman_total) if E_UNEX in error_map else 0
                usage = resource.getrusage(resource.RUSAGE_SELF)
                logger.info(u"bman_chunk (%d/%d) chunk_total:%s InQueue:%d fail_ratio:%s > %s Mem:%s KB" % (counter, bman_total, len(bman_chunk), len(next_bman_list), actual_fail_ratio, fail_ratio, getattr(usage, "ru_maxrss")/(1024.0)))

                if E_QUOTA in error_map:
                    logger.info("Quota error, waiting for 10 minutes")
                    del error_map[E_QUOTA]
                    time.sleep(10*60)
                
                if (time.time() - lap_start) < 1:
                    logger.info(u"Speed too fast. will wait 1 sec")
                    time.sleep(1)

                lap_start = time.time()
                error = gbp_core(harvester, bman_chunk, error_map, next_bman_list, failed_list)
                error_sum = error_sum + 1 if error else 0
                logger.info(u"gbp_core: len(next_bman_list): %s" % len(next_bman_list))
            elif E_USER_QUOTA in error_map:
                logger.error("bman(%d/%d) User quota reached. Aborting the harvest!" % (counter, bman_total))
                failed_list += bman_chunk
            else:
                logger.info("bman(%d/%d) Failed ratio too high. Retrying with smaller batch" % (counter, bman_total))
                next_bman_list += bman_chunk

        bman_list = next_bman_list
        next_bman_list = []

    usage = resource.getrusage(resource.RUSAGE_SELF)
    readable_failed_list = [failed_list[j]["request"]["relative_url"] for j in range(0, len(failed_list))]
    logger.debug(u"END harvesting. Mem:%s MB" % (getattr(usage, "ru_maxrss")/(1024.0)))
    logger.debug(u"Failed list: %s" % (readable_failed_list))

	def run_protocol(self):

		if self.sockets == None:
			""" Need to set up sockets """
			self.setup_sockets()
		elif len(self.sockets) == 1:
			""" This is a non-leader node """
			self.leader_socket = self.sockets[0]
			self.sockets = None

		try:
			self.run_phase2()
			self.run_phase3()
			self.run_phase4()
			self.run_phase5()
		except:
			self.cleanup_sockets()
			raise
		self.cleanup_sockets()

		self.info("Finished in %g seconds" % (time() - self.start_time))
		self.critical("SUCCESSROUND:SHUFFLE, RID:%d, NNOD:%d, WALLTIME:%g, USR:%g, SYS:%g\n\t%s" % \
				(self.round_id,
				 self.n_nodes, 
				 time() - self.start_time, 
				 resource.getrusage(resource.RUSAGE_SELF).ru_utime - self.rusage_start[0],
				 resource.getrusage(resource.RUSAGE_SELF).ru_stime - self.rusage_start[1],
				 self.size_string()))

    def __mmc_store_log(self, final=False):
        try:
            from mmc.models import MMCLog

            memory = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
            resources = resource.getrusage(resource.RUSAGE_SELF)

            utime = resources.ru_utime - self._mmc_resources.ru_utime
            stime = resources.ru_stime - self._mmc_resources.ru_stime
            div = 1024.0 if 'linux' in sys.platform else 1048576.0
            self._mmc_elapsed_time = time.time() - self._mmc_start_time

            MMCLog.logging(
                instance=self._mmc_log_instance,
                start=self._mmc_start_date,
                hostname=self._mmc_hostname,
                script=self._mmc_script,
                elapsed="%0.2f" % self._mmc_elapsed_time,
                success=self._mmc_success,
                error_message=self._mmc_error_message,
                traceback=self._mmc_traceback,
                sys_argv=self.__mmc_get_sys_argv(),
                memory="%0.2f" % (memory / div),
                cpu_time="%0.2f" % (utime + stime),
                stdout_messages=self.__mmc_get_stdout(),
                pid=os.getpid(),
                queries=self.__mmc_get_queries(),
                is_fixed=False if self._mmc_success is False else None,
                end=now() if final else self._mmc_log_instance.end
            )
        except Exception as err:
            stderr("[MMC] Logging broken with message: {0}".format(err))

    def dfs(self):
        visitedStates= Set()
        stack = []
        maxSize = 1
        current = self
        stack.append(self)
        nodeExpand = 1
        foundGoal = False

        if current.isGoalState():
            mem  = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
            return True, current, maxSize, nodeExpand, mem

        while len(stack) != 0:
            current = stack.pop()
            nodeExpand += 1
            for direct in DIRECTION:
                childList = current.simulateMove(direct)
                if childList != None:
                   if tuple(childList) not in visitedStates:
                        current.child.append(State(self.n, childList, current, direct))
                        stack.append(current.child[-1])
                        visitedStates.add(tuple(current.child[-1].list))
                        maxSize = max(len(stack),maxSize)
                        if current.child[-1].isGoalState():
                            current = current.child[-1]
                            foundGoal = True
                            break
            # compare for max size of the queue
            if foundGoal:
                break
        mem  = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000
        return foundGoal, current, maxSize, nodeExpand, mem