Python resource.getrusage方法代码示例

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def execute(self, verbose=False):
        if verbose:
            note('executing: %s' % ' '.join("'%s'" % arg
                                            for arg in self.command))

        start_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
        start_time = time.time()

        p = subprocess.Popen(self.command,
                             stdout=open(self.stdout_path, 'w'),
                             stderr=open(self.stderr_path, 'w'),
                             env=self.env)
        self.result = p.wait() == 0

        end_time = time.time()
        end_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
        self.metrics["user_time"] = end_rusage.ru_utime - start_rusage.ru_utime
        self.metrics["sys_time"] = end_rusage.ru_stime - start_rusage.ru_stime
        self.metrics["wall_time"] = end_time - start_time

        if verbose:
            note("command executed in -- "
                 "user: %.4fs, wall: %.4fs, sys: %.4fs" % (
                    self.metrics["user_time"], self.metrics["wall_time"],
                    self.metrics["sys_time"])) 

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def _test_get_segment_info_memory_usage(self, ec_driver):
        # 1. Preapre the expected memory allocation
        ec_driver.get_segment_info(1024 * 1024, 1024 * 1024)
        loop_range = range(1000)

        # 2. Get current memory usage
        usage = resource.getrusage(resource.RUSAGE_SELF)[2]

        # 3. Loop to call get_segment_info
        for x in loop_range:
            ec_driver.get_segment_info(1024 * 1024, 1024 * 1024)

        # 4. memory usage shoudln't be increased
        self.assertEqual(usage, resource.getrusage(resource.RUSAGE_SELF)[2],
                         'Memory usage is increased unexpectedly %s - %s' %
                         (usage, resource.getrusage(resource.RUSAGE_SELF)[2])) 

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def _test_get_metadata_memory_usage(self, ec_driver):
        # 1. Prepare the expected memory allocation
        encoded = ec_driver.encode(b'aaa')
        ec_driver.get_metadata(encoded[0], formatted=True)
        loop_range = range(400000)

        # 2. Get current memory usage
        baseline_usage = resource.getrusage(resource.RUSAGE_SELF)[2]

        # 3. Loop to call get_metadata
        for x in loop_range:
            ec_driver.get_metadata(encoded[0], formatted=True)

        # 4. memory usage shouldn't increase
        new_usage = resource.getrusage(resource.RUSAGE_SELF)[2]
        self.assertEqual(baseline_usage, new_usage,
                         'Memory usage is increased unexpectedly %s -> %s' %
                         (baseline_usage, new_usage)) 

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def _get_internal_metrics(tasks: List[Task]) -> List[Metric]:
    """Internal wca metrics e.g. memory usage, profiling information."""

    # Memory usage.
    memory_usage_rss_self = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
    memory_usage_rss_children = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
    memory_usage_rss = memory_usage_rss_self + memory_usage_rss_children

    metrics = [
        Metric.create_metric_with_metadata(MetricName.WCA_UP, value=time.time()),
        Metric.create_metric_with_metadata(MetricName.WCA_TASKS, value=len(tasks)),
        Metric.create_metric_with_metadata(MetricName.WCA_MEM_USAGE_BYTES,
                                           value=int(memory_usage_rss * 1024)),
    ]

    return metrics 

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def start(self, verbose=False, payload=None):
        if verbose:
            self.process = subprocess.Popen(filter(None, self.cmd),
                                            stdin=None,
                                            stdout=None,
                                            stderr=None)
        else:
            self.process = subprocess.Popen(filter(None, self.cmd),
                                            stdin=subprocess.PIPE,
                                            stdout=subprocess.PIPE,
                                            stderr=None)

        self.stat_fd = open("/proc/" + str(self.process.pid) + "/stat")
        self.init()
        try:
            self.set_init_state(payload=payload)
        except:
            return False
        self.initial_mem_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
        # time.sleep(1)
        self.kafl_shm.seek(0x0)
        self.kafl_shm.write(self.virgin_bitmap)
        # self.kafl_shm.flush()
        return True 

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def print_used_time(func):
        ''' 打印运行时间

        :param func: 运行的函数名称
        :return:
        '''

        @wraps(func)
        def wrapper(*args, **kwargs):
            start_time, start_resources = timestamp(), resource_usage(RUSAGE_SELF)
            func(*args, **kwargs)
            end_resources, end_time = resource_usage(RUSAGE_SELF), timestamp()
            print({'消耗时间': {'real': end_time - start_time,
                            'sys': end_resources.ru_stime - start_resources.ru_stime,
                            'user': end_resources.ru_utime - start_resources.ru_utime}})
            return True
        return wrapper 

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def _get_mem_usage(self):
        """
        Gets the RUSAGE memory usage, returns in K. Encapsulates the difference
        between macOS and Linux rss reporting

        :returns: memory usage in kilobytes
        """

        from resource import getrusage, RUSAGE_SELF
        mem = getrusage(RUSAGE_SELF).ru_maxrss
        if sys.platform == 'darwin':
            # man 2 getrusage:
            #     ru_maxrss
            # This is the maximum resident set size utilized (in bytes).
            return mem / 1024  # Kb
        else:
            # linux
            # man 2 getrusage
            #        ru_maxrss (since Linux 2.6.32)
            #  This is the maximum resident set size used (in kilobytes).
            return mem  # Kb 

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def get_usage():
    # This function only works if you have a signal handler for the
    # signal.SIGCHLD signal.
    raw_usage = resource.getrusage(resource.RUSAGE_CHILDREN)
    # 0   ru_utime    time in user mode (float)
    # 1   ru_stime    time in system mode (float)
    # 2   ru_maxrss   maximum resident set size
    #
    # These fields are always zero on Linux
    # 3   ru_ixrss    shared memory size
    # 4   ru_idrss    unshared memory size
    # 5   ru_isrss    unshared stack size
    return "User:{}s System:{}s".format(
        int(raw_usage.ru_utime),
        int(raw_usage.ru_stime),
    ) 

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def start(self, verbose=False):
        if verbose:
            self.process = subprocess.Popen(filter(None, self.cmd.split(" ")),
                                            stdin=None,
                                            stdout=None,
                                            stderr=None)
        else:
            self.process = subprocess.Popen(filter(None, self.cmd.split(" ")),
                                            stdin=subprocess.PIPE,
                                            stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE)

        self.stat_fd = open("/proc/" + str(self.process.pid) + "/stat")
        self.init()
        try:
            self.set_init_state()
        except:
            return False
        self.initial_mem_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
        #time.sleep(1)
        self.kafl_shm.seek(0x0)
        self.kafl_shm.write(self.virgin_bitmap)
        self.kafl_shm.flush()
        return True 

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def get_used_memory():
        """ Return the used memory in MB """
        if platform.system() == 'Linux':
            for line in open('/proc/self/status'):
                if line.startswith('VmRSS:'):
                    return int(line.split()[1]) >> 10

        else:
            warnings.warn("Please install psutil to have better "
                          "support with spilling")
            if platform.system() == "Darwin":
                import resource
                rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
                return rss >> 20
            # TODO: support windows

        return 0 

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def update(self):
            """
            Get memory metrics of current process through `getrusage`.  Only
            available on Unix, on Linux most of the fields are not set,
            and on BSD units are used that are not very helpful, see:

            http://www.perlmonks.org/?node_id=626693

            Furthermore, getrusage only provides accumulated statistics (e.g.
            max rss vs current rss).
            """
            usage = getrusage(RUSAGE_SELF)
            self.rss = usage.ru_maxrss * 1024
            self.data_segment = usage.ru_idrss * 1024 # TODO: ticks?
            self.shared_segment = usage.ru_ixrss * 1024 # TODO: ticks?
            self.stack_segment = usage.ru_isrss * 1024 # TODO: ticks?
            self.vsz = self.data_segment + self.shared_segment + \
                       self.stack_segment

            self.pagefaults = usage.ru_majflt
            return self.rss != 0 

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def load_dev_data2(self,data_path:str, start:int, stop:int):
        pref = self.parameters.pref

        dev_data = []

        logging.info('dev_data')
        logging.info('Memory usage: %s', resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
        for x in pad_generator(batch_generator(
                generate_arrays_from_file(data_path, self.unique_features_dict,
                                          pref['feature_name'], pref['label_name'],
                                          start=start,
                                          stop=stop, keep_infinity=False,
                                          keep_unaligned=True),
                batch_size=pref['batch_size'])):
            dev_data.append(x)

        self.dev_data = DataList(dev_data) 

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def x(self):
        pref = self.parameters.pref

        self.load_test_data()
        self.load_dev_data()


        logging.info('train_data')
        logging.info('Memory usage: %s', resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
        train_data = DataGenerator(self.data_path, self.unique_features_dict, pref,
                                   range(pref['train_data_size']))
        self.train_data = train_data




        logging.info('Data created') 

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def peak_memory_mb()         :
    u"""
    Get peak memory usage for this process, as measured by
    max-resident-set size:

    https://unix.stackexchange.com/questions/30940/getrusage-system-call-what-is-maximum-resident-set-size

    Only works on OSX and Linux, returns 0.0 otherwise.
    """
    if resource is None or sys.platform not in (u'linux', u'darwin'):
        return 0.0

    # TODO(joelgrus): For whatever, our pinned version 0.521 of mypy does not like
    # next line, but later versions (e.g. 0.530) are fine with it. Once we get that
    # figured out, remove the type: ignore.
    peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss  # type: ignore

    if sys.platform == u'darwin':
        # On OSX the result is in bytes.
        return peak / 1000000

    else:
        # On Linux the result is in kilobytes.
        return peak / 1000 

# 需要导入模块: import resource [as 别名]
# 或者: from resource import getrusage [as 别名]
def testMemory(self):
        engine, = PFAEngine.fromYaml("""
input: string
output: {type: array, items: int}
action:
  - {re.rindex: [input, [ab(c|d)*]]}
""")
        import resource, time
        memusage_1 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
        for i in range(0, 10000):
            engine.action("abcccdc")
        memusage_2 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss

        print("\nMemory usage before: {0}, after: {1}".format(memusage_1, memusage_2))