Python pynvml.NVMLError方法代码示例

# 需要导入模块: import pynvml [as 别名]
# 或者: from pynvml import NVMLError [as 别名]
def _get_driver(self):
        """ Obtain and return the installed driver version for the system's GPUs.

        Returns
        -------
        str
            The currently installed GPU driver version
        """
        if self._is_plaidml:
            driver = self._plaid.drivers
        elif IS_MACOS:
            driver = pynvx.cudaSystemGetDriverVersion(ignore=True)
        else:
            try:
                driver = pynvml.nvmlSystemGetDriverVersion().decode("utf-8")
            except pynvml.NVMLError:
                driver = "No Nvidia driver found"
        self._log("debug", "GPU Driver: {}".format(driver))
        return driver 

# 需要导入模块: import pynvml [as 别名]
# 或者: from pynvml import NVMLError [as 别名]
def _init_nvml(self):
        if self._load_nvidia_lib() == -1:
            return -1

        try:
            global pynvml
            import pip
            pip.main(['install', '--quiet', 'nvidia-ml-py'])
            import pynvml as pynvml
            pynvml.nvmlInit()
            return 0
        except pynvml.NVMLError, err:
            logger.debug('Failed to initialize NVML: ', err)
            return -1 

# 需要导入模块: import pynvml [as 别名]
# 或者: from pynvml import NVMLError [as 别名]
def _shutdown_nvml(self):
        try:
            pynvml.nvmlShutdown()
        except pynvml.NVMLError, err:
            logger.debug('Failed to shutdown NVML: ', err) 

# 需要导入模块: import pynvml [as 别名]
# 或者: from pynvml import NVMLError [as 别名]
def _get_container_id(self, gpuhandle):
        cont_ids = []
        pids = []
        try:
            proc_objs = pynvml.nvmlDeviceGetComputeRunningProcesses(gpuhandle)
            if not proc_objs:
                return ['NA.NA']
            for proc_obj in proc_objs:
                pids.append(proc_obj.pid)
            for pid in pids:
                cont_ids.append(self._get_containerid_from_pid(pid))
            return cont_ids
        except pynvml.NVMLError, err:
            logger.debug('Failed to get pid on gpu: ', err) 

# 需要导入模块: import pynvml [as 别名]
# 或者: from pynvml import NVMLError [as 别名]
def __exit__(self, exception_type, exception_value, traceback):
        # Do nothing if the exception is not from pynvml or there is no exception
        if traceback is None:
            return False
        if not issubclass(exception_type, pynvml.NVMLError):
            return False

        # Suppress pynvml exceptions so we can continue
        if self.name in NvmlCall.previously_printed_errors:
            return True
        NvmlCall.previously_printed_errors.add(self.name)
        self.log.warning("Unable to execute NVML function: %s: %s", self.name, exception_value)
        return True 

# 需要导入模块: import pynvml [as 别名]
# 或者: from pynvml import NVMLError [as 别名]
def mem_used_for(device_handle):
    """Get GPU device memory consumption in percent"""
    try:
        return pynvml.nvmlDeviceGetMemoryInfo(device_handle).used
    except pynvml.NVMLError:
        return None 

# 需要导入模块: import pynvml [as 别名]
# 或者: from pynvml import NVMLError [as 别名]
def mem_used_percent_for(device_handle):
    """Get GPU device memory consumption in percent"""
    try:
        memory_info = pynvml.nvmlDeviceGetMemoryInfo(device_handle)
        return memory_info.used * 100.0 / memory_info.total
    except pynvml.NVMLError:
        return None 

# 需要导入模块: import pynvml [as 别名]
# 或者: from pynvml import NVMLError [as 别名]
def utilization_for(device_handle):
    """Get GPU device consumption in percent
        Percent of time over the past sample period during which one or more kernels was executing on the GPU.
    """
    try:
        return pynvml.nvmlDeviceGetUtilizationRates(device_handle).gpu
    except pynvml.NVMLError:
        return None 

# 需要导入模块: import pynvml [as 别名]
# 或者: from pynvml import NVMLError [as 别名]
def mem_utilization_for(device_handle):
    """
        Percent of time over the past sample period during which global (device) memory was being read or written.
    """
    try:
        return pynvml.nvmlDeviceGetUtilizationRates(device_handle).memory
    except pynvml.NVMLError:
        return None 

# 需要导入模块: import pynvml [as 别名]
# 或者: from pynvml import NVMLError [as 别名]
def power_for(device_handle):
    try:
        return pynvml.nvmlDeviceGetPowerUsage(device_handle)
    except pynvml.NVMLError:
        return None 

# 需要导入模块: import pynvml [as 别名]
# 或者: from pynvml import NVMLError [as 别名]
def _get_device_count(self):
        """ Detect the number of GPUs attached to the system and allocate to
        :attr:`_device_count`. """
        if self._is_plaidml:
            self._device_count = self._plaid.device_count
        elif IS_MACOS:
            self._device_count = pynvx.cudaDeviceGetCount(ignore=True)
        else:
            try:
                self._device_count = pynvml.nvmlDeviceGetCount()
            except pynvml.NVMLError:
                self._device_count = 0
        self._log("debug", "GPU Device count: {}".format(self._device_count)) 

# 需要导入模块: import pynvml [as 别名]
# 或者: from pynvml import NVMLError [as 别名]
def can_log_gpu_resources():
    if pynvml is None:
        return False

    try:
        pynvml.nvmlInit()
        return True
    except pynvml.NVMLError:
        return False 

# 需要导入模块: import pynvml [as 别名]
# 或者: from pynvml import NVMLError [as 别名]
def get_gpu_metrics() -> List:
    try:
        pynvml.nvmlInit()
        device_count = pynvml.nvmlDeviceGetCount()
        results = []

        for i in range(device_count):
            handle = pynvml.nvmlDeviceGetHandleByIndex(i)
            results += metrics_dict_to_list(query_gpu(handle))
        return results
    except pynvml.NVMLError:
        logger.debug("Failed to collect gpu resources", exc_info=True)
        return [] 

# 需要导入模块: import pynvml [as 别名]
# 或者: from pynvml import NVMLError [as 别名]
def _monitor(self):
        pynvml.nvmlInit()
        self._find_gpu()
        current_sample = []
        while not self.should_stop:
            used_cpu = None
            used_cpumem = None
            used_gpu = None
            used_gpumem = None
            
            cpu_process = psutil.Process(self.pid)
            used_cpu = cpu_process.cpu_percent() / psutil.cpu_count() # CPU utilization in %
            used_cpumem = cpu_process.memory_info().rss // (1024*1024) # Memory use in MB

            gpu_processes = pynvml.nvmlDeviceGetComputeRunningProcesses(self.gpu)
            for gpu_process in gpu_processes:
                if gpu_process.pid == self.pid:
                    used_gpumem = gpu_process.usedGpuMemory // (1024*1024) # GPU memory use in MB
                    break

            if self.accounting_enabled:
                try:
                    stats = pynvml.nvmlDeviceGetAccountingStats(self.gpu, self.pid)
                    used_gpu = stats.gpuUtilization
                except pynvml.NVMLError: # NVMLError_NotFound
                    pass

            if not used_gpu:
                util = pynvml.nvmlDeviceGetUtilizationRates(self.gpu)
                used_gpu = util.gpu / len(gpu_processes) # Approximate based on number of processes

            current_sample.append((used_cpu, used_cpumem, used_gpu, used_gpumem))

            time.sleep(self.sampling_rate)

        self.stats.append([round(sum(x) / len(x)) for x in zip(*current_sample)])
        pynvml.nvmlShutdown()