Python portalocker.Lock方法代码示例

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def write_file(filename, obj):
    file_data = 0
    while file_data is not None:
        lock = portalocker.Lock(filename, mode='a+b', flags=portalocker.LOCK_EX)
        lock.acquire()
        fh = lock.fh
        fh.seek(0)
        if len(fh.read()) is 0:
            file_data = None
        else:
            fh.seek(0)
            file_data = pickle.load(fh)
        if file_data is None:
            clear_file(fh)
            pickle.dump(obj, fh)
        lock.release() 

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def setAccount(account_id):
    try:
        config.logger.debug("account:setAccount(" + str(account_id) + ")")
        account_cache_semaphore.acquire(1)
        with portalocker.Lock(account_cache_lock_path, timeout=1) as _:
            with shelve.open(account_cache_path) as db:
                if account_id in db:
                    return db[account_id]
                else:
                    new_nr = len(db)
                    db[account_id] = new_nr
                    return new_nr
    except Exception as e:
        import sys
        _, _, exc_tb = sys.exc_info()
        config.logger.error("account: Exception in setAccount() line %s: %s",exc_tb.tb_lineno,e)
        return None
    finally:
        if account_cache_semaphore.available() < 1:
            account_cache_semaphore.release(1) 

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def getPath(uuid):
    try:
        config.logger.debug("register:getPath(" + str(uuid) + ")")
        register_semaphore.acquire(1)
        with portalocker.Lock(uuid_cache_path, timeout=1) as _:
            with shelve.open(uuid_cache_path) as db:
                try:
                    return str(db[uuid])
                except:
                    return None
    except Exception as e:
        config.logger.error("roast: Exception in getPath() %s",e)
        return None
    finally:
        if register_semaphore.available() < 1:
            register_semaphore.release(1) 

# scanns all .alog files for uuids and registers them in the cache 

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def step(self):
    actions = self.get_action(np.stack(self.obs))
    self.first = False
    [pipe.send(action) for pipe, action in zip(self.agent_pipes, actions)]
    next_obs, rewards, dones, resets = list(zip(*[pipe.recv() for pipe in self.agent_pipes]))

    frames = list(zip(self.obs, next_obs, actions, rewards, dones))

    self.obs = [o if resets[i] is False else self.agent_pipes[i].recv() for i, o in enumerate(next_obs)]

    for i, (t,r,reset) in enumerate(zip(self.total_rewards, rewards, resets)):
      if reset:
        self.total_rewards[i] = 0.
        if self.evaluation and self.loaded_policy:
          with portalocker.Lock(self.log_path+'.greedy.csv', mode="a") as f: f.write("%2f,%d,%d,%2f\n" % (self.hours, self.epoch, self.frame_total, t+r))

      else:
        self.total_rewards[i] = t + r

    if self.evaluation and np.any(resets): self.reload()

    self.rollout_i += 1
    return frames 

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def __init__(self,
                 num_procs,
                 dirname='file_comm',
                 worker_queue_file='worker_queue',
                 worker_results_prefix='worker_results_'):
        # make directory where communication files are created
        try:
            os.makedirs(dirname)
        except OSError:
            pass

        # claim a rank for the process
        lock = portalocker.Lock(os.path.join(dirname, 'init'),
                                mode='a+',
                                flags=portalocker.LOCK_EX)
        lock.acquire()
        fh = lock.fh
        fh.seek(0)
        curnum = fh.read()
        if len(curnum) is 0:
            rank = 0
        else:
            rank = int(curnum)

        if rank >= num_procs:
            raise ValueError('Number of processes > the number of workers')
        fh.seek(0)
        fh.truncate(0)
        fh.write(str(rank + 1))
        lock.release()

        super(FileCommunicator, self).__init__(num_procs - 1, rank)
        self.worker_queue_file = os.path.join(dirname, worker_queue_file)
        self.worker_results_prefix = os.path.join(dirname,
                                                  worker_results_prefix)
        self.done = False 

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def consume_file(filename):
    lock = portalocker.Lock(filename, mode='a+b', flags=portalocker.LOCK_EX)
    lock.acquire()
    fh = lock.fh
    fh.seek(0)
    if len(fh.read()) is 0:
        file_data = None
    else:
        fh.seek(0)
        file_data = pickle.load(fh)
    if file_data is not None:
        clear_file(fh)
        pickle.dump(None, fh)
    lock.release()
    return file_data 

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def read_file(filename):
    lock = portalocker.Lock(filename, mode='a+b', flags=portalocker.LOCK_EX)
    lock.acquire()
    fh = lock.fh
    fh.seek(0)
    if len(fh.read()) is 0:
        file_data = None
    else:
        fh.seek(0)
        file_data = pickle.load(fh)
    lock.release()
    return file_data 

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def handle_submit(self, sender):
        import portalocker
        with portalocker.Lock(self.results_filename, "a+") as g:
            g.write("%s::%s::%s::%s\n" % (self.id, getpass.getuser(), datetime.datetime.today(), sender.description))
            g.flush()
            os.fsync(g.fileno())
        self.output.clear_output()
        with self.output:
            print("Received: " + sender.description) 

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def addSync(uuid,modified_at):
    try:
        config.logger.debug("sync:addSync(" + str(uuid) + "," + str(modified_at) + ")")
        sync_cache_semaphore.acquire(1)
        with portalocker.Lock(getSyncPath(lock=True), timeout=1) as _:
            with shelve.open(getSyncPath()) as db:
                db[uuid] = modified_at
    except Exception as e:
        config.logger.error("sync: Exception in addSync() %s",e)
    finally:
        if sync_cache_semaphore.available() < 1:
            sync_cache_semaphore.release(1)  
    
# returns None if given uuid is not registered for syncing, otherwise the last modified_at timestamp in EPOC milliseconds 

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def addPath(uuid,path):
    try:
        config.logger.debug("register:setPath(" + str(uuid) + "," + str(path) + ")")
        register_semaphore.acquire(1)
        with portalocker.Lock(uuid_cache_path, timeout=1) as _:
            with shelve.open(uuid_cache_path) as db:
                db[uuid] = str(path)
    except Exception as e:
        config.logger.error("roast: Exception in addPath() %s",e)
    finally:
        if register_semaphore.available() < 1:
            register_semaphore.release(1)  
    
# returns None if given uuid is not registered, otherwise the registered path 

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def load(cls, apps_dir=None):
        if os.path.exists(cls.get_apps_json_path()):
            with portalocker.Lock(
                    cls.get_apps_json_path(apps_dir=apps_dir), 'r',
                    portalocker.LOCK_EX) as jf:
                data = jf.read()
                CartoviewApp.objects.from_json(data) 

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def save(cls, apps_dir=None):
        with portalocker.Lock(
                cls.get_apps_json_path(apps_dir=apps_dir), 'w',
                portalocker.LOCK_EX) as jf:
            data = CartoviewApp.objects.to_json()
            jf.write(data) 

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def write(self, file_or_path, append=False, timeout=10):
        """
        Write Smother results to a file.

        Parameters
        ----------
        fiile_or_path : str
            Path to write report to
        append : bool
            If True, read an existing smother report from `outpath`
            and combine it with this file before writing.
        timeout : int
            Time in seconds to wait to acquire a file lock, before
            raising an error.

        Note
        ----
        Append mode is atomic when file_or_path is a path,
        and can be safely run in a multithreaded or
        multiprocess test environment.

        When using `parallel_mode`, file_or_path is given a unique
        suffix based on the machine name and process id.
        """
        if isinstance(file_or_path, six.string_types):
            if self.coverage:
                file_or_path = get_smother_filename(
                    file_or_path, self.coverage.config.parallel)

            outfile = Lock(
                file_or_path, mode='a+',
                timeout=timeout,
                fail_when_locked=False
            )
        else:
            outfile = noclose(file_or_path)

        with outfile as fh:

            if append:
                fh.seek(0)
                try:
                    other = Smother.load(fh)
                except ValueError:  # no smother data
                    pass
                else:
                    self |= other

            fh.seek(0)
            fh.truncate()  # required to overwrite data in a+ mode
            json.dump(self.data, fh) 

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def import_try_install(package, extern_url=None):
    """Try import the specified package.
    If the package not installed, try use pip to install and import if success.

    Parameters
    ----------
    package : str
        The name of the package trying to import.
    extern_url : str or None, optional
        The external url if package is not hosted on PyPI.
        For example, you can install a package using:
         "pip install git+http://github.com/user/repo/tarball/master/egginfo=xxx".
        In this case, you can pass the url to the extern_url.

    Returns
    -------
    <class 'Module'>
        The imported python module.

    """
    import tempfile
    import portalocker
    lockfile = os.path.join(tempfile.gettempdir(), package + '_install.lck')
    with portalocker.Lock(lockfile):
        try:
            return __import__(package)
        except ImportError:
            try:
                from pip import main as pipmain
            except ImportError:
                from pip._internal import main as pipmain
                from types import ModuleType
                # fix for pip 19.3
                if isinstance(pipmain, ModuleType):
                    from pip._internal.main import main as pipmain

            # trying to install package
            url = package if extern_url is None else extern_url
            pipmain(['install', '--user', url])  # will raise SystemExit Error if fails

            # trying to load again
            try:
                return __import__(package)
            except ImportError:
                import sys
                import site
                user_site = site.getusersitepackages()
                if user_site not in sys.path:
                    sys.path.append(user_site)
                return __import__(package)
    return __import__(package) 

# 需要导入模块: import portalocker [as 别名]
# 或者: from portalocker import Lock [as 别名]
def get_model_file(name, root=os.path.join('~', '.encoding', 'models')):
    r"""Return location for the pretrained on local file system.

    This function will download from online model zoo when model cannot be found or has mismatch.
    The root directory will be created if it doesn't exist.

    Parameters
    ----------
    name : str
        Name of the model.
    root : str, default '~/.encoding/models'
        Location for keeping the model parameters.

    Returns
    -------
    file_path
        Path to the requested pretrained model file.
    """
    if name not in _model_sha1:
        from torchvision.models.resnet import model_urls
        if name not in model_urls:
            raise ValueError('Pretrained model for {name} is not available.'.format(name=name))
        root = os.path.expanduser(root)
        return download(model_urls[name],
                        path=root,
                        overwrite=True)
    file_name = '{name}-{short_hash}'.format(name=name, short_hash=short_hash(name))
    root = os.path.expanduser(root)
    if not os.path.exists(root):
        os.makedirs(root)

    file_path = os.path.join(root, file_name+'.pth')
    sha1_hash = _model_sha1[name]

    lockfile = os.path.join(root, file_name + '.lock')
    with portalocker.Lock(lockfile, timeout=300):
        if os.path.exists(file_path):
            if check_sha1(file_path, sha1_hash):
                return file_path
            else:
                print('Mismatch in the content of model file {} detected.' +
                      ' Downloading again.'.format(file_path))
        else:
            print('Model file {} is not found. Downloading.'.format(file_path))

        zip_file_path = os.path.join(root, file_name+'.zip')
        repo_url = os.environ.get('ENCODING_REPO', encoding_repo_url)
        if repo_url[-1] != '/':
            repo_url = repo_url + '/'
        download(_url_format.format(repo_url=repo_url, file_name=file_name),
                 path=zip_file_path,
                 overwrite=True)
        with zipfile.ZipFile(zip_file_path) as zf:
            zf.extractall(root)
        os.remove(zip_file_path)

        if check_sha1(file_path, sha1_hash):
            return file_path
        else:
            raise ValueError('Downloaded file has different hash. Please try again.')