Python filelock.FileLock方法代码示例

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def __init__(
            self,
            seed=0,
            episode_len=None,
            no_images=None
    ):
        from tensorflow.examples.tutorials.mnist import input_data
        # we could use temporary directory for this with a context manager and 
        # TemporaryDirecotry, but then each test that uses mnist would re-download the data
        # this way the data is not cleaned up, but we only download it once per machine
        mnist_path = osp.join(tempfile.gettempdir(), 'MNIST_data')
        with filelock.FileLock(mnist_path + '.lock'):
           self.mnist = input_data.read_data_sets(mnist_path)

        self.np_random = np.random.RandomState()
        self.np_random.seed(seed)

        self.observation_space = Box(low=0.0, high=1.0, shape=(28,28,1))
        self.action_space = Discrete(10)
        self.episode_len = episode_len
        self.time = 0
        self.no_images = no_images

        self.train_mode()
        self.reset() 

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def _load_frozen_paths():
    library_directory = config.get('MINEMELD_LOCAL_LIBRARY_PATH', None)
    if library_directory is None:
        LOG.error('freeze not updated - MINEMELD_LOCAL_LIBRARY_PATH not set')
        return

    freeze_path = os.path.join(library_directory, 'freeze.txt')

    if not os.path.isfile(freeze_path):
        LOG.info('Extensions frigidaire not found, paths not loaded')
        return

    freeze_lock = filelock.FileLock('{}.lock'.format(freeze_path))
    with freeze_lock.acquire(timeout=30):
        with open(freeze_path, 'r') as ff:
            minemeld.extensions.load_frozen_paths(ff) 

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def read(self):
        fdfname = self.datafilename+'.yml'

        lockfname = os.path.join(self.cpath, fdfname+'.lock')
        lock = filelock.FileLock(lockfname)

        os.listdir(self.cpath)
        if fdfname not in os.listdir(self.cpath):
            return jsonify(error={
                'message': 'Unknown config data file'
            }), 400

        try:
            with lock.acquire(timeout=10):
                with open(os.path.join(self.cpath, fdfname), 'r') as f:
                    result = yaml.safe_load(f)

        except Exception as e:
            return jsonify(error={
                'message': 'Error loading config data file: %s' % str(e)
            }), 500

        return jsonify(result=result) 

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def create(self):
        tdir = os.path.dirname(os.path.join(self.cpath, self.datafilename))

        if not os.path.samefile(self.cpath, tdir):
            return jsonify(error={'msg': 'Wrong config data filename'}), 400

        fdfname = os.path.join(self.cpath, self.datafilename+'.yml')

        lockfname = fdfname+'.lock'
        lock = filelock.FileLock(lockfname)

        try:
            body = request.get_json()
        except Exception as e:
            return jsonify(error={'message': str(e)}), 400

        try:
            with lock.acquire(timeout=10):
                with open(fdfname, 'w') as f:
                    yaml.safe_dump(body, stream=f)
        except Exception as e:
            return jsonify(error={
                'message': str(e)
            }), 500 

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def init():
    global API_CONFIG_PATH
    global API_CONFIG_LOCK

    config_path = os.environ.get('MM_CONFIG', None)
    if config_path is None:
        LOG.critical('MM_CONFIG environment variable not set')
        raise RuntimeError('MM_CONFIG environment variable not set')

    if not os.path.isdir(config_path):
        config_path = os.path.dirname(config_path)

    # init global vars
    API_CONFIG_PATH = os.path.join(config_path, 'api')
    API_CONFIG_LOCK = filelock.FileLock(
        os.environ.get('API_CONFIG_LOCK', '/var/run/minemeld/api-config.lock')
    )

    _load_config(config_path)
    _load_auth_dbs(config_path)
    if config_path is not None:
        gevent.spawn(_config_monitor, config_path) 

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def save(self, out, timeout):
        filepath = os.path.join(out, self.filename)
        lockpath = filepath + '.lock'

        try:
            with filelock.FileLock(lockpath, timeout=timeout):
                saved_assets_list = []
                if os.path.isfile(filepath):
                    with open(filepath) as f:
                        saved_assets_list = json.load(f)
                saved_assets_list.extend(self.cache[self.saved_idx:])
                with open(filepath, 'w') as f:
                    json.dump(saved_assets_list, f, indent=4)
                self.saved_idx = len(self.cache)
        except filelock.Timeout:
            logger.error('Process to write a list of assets is timeout') 

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def get_lock():
    """Create a filelock"""

    # The run() method uses a file lock in the user's ~/.resilient directory to prevent multiple instances
    # of resilient circuits running.  You can override the lockfile name in the
    # (and so allow multiple) by setting APP_LOCK_FILE in the environment.
    app_lock_file = os.environ.get("APP_LOCK_FILE", "")

    if not app_lock_file:
        lockfile = os.path.expanduser(os.path.join("~", ".resilient", "resilient_circuits_lockfile"))
        resilient_dir = os.path.dirname(lockfile)
        if not os.path.exists(resilient_dir):
            os.makedirs(resilient_dir)
    else:
        lockfile = os.path.expanduser(app_lock_file)
    lock = filelock.FileLock(lockfile)
    return lock 

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def get_sbd():
    # To support ChainerMN, the target directory should be locked.
    with filelock.FileLock(os.path.join(download.get_dataset_directory(
            'pfnet/chainercv/.lock'), 'sbd.lock')):
        data_root = download.get_dataset_directory(root)
        base_path = os.path.join(data_root, 'benchmark_RELEASE/dataset')

        train_voc2012_file = os.path.join(base_path, 'train_voc2012.txt')
        if os.path.exists(train_voc2012_file):
            # skip downloading
            return base_path

        download_file_path = utils.cached_download(url)
        ext = os.path.splitext(url)[1]
        utils.extractall(download_file_path, data_root, ext)

        six.moves.urllib.request.urlretrieve(
            train_voc2012_url, train_voc2012_file)
        _generate_voc2012_txt(base_path)

    return base_path 

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def get_voc(year, split):
    if year not in urls:
        raise ValueError
    key = year

    if split == 'test' and year == '2007':
        key = '2007_test'

    # To support ChainerMN, the target directory should be locked.
    with filelock.FileLock(os.path.join(download.get_dataset_directory(
            'pfnet/chainercv/.lock'), 'voc.lock')):
        data_root = download.get_dataset_directory(root)
        base_path = os.path.join(data_root, 'VOCdevkit/VOC{}'.format(year))
        split_file = os.path.join(
            base_path, 'ImageSets/Main/{}.txt'.format(split))
        if os.path.exists(split_file):
            # skip downloading
            return base_path

        download_file_path = utils.cached_download(urls[key])
        ext = os.path.splitext(urls[key])[1]
        utils.extractall(download_file_path, data_root, ext)
    return base_path 

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def experiment_id(kfp_client, tmp_path_factory, worker_id):
    if not worker_id:
        return get_experiment_id(kfp_client)

    # Locking taking as an example from
    # https://github.com/pytest-dev/pytest-xdist#making-session-scoped-fixtures-execute-only-once
    # get the temp directory shared by all workers
    root_tmp_dir = tmp_path_factory.getbasetemp().parent

    fn = root_tmp_dir / "experiment_id"
    with FileLock(str(fn) + ".lock"):
        if fn.is_file():
            data = fn.read_text()
        else:
            data = get_experiment_id(kfp_client)
            fn.write_text(data)
    return data 

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def setup(self, config):
        use_cuda = config.get("use_gpu") and torch.cuda.is_available()
        self.device = torch.device("cuda" if use_cuda else "cpu")
        self.netD = Discriminator().to(self.device)
        self.netD.apply(weights_init)
        self.netG = Generator().to(self.device)
        self.netG.apply(weights_init)
        self.criterion = nn.BCELoss()
        self.optimizerD = optim.Adam(
            self.netD.parameters(),
            lr=config.get("lr", 0.01),
            betas=(beta1, 0.999))
        self.optimizerG = optim.Adam(
            self.netG.parameters(),
            lr=config.get("lr", 0.01),
            betas=(beta1, 0.999))
        with FileLock(os.path.expanduser("~/.data.lock")):
            self.dataloader = get_data_loader() 

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def get_data_loaders():
    mnist_transforms = transforms.Compose(
        [transforms.ToTensor(),
         transforms.Normalize((0.1307, ), (0.3081, ))])

    # We add FileLock here because multiple workers will want to
    # download data, and this may cause overwrites since
    # DataLoader is not threadsafe.
    with FileLock(os.path.expanduser("~/data.lock")):
        train_loader = torch.utils.data.DataLoader(
            datasets.MNIST(
                "~/data",
                train=True,
                download=True,
                transform=mnist_transforms),
            batch_size=64,
            shuffle=True)
    test_loader = torch.utils.data.DataLoader(
        datasets.MNIST("~/data", train=False, transform=mnist_transforms),
        batch_size=64,
        shuffle=True)
    return train_loader, test_loader 

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def get_data_loader():
    """Safely downloads data. Returns training/validation set dataloader."""
    mnist_transforms = transforms.Compose(
        [transforms.ToTensor(),
         transforms.Normalize((0.1307, ), (0.3081, ))])

    # We add FileLock here because multiple workers will want to
    # download data, and this may cause overwrites since
    # DataLoader is not threadsafe.
    with FileLock(os.path.expanduser("~/data.lock")):
        train_loader = torch.utils.data.DataLoader(
            datasets.MNIST(
                "~/data",
                train=True,
                download=True,
                transform=mnist_transforms),
            batch_size=128,
            shuffle=True)
        test_loader = torch.utils.data.DataLoader(
            datasets.MNIST("~/data", train=False, transform=mnist_transforms),
            batch_size=128,
            shuffle=True)
    return train_loader, test_loader 

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def __init__(
            self,
            episode_len=None,
            no_images=None
    ):
        import filelock
        from tensorflow.examples.tutorials.mnist import input_data
        # we could use temporary directory for this with a context manager and
        # TemporaryDirecotry, but then each test that uses mnist would re-download the data
        # this way the data is not cleaned up, but we only download it once per machine
        mnist_path = osp.join(tempfile.gettempdir(), 'MNIST_data')
        with filelock.FileLock(mnist_path + '.lock'):
           self.mnist = input_data.read_data_sets(mnist_path)

        self.np_random = np.random.RandomState()

        self.observation_space = Box(low=0.0, high=1.0, shape=(28,28,1))
        self.action_space = Discrete(10)
        self.episode_len = episode_len
        self.time = 0
        self.no_images = no_images

        self.train_mode()
        self.reset() 

# 需要导入模块: import filelock [as 别名]
# 或者: from filelock import FileLock [as 别名]
def worker_finished(worker_ip):
    """
    The scheduler received a notification indicating an indexing worker has
    finished its task.
    :param worker_ip: The private IP of the worker.
    :return: The target index if all workers are finished, else False.
    """
    with FileLock('lock'), shelve.open('db', writeback=True) as db:
        try:
            _ = db['worker_statuses'][worker_ip]
            db['worker_statuses'][worker_ip] = WorkerStatus.FINISHED
            log.info(f'Received worker_finished signal from {worker_ip}')
        except KeyError:
            log.error(
                'An indexer worker notified us it finished its task, but '
                'we are not tracking it.'
            )
        for worker_key in db['worker_statuses']:
            if db['worker_statuses'][worker_key] == WorkerStatus.RUNNING:
                log.info(f'{worker_key} is still indexing')
                return False
        return db['target_index']