Python gfile.Exists方法代码示例

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def main(unused_argv):
  request = inference_flags.request_from_flags()

  if not gfile.Exists(request.segmentation_output_dir):
    gfile.MakeDirs(request.segmentation_output_dir)

  bbox = bounding_box_pb2.BoundingBox()
  text_format.Parse(FLAGS.bounding_box, bbox)

  runner = inference.Runner()
  runner.start(request)
  runner.run((bbox.start.z, bbox.start.y, bbox.start.x),
             (bbox.size.z, bbox.size.y, bbox.size.x))

  counter_path = os.path.join(request.segmentation_output_dir, 'counters.txt')
  if not gfile.Exists(counter_path):
    runner.counters.dump(counter_path) 

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def get_meta_filename(self, start_new_model, train_dir):
    if start_new_model:
      logging.info("%s: Flag 'start_new_model' is set. Building a new model.",
                   task_as_string(self.task))
      return None
    
    latest_checkpoint = tf.train.latest_checkpoint(train_dir)
    if not latest_checkpoint: 
      logging.info("%s: No checkpoint file found. Building a new model.",
                   task_as_string(self.task))
      return None
    
    meta_filename = latest_checkpoint + ".meta"
    if not gfile.Exists(meta_filename):
      logging.info("%s: No meta graph file found. Building a new model.",
                     task_as_string(self.task))
      return None
    else:
      return meta_filename 

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def get_meta_filename(self, start_new_model, train_dir):
        if start_new_model:
            logging.info("%s: Flag 'start_new_model' is set. Building a new model.",
                         task_as_string(self.task))
            return None

        latest_checkpoint = tf.train.latest_checkpoint(train_dir)
        if not latest_checkpoint:
            logging.info("%s: No checkpoint file found. Building a new model.",
                         task_as_string(self.task))
            return None

        meta_filename = latest_checkpoint + ".meta"
        if not gfile.Exists(meta_filename):
            logging.info("%s: No meta graph file found. Building a new model.",
                         task_as_string(self.task))
            return None
        else:
            return meta_filename 

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def get_meta_filename(self, start_new_model, train_dir):
    if start_new_model:
      logging.info("%s: Flag 'start_new_model' is set. Building a new model.",
                   task_as_string(self.task))
      return None

    latest_checkpoint = tf.train.latest_checkpoint(train_dir)
    if not latest_checkpoint:
      logging.info("%s: No checkpoint file found. Building a new model.",
                   task_as_string(self.task))
      return None

    meta_filename = latest_checkpoint + ".meta"
    if not gfile.Exists(meta_filename):
      logging.info("%s: No meta graph file found. Building a new model.",
                     task_as_string(self.task))
      return None
    else:
      return meta_filename 

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def get_meta_filename(self, start_new_model, train_dir):
    if start_new_model:
      logging.info("%s: Flag 'start_new_model' is set. Building a new model.",
                   task_as_string(self.task))
      return None

    latest_checkpoint = tf.train.latest_checkpoint(train_dir)
    if not latest_checkpoint:
      logging.info("%s: No checkpoint file found. Building a new model.",
                   task_as_string(self.task))
      return None

    meta_filename = latest_checkpoint + ".meta"
    if not gfile.Exists(meta_filename):
      logging.info("%s: No meta graph file found. Building a new model.",
                   task_as_string(self.task))
      return None
    else:
      return meta_filename 

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def count_file(self, path, verbose=False, add_eos=False):
        if verbose: print('counting file {} ...'.format(path))
        assert exists(path)

        sents = []
        with open(path, 'r') as f:
            for idx, line in enumerate(f):
                if verbose and idx > 0 and idx % 500000 == 0:
                    print('  line {}'.format(idx))
                symbols = self.tokenize(line, add_eos=True)
                self.counter.update(symbols)
                sents.append(symbols)

        return sents

    # 更新counter 中的token 

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def encode_file(self, path, ordered=False, verbose=False,
                    add_double_eos=False):
        if verbose: print('encoding file {} ...'.format(path))
        assert exists(path)
        encoded = []
        with open(path, 'r') as f:
            for idx, line in enumerate(f):
                if verbose and idx > 0 and idx % 500000 == 0:
                    print('  line {}'.format(idx))
                symbols = self.tokenize(line, add_eos=True, add_double_eos=add_double_eos)

                encoded.append(self.convert_to_nparray(symbols))

        if ordered:
            encoded = np.concatenate(encoded)

        return encoded

    # 

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def main(unused_argv):
    del unused_argv  # Unused

    corpus = get_lm_corpus(FLAGS.data_dir, FLAGS.dataset)  #

    save_dir = os.path.join(FLAGS.data_dir, "tfrecords")
    if not exists(save_dir):
        makedirs(save_dir)

    # test mode
    if FLAGS.per_host_test_bsz > 0:
        corpus.convert_to_tfrecords("test", save_dir, FLAGS.per_host_test_bsz,
                                    FLAGS.tgt_len, FLAGS.num_core_per_host,
                                    FLAGS=FLAGS)
        return

    for split, batch_size in zip(
            ["train", "valid"],
            [FLAGS.per_host_train_bsz, FLAGS.per_host_valid_bsz]):

        if batch_size <= 0: continue
        print("Converting {} set...".format(split))
        corpus.convert_to_tfrecords(split, save_dir, batch_size, FLAGS.tgt_len,
                                    FLAGS.num_core_per_host, FLAGS=FLAGS) 

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def count_file(self, path, verbose=False, add_eos=False):
        if verbose: print('counting file {} ...'.format(path))
        assert exists(path)

        sents = []
        with open(path, 'r') as f:
            for idx, line in enumerate(f):
                if verbose and idx > 0 and idx % 500000 == 0:
                    print('  line {}'.format(idx))
                symbols = self.tokenize(line, add_eos=add_eos)
                self.counter.update(symbols)
                sents.append(symbols)

        return sents

    # 更新counter 中的token 

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def encode_file(self, path, ordered=False, verbose=False, add_eos=True,
          add_double_eos=False):
    if verbose: print('encoding file {} ...'.format(path))
    assert exists(path)
    encoded = []
    with open(path, 'r') as f:
      for idx, line in enumerate(f):
        if verbose and idx > 0 and idx % 500000 == 0:
          print('  line {}'.format(idx))
        symbols = self.tokenize(line, add_eos=add_eos,
          add_double_eos=add_double_eos)
        encoded.append(self.convert_to_nparray(symbols))

    if ordered:
      encoded = np.concatenate(encoded)

    return encoded 

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def main(unused_argv):
  del unused_argv  # Unused

  corpus = get_lm_corpus(FLAGS.data_dir, FLAGS.dataset)

  save_dir = os.path.join(FLAGS.data_dir, "tfrecords")
  if not exists(save_dir):
    makedirs(save_dir)

  # test mode
  if FLAGS.per_host_test_bsz > 0:
    corpus.convert_to_tfrecords("test", save_dir, FLAGS.per_host_test_bsz,
                                FLAGS.tgt_len, FLAGS.num_core_per_host, 
                                FLAGS=FLAGS)
    return

  for split, batch_size in zip(
      ["train", "valid"],
      [FLAGS.per_host_train_bsz, FLAGS.per_host_valid_bsz]):

    if batch_size <= 0: continue
    print("Converting {} set...".format(split))
    corpus.convert_to_tfrecords(split, save_dir, batch_size, FLAGS.tgt_len,
                                FLAGS.num_core_per_host, FLAGS=FLAGS) 

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def main(_argv):
    # load flags from config file
    model_configs = load_from_config_path(FLAGS.config_paths)
    # replace parameters in configs_file with tf FLAGS
    model_configs = update_configs_from_flags(model_configs, FLAGS, TRAIN_ARGS.keys())
    model_dir = model_configs["model_dir"]
    if not gfile.Exists(model_dir):
        gfile.MakeDirs(model_dir)

    if "CUDA_VISIBLE_DEVICES" not in os.environ.keys():
        raise OSError("need CUDA_VISIBLE_DEVICES environment variable")
    tf.logging.info("CUDA_VISIBLE_DEVICES={}".format(os.environ["CUDA_VISIBLE_DEVICES"]))

    training_runner = TrainingExperiment(
        model_configs=model_configs)

    training_runner.run() 

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def access_multiple_files(name):
    """ Gets the list of files.

    Args:
        name: A string, the prefix of the files.

    Returns: A list or None.
    """
    assert name
    ret = []
    if gfile.Exists(name):
        ret.append(name)
    else:
        idx = 0
        while gfile.Exists(name + str(idx)):
            ret.append(name + str(idx))
            idx += 1
    assert len(ret) > 0, (
        "Fail to access file {} or {}0...".format(name, name))
    return ret 

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def load_from_config_path(config_paths):
    """ Loads configurations from files of yaml format.

    Args:
        config_paths: A string (each file name is seperated by ",") or
          a list of strings (file names).

    Returns: A dictionary of model configurations, parsed from config files.
    """
    if isinstance(config_paths, six.string_types):
        config_paths = config_paths.strip().split(",")
    assert isinstance(config_paths, list) or isinstance(config_paths, tuple)
    model_configs = dict()
    for config_path in config_paths:
        config_path = config_path.strip()
        if not config_path:
            continue
        if not gfile.Exists(config_path):
            raise OSError("config file does not exist: {}".format(config_path))
        config_path = os.path.abspath(config_path)
        tf.logging.info("loading configurations from {}".format(config_path))
        with open_file(config_path, mode="r") as config_file:
            config_flags = yaml.load(config_file)
            model_configs = deep_merge_dict(model_configs, config_flags)
    return model_configs 

# 需要导入模块: from tensorflow import gfile [as 别名]
# 或者: from tensorflow.gfile import Exists [as 别名]
def get_existing_subvolume_path(segmentation_dir, corner, allow_cpoint=False):
  """Returns the path to an existing FFN subvolume.

  This like `get_subvolume_path`, but returns paths to existing data only.

  Args:
    segmentation_dir: directory containing FFN subvolumes
    corner: lower corner of the FFN subvolume as a (z, y, x) tuple
    allow_cpoint: whether to return a checkpoint path in case the final
        segmentation is not ready

  Returns:
    path to an existing FFN subvolume (string) or None if no such subvolume
    is found
  """
  target_path = segmentation_path(segmentation_dir, corner)
  if gfile.Exists(target_path):
    return target_path

  target_path = legacy_segmentation_path(segmentation_dir, corner)
  if gfile.Exists(target_path):
    return target_path

  if allow_cpoint:
    target_path = checkpoint_path(segmentation_dir, corner)
    if gfile.Exists(target_path):
      return target_path

  return None