本文整理汇总了Python中horovod.tensorflow.size方法的典型用法代码示例。如果您正苦于以下问题:Python tensorflow.size方法的具体用法?Python tensorflow.size怎么用?Python tensorflow.size使用的例子?那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类horovod.tensorflow
的用法示例。
在下文中一共展示了tensorflow.size方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: train_input_fn
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def train_input_fn(input_file, _parse_fn, name_to_features,
params, **kargs):
if_shard = kargs.get("if_shard", "1")
dataset = tf.data.TFRecordDataset(input_file, buffer_size=params.get("buffer_size", 100))
print("==hvd size {}, rank {}==".format(hvd.size(), hvd.rank()))
if if_shard == "1":
dataset = dataset.shard(hvd.size(), hvd.rank())
dataset = dataset.map(lambda x:_parse_fn(x, name_to_features))
dataset = dataset.shuffle(
buffer_size=params.get("buffer_size", 1024)+3*params.get("batch_size", 32),
seed=np.random.randint(0,1e10,1)[0],
reshuffle_each_iteration=True)
dataset = dataset.batch(params.get("batch_size", 32))
dataset = dataset.repeat(params.get("epoch", 100))
iterator = dataset.make_one_shot_iterator()
features = iterator.get_next()
return features
示例2: get_train_op
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def get_train_op(self, loss, tvars, init_lr,
num_train_steps, **kargs):
learning_rate = self.lr_decay_fn(init_lr, num_train_steps, **kargs)
learning_rate = self.warm_up(learning_rate, init_lr, **kargs)
print("==optimizer hvd size=={}".format(hvd.size()))
opt = self.optimizer_op(learning_rate*hvd.size(), **kargs)
# add uber horvod distributed optimizer
self.opt = hvd.DistributedOptimizer(opt)
grads = self.grad_clip_fn(self.opt, loss, tvars, **kargs)
# self.grad_summaries_merged = optimizer_utils.add_grad_summaries(
# zip(grads, tvars))
train_op = self.opt.apply_gradients(
zip(grads, tvars), global_step=self.global_step)
new_global_step = self.global_step + 1
train_op = tf.group(train_op, [self.global_step.assign(new_global_step)])
return train_op
示例3: get_gradients
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def get_gradients(self, loss, params):
"""
Compute gradients of all trainable variables.
See Optimizer.get_gradients() for more info.
In DistributedOptimizer, get_gradients() is overriden to also
allreduce the gradients before returning them.
"""
gradients = super(self.__class__, self).get_gradients(loss, params)
if hvd.size() > 1:
averaged_gradients = []
with tf.name_scope(self._name + "_Allreduce"):
for grad in gradients:
if grad is not None:
avg_grad = hvd.allreduce(grad, device_dense=self._device_dense,
device_sparse=self._device_sparse)
averaged_gradients.append(avg_grad)
else:
averaged_gradients.append(None)
return averaged_gradients
else:
return gradients
示例4: test_horovod_allreduce_type_error
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def test_horovod_allreduce_type_error(self):
"""Test that the allreduce raises an error if different ranks try to
send tensors of different type."""
hvd.init()
rank = hvd.rank()
size = hvd.size()
# This test does not apply if there is only one worker.
if size == 1:
return
with self.test_session(config=self.config) as session:
# Same rank, different dimension
dims = [17] * 3
tensor = tf.ones(dims,
dtype=tf.int32 if rank % 2 == 0 else tf.float32)
with self.assertRaises(tf.errors.FailedPreconditionError):
session.run(hvd.allreduce(tensor))
示例5: test_horovod_allreduce_cpu_gpu_error
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def test_horovod_allreduce_cpu_gpu_error(self):
"""Test that the allreduce raises an error if different ranks try to
perform reduction on CPU and GPU."""
# Only do this test if there are GPUs available.
if not tf.test.is_gpu_available(cuda_only=True):
return
hvd.init()
local_rank = hvd.local_rank()
size = hvd.size()
# This test does not apply if there is only one worker.
if size == 1:
return
device = "/gpu:%d" % local_rank if local_rank % 2 == 0 else "/cpu:0"
with self.test_session(config=self.config) as session:
with tf.device(device):
# Same rank, different dimension
dims = [17] * 3
tensor = tf.ones(dims, dtype=tf.int32)
with self.assertRaises(tf.errors.FailedPreconditionError):
session.run(hvd.allreduce(tensor))
示例6: test_horovod_allgather_error
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def test_horovod_allgather_error(self):
"""Test that the allgather returns an error if any dimension besides
the first is different among the tensors being gathered."""
hvd.init()
rank = hvd.rank()
size = hvd.size()
# This test does not apply if there is only one worker.
if size == 1:
return
with self.test_session(config=self.config) as session:
tensor_size = [17] * 3
tensor_size[1] = 10 * (rank + 1)
tensor = tf.ones(tensor_size, dtype=tf.float32) * rank
with self.assertRaises(tf.errors.FailedPreconditionError):
session.run(hvd.allgather(tensor))
示例7: test_horovod_allgather_type_error
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def test_horovod_allgather_type_error(self):
"""Test that the allgather returns an error if the types being gathered
differ among the processes"""
hvd.init()
rank = hvd.rank()
size = hvd.size()
# This test does not apply if there is only one worker.
if size == 1:
return
with self.test_session(config=self.config) as session:
tensor_size = [17] * 3
dtype = tf.int32 if rank % 2 == 0 else tf.float32
tensor = tf.ones(tensor_size, dtype=dtype) * rank
with self.assertRaises(tf.errors.FailedPreconditionError):
session.run(hvd.allgather(tensor))
示例8: test_horovod_broadcast_error
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def test_horovod_broadcast_error(self):
"""Test that the broadcast returns an error if any dimension besides
the first is different among the tensors being broadcasted."""
hvd.init()
rank = hvd.rank()
size = hvd.size()
# This test does not apply if there is only one worker.
if size == 1:
return
with self.test_session(config=self.config) as session:
tensor_size = [17] * 3
tensor_size[1] = 10 * (rank + 1)
tensor = tf.ones(tensor_size, dtype=tf.float32) * rank
with self.assertRaises(tf.errors.FailedPreconditionError):
session.run(hvd.broadcast(tensor, 0))
示例9: setup
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def setup():
if not horovod_installed:
return False
global horovod_initialized
if horovod_initialized:
return hvd
hvd.init()
horovod_initialized = True
horovod_num_worker = hvd.size()
horovod_rank = hvd.rank()
# verify that MPI multi-threading is supported.
assert hvd.mpi_threads_supported()
# make sure MPI is not re-initialized.
import mpi4py.rc
mpi4py.rc.initialize = False
# import mpi4py
from mpi4py import MPI
comm = MPI.COMM_WORLD
# check size and rank are synchronized
assert horovod_num_worker == comm.Get_size()
assert horovod_rank == comm.Get_rank()
return hvd
示例10: _log_summary
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def _log_summary(total_images, batch_size, duration):
logger = logging.getLogger(__name__)
images_per_second = total_images / duration
logger.info("Data length: {}".format(total_images))
logger.info("Total duration: {:.3f}".format(duration))
logger.info("Total images/sec: {:.3f}".format(images_per_second))
logger.info(
"Batch size: (Per GPU {}: Total {})".format(
batch_size, hvd.size() * batch_size if defaults.DISTRIBUTED else batch_size
)
)
logger.info(
"Distributed: {}".format("True" if defaults.DISTRIBUTED else "False")
)
logger.info(
"Num GPUs: {:.3f}".format(hvd.size() if defaults.DISTRIBUTED else 1)
)
示例11: setup_horovod
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def setup_horovod():
import horovod.tensorflow as hvd
# Initialize Horovod
hvd.init()
# Verify that MPI multi-threading is supported.
assert hvd.mpi_threads_supported()
from mpi4py import MPI
assert hvd.size() == MPI.COMM_WORLD.Get_size()
is_root = hvd.rank() == 0
def mpi_average(local_list):
# _local_list_orig = local_list
local_list = list(map(float, local_list))
# print('RANK {} AVERAGING {} -> {}'.format(hvd.rank(), _local_list_orig, local_list))
sums = MPI.COMM_WORLD.gather(sum(local_list), root=0)
counts = MPI.COMM_WORLD.gather(len(local_list), root=0)
sum_counts = sum(counts) if is_root else None
avg = (sum(sums) / sum_counts) if is_root else None
return avg, sum_counts
return hvd, MPI, is_root, mpi_average
示例12: _train
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def _train(model: Model,
data: TrainingData,
checkpoint: Union[str, None],
parameter_checkpoint: Union[str, None],
save_start: bool,
train_params: trainer.TrainParams,
evaluators: List[Evaluator],
out: ModelDir,
notes=None,
dry_run=False,
start_eval=False):
print('Horovod size: ', hvd.size())
print('Horovod rank: ', hvd.rank())
print('Horovod local rank: ', hvd.local_rank())
if train_params.async_encoding:
_train_async(model, data, checkpoint, parameter_checkpoint, save_start, train_params,
evaluators, out, notes, dry_run, start_eval)
return
else:
raise NotImplementedError('Syncronous training with Horovod not supported yet')
示例13: get_training_params
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def get_training_params(train_config):
return TrainParams(
SerializableOptimizer(
train_config.optimizer,
dict(learning_rate=train_config.learning_rate * hvd.size())
),
num_epochs=train_config.n_epochs,
ema=train_config.ema,
max_checkpoints_to_keep=train_config.max_checkpoints_to_keep,
async_encoding=train_config.async_encoding,
log_period=train_config.log_period,
eval_period=train_config.eval_period,
save_period=train_config.save_period,
best_weights=("dev", "b8/question-text-f1"),
eval_samples=dict(dev=None, train=6000),
eval_at_zero=False
)
示例14: main
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def main(_):
if not FLAGS.vocab_size:
FLAGS.vocab_size = len(open(FLAGS.vocab_file).readlines())
if not FLAGS.num_labels:
FLAGS.num_labels = len(open(FLAGS.label_file).readlines())
if FLAGS.horovod:
nproc = hvd.size()
total = FLAGS.train_steps
FLAGS.train_steps = total / nproc
print("Running %d steps on each of %d processes for %d total" % (
FLAGS.train_steps, nproc, total))
FastTrain()
示例15: adam2_old
# 需要导入模块: from horovod import tensorflow [as 别名]
# 或者: from horovod.tensorflow import size [as 别名]
def adam2_old(params, cost_or_grads, lr=3e-4, mom1=0.9, mom2=0.999, epsilon=1e-8):
updates = []
if type(cost_or_grads) is not list:
gs = tf.gradients(cost_or_grads, params)
else:
gs = cost_or_grads
# all-reduce
grads1 = [Z.allreduce_mean(g) for g in gs]
grads2 = [Z.allreduce_mean(tf.square(g)) for g in gs]
mom2 = tf.maximum(0., 1. - (hvd.size() * (1 - mom2)))
t = tf.Variable(1., 'adam_t')
lr_t = lr * tf.sqrt((1. - tf.pow(mom2, t))) / (1. - tf.pow(mom1, t))
updates.append(t.assign_add(1))
for p, g1, g2 in zip(params, grads1, grads2):
mg = tf.Variable(tf.zeros(p.get_shape()), p.name + '_adam_mg')
if mom1 > 0:
v = tf.Variable(tf.zeros(p.get_shape()), p.name + '_adam_v')
v_t = mom1 * v + (1. - mom1) * g1
updates.append(v.assign(v_t))
else:
v_t = g1
mg_t = mom2 * mg + (1. - mom2) * g2
delta_t = v_t / (tf.sqrt(mg_t) + epsilon)
p_t = p - lr_t * delta_t
updates.append(mg.assign(mg_t))
updates.append(p.assign(p_t))
return tf.group(*updates)