本文整理汇总了Python中pymesos.MesosSchedulerDriver.join方法的典型用法代码示例。如果您正苦于以下问题：Python MesosSchedulerDriver.join方法的具体用法？Python MesosSchedulerDriver.join怎么用？Python MesosSchedulerDriver.join使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类pymesos.MesosSchedulerDriver的用法示例。
在下文中一共展示了MesosSchedulerDriver.join方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: TFMesosScheduler
# 需要导入模块: from pymesos import MesosSchedulerDriver [as 别名]
# 或者: from pymesos.MesosSchedulerDriver import join [as 别名]
class TFMesosScheduler(Scheduler):
MAX_FAILURE_COUNT = 3
def __init__(self, task_spec, role=None, master=None, name=None,
quiet=False, volumes={}, containerizer_type=None,
force_pull_image=False, forward_addresses=None,
protocol='grpc', env={}, extra_config={}):
self.started = False
self.master = master or os.environ['MESOS_MASTER']
self.name = name or '[tensorflow] %s %s' % (
os.path.abspath(sys.argv[0]), ' '.join(sys.argv[1:]))
self.task_spec = task_spec
self.containerizer_type = containerizer_type
self.force_pull_image = force_pull_image
self.protocol = protocol
self.extra_config = extra_config
self.forward_addresses = forward_addresses
self.role = role or '*'
self.tasks = {}
self.task_failure_count = {}
self.job_finished = {}
for job in task_spec:
self.job_finished[job.name] = 0
for task_index in range(job.start, job.num):
mesos_task_id = str(uuid.uuid4())
task = Task(
mesos_task_id,
job.name,
task_index,
cpus=job.cpus,
mem=job.mem,
gpus=job.gpus,
cmd=job.cmd,
volumes=volumes,
env=env
)
self.tasks[mesos_task_id] = task
self.task_failure_count[self.decorated_task_index(task)] = 0
if not quiet:
global logger
setup_logger(logger)
def resourceOffers(self, driver, offers):
'''
Offer resources and launch tasks
'''
for offer in offers:
if all(task.offered for id, task in iteritems(self.tasks)):
self.driver.suppressOffers()
driver.declineOffer(offer.id, Dict(refuse_seconds=FOREVER))
continue
offered_cpus = offered_mem = 0.0
offered_gpus = []
offered_tasks = []
gpu_resource_type = None
for resource in offer.resources:
if resource.name == 'cpus':
offered_cpus = resource.scalar.value
elif resource.name == 'mem':
offered_mem = resource.scalar.value
elif resource.name == 'gpus':
if resource.type == 'SET':
offered_gpus = resource.set.item
else:
offered_gpus = list(range(int(resource.scalar.value)))
gpu_resource_type = resource.type
for id, task in iteritems(self.tasks):
if task.offered:
continue
if not (task.cpus <= offered_cpus and
task.mem <= offered_mem and
task.gpus <= len(offered_gpus)):
continue
offered_cpus -= task.cpus
offered_mem -= task.mem
gpus = int(math.ceil(task.gpus))
gpu_uuids = offered_gpus[:gpus]
offered_gpus = offered_gpus[gpus:]
task.offered = True
offered_tasks.append(
task.to_task_info(
offer, self.addr, gpu_uuids=gpu_uuids,
gpu_resource_type=gpu_resource_type,
containerizer_type=self.containerizer_type,
force_pull_image=self.force_pull_image
)
)
driver.launchTasks(offer.id, offered_tasks)
@property
#.........这里部分代码省略.........
示例2: spawn_rconsole
# 需要导入模块: from pymesos import MesosSchedulerDriver [as 别名]
# 或者: from pymesos.MesosSchedulerDriver import join [as 别名]
signal.signal(signal.SIGABRT, handler)
signal.signal(signal.SIGQUIT, handler)
spawn_rconsole(locals())
try:
driver.start()
sched.run(driver)
except KeyboardInterrupt:
logger.warning('stopped by KeyboardInterrupt')
sched.stop(EXIT_KEYBORAD)
except Exception as e:
import traceback
logger.warning('catch unexpected Exception, exit now. %s',
traceback.format_exc())
sched.stop(EXIT_EXCEPTION)
finally:
try:
sched.dump_stats()
except:
logger.exception("dump stats fail, ignore it.")
# sched.lock may be in WRONG status.
# if any thread of sched may use lock or call driver, join it first
driver.stop(False)
driver.join()
# mesos resourses are released, and no racer for lock any more
sched.cleanup()
ctx.term()
sys.exit(sched.ec)