Python EmrConnection.add_tags方法代码示例

# 需要导入模块: from boto.emr.connection import EmrConnection [as 别名]
# 或者: from boto.emr.connection.EmrConnection import add_tags [as 别名]
class EMRCluster(object):
  '''Representation of an EMR cluster.
     TODO: add bridge to boto interface for unit test.
  '''
  emr_status_delay = 10      # in sec
  emr_status_max_delay = 60  # in sec
  emr_status_max_error = 30  # number of errors
  emr_max_idle = 10 * 60     # 10 min (in sec)
  rate_limit_lock = RateLimitLock()

  def __init__(self, prop):
    '''Constructor, initialize EMR connection.'''
    self.prop = prop
    self.conn = EmrConnection(self.prop.ec2.key, self.prop.ec2.secret)
    self.jobid = None
    self.retry = 0
    self.level = 0
    self.last_update = -1

  @property
  def priority(self):
    '''The priority used in EMRManager.
       The lower value, the higher priority.
    '''
    with EMRCluster.rate_limit_lock:
      if self.jobid is None:
        return 1
      return 0

  def get_instance_groups(self):
    '''Get instance groups to start a cluster.
       It calculates the price with self.level, which indicates the
       price upgrades from the original price.
    '''
    instance_groups = []
    for group in self.prop.emr.instance_groups:
      (num, group_name, instance_type) = group
      level = max(0, min(self.level, len(self.prop.emr.price_upgrade_rate) - 1))  # 0 <= level < len(...)
      bprice = self.prop.emr.prices[instance_type] * self.prop.emr.price_upgrade_rate[level]
      name = '%s-%[email protected]%f' % (group_name, 'SPOT', bprice)

      # Use on-demand instance if prices are zero.
      if bprice > 0:
        ig = InstanceGroup(num, group_name, instance_type, 'SPOT', name, '%.3f' % bprice)
      else:
        ig = InstanceGroup(num, group_name, instance_type, 'ON_DEMAND', name)

      instance_groups.append(ig)      

    return instance_groups

  def get_bootstrap_actions(self):
    '''Get list of bootstrap actions from property'''
    actions = []
    for bootstrap_action in self.prop.emr.bootstrap_actions:
      assert len(bootstrap_action) >= 2, 'Wrong bootstrap action definition: ' + str(bootstrap_action)
      actions.append(BootstrapAction(bootstrap_action[0], bootstrap_action[1], bootstrap_action[2:]))
    return actions

  @synchronized
  def start(self):
    '''Start a EMR cluster.'''
    # emr.project_name is required
    if self.prop.emr.project_name is None:
      raise ValueError('emr.project_name is not set')

    self.last_update = time.time()
    with EMRCluster.rate_limit_lock:
      self.jobid = self.conn.run_jobflow(name=self.prop.emr.cluster_name,
                                         ec2_keyname=self.prop.emr.keyname,
                                         log_uri=self.prop.emr.log_uri,
                                         ami_version=self.prop.emr.ami_version,
                                         bootstrap_actions=self.get_bootstrap_actions(),
                                         keep_alive=True,
                                         action_on_failure='CONTINUE',
                                         api_params={'VisibleToAllUsers': 'true'},
                                         instance_groups=self.get_instance_groups())
    message('Job flow created: %s', self.jobid)

    # Tag EC2 instances to allow future analysis
    tags = {'FlowControl': 'Briefly',
            'Project': self.prop.emr.project_name}
    if self.prop.emr.tags is not None:
      assert isinstance(self.prop.emr.tags, dict)
      tags = dict(tags.items() + self.prop.emr.tags.items())
    self.conn.add_tags(self.jobid, tags)

  @synchronized
  def terminate(self, level_upgrade=0):
    '''Terminate this EMR cluster.'''
    if self.jobid is None:
      return

    self.level += level_upgrade # upgrade to another price level

    message('Terminate jobflow: %s', self.jobid)
    for i in xrange(3):
      try:
        with EMRCluster.rate_limit_lock:
          self.conn.terminate_jobflow(self.jobid)
#.........这里部分代码省略.........