Python appscale_logger.AppScaleLogger类代码示例

  def add_access_config(self, parameters, instance_id, static_ip):
    """ Instructs Google Compute Engine to use the given IP address as the
    public IP for the named instance.

    This assumes that there is no existing public IP address for the named
    instance. If this is not the case, callers should use delete_access_config
    first to remove it.

    Args:
      parameters: A dict with keys for each parameter needed to connect to
        Google Compute Engine, and an additional key mapping to a list of
        instance names that should be deleted.
      instance_id: A str naming the running instance that the new public IP
        address should be added to.
      static_ip: A str naming the already allocated static IP address that
        will be used for the named instance.
    """
    gce_service, credentials = self.open_connection(parameters)
    http = httplib2.Http()
    auth_http = credentials.authorize(http)
    request = gce_service.instances().addAccessConfig(
      project=parameters[self.PARAM_PROJECT],
      instance=instance_id,
      networkInterface="nic0",
      zone=parameters[self.PARAM_ZONE],
      body={
        "kind": "compute#accessConfig",
        "type" : "ONE_TO_ONE_NAT",
        "name" : "External NAT",
        "natIP" : static_ip
      }
    )
    response = request.execute(http=auth_http)
    AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])

  def create_security_group(self, parameters, group):
    """Creates a new security group in AWS with the given name.

    Args:
      parameters: A dict that contains the credentials necessary to authenticate
        with AWS.
      group: A str that names the group that should be created.
    Raises:
      AgentRuntimeException: If the security group could not be created.
    """
    AppScaleLogger.log('Creating security group: {0}'.format(group))
    conn = self.open_connection(parameters)
    retries_left = self.SECURITY_GROUP_RETRY_COUNT
    while retries_left:
      try:
        conn.create_security_group(group, 'AppScale security group')
      except EC2ResponseError:
        pass
      try:
        conn.get_all_security_groups(group)
        return
      except EC2ResponseError:
        pass
      time.sleep(self.SLEEP_TIME)
      retries_left -= 1

    raise AgentRuntimeException("Couldn't create security group with " \
      "name {0}".format(group))

  def does_address_exist(self, parameters):
    """ Queries Google Compute Engine to see if the specified static IP address
    exists for this user.

    Args:
      parameters: A dict with keys for each parameter needed to connect to
        Google Compute Engine, and an additional key indicating the name of the
        static IP address that we should check for existence.
    Returns:
      True if the named address exists, and False otherwise.
    """
    gce_service, credentials = self.open_connection(parameters)
    http = httplib2.Http()
    auth_http = credentials.authorize(http)
    request = gce_service.addresses().list(
      project=parameters[self.PARAM_PROJECT],
      filter="address eq {0}".format(parameters[self.PARAM_STATIC_IP]),
      region=parameters[self.PARAM_REGION]
    )
    response = request.execute(http=auth_http)
    AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])

    if 'items' in response:
      return True
    else:
      return False

  def terminate_instances(self, parameters):
    """ Deletes the instances specified in 'parameters' running in Google
    Compute Engine.

    Args:
      parameters: A dict with keys for each parameter needed to connect to
        Google Compute Engine, and an additional key mapping to a list of
        instance names that should be deleted.
    """
    instance_ids = parameters[self.PARAM_INSTANCE_IDS]
    responses = []
    for instance_id in instance_ids:
      gce_service, credentials = self.open_connection(parameters)
      http = httplib2.Http()
      auth_http = credentials.authorize(http)
      request = gce_service.instances().delete(
        project=parameters[self.PARAM_PROJECT],
        zone=parameters[self.PARAM_ZONE],
        instance=instance_id
      )
      response = request.execute(http=auth_http)
      AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
      responses.append(response)

    for response in responses:
      gce_service, credentials = self.open_connection(parameters)
      http = httplib2.Http()
      auth_http = credentials.authorize(http)
      self.ensure_operation_succeeds(gce_service, auth_http, response,
        parameters[self.PARAM_PROJECT])

  def create_network(self, parameters):
    """ Creates a new network in Google Compute Engine with the specified name.

    Args:
      parameters: A dict with keys for each parameter needed to connect to
        Google Compute Engine, and an additional key indicating the name of the
        network that we should create in GCE.
    Returns:
      The URL corresponding to the name of the network that was created, for use
      with binding this network to one or more firewalls.
    """
    gce_service, credentials = self.open_connection(parameters)
    http = httplib2.Http()
    auth_http = credentials.authorize(http)
    request = gce_service.networks().insert(
      project=parameters[self.PARAM_PROJECT],
      body={
        "name" : parameters[self.PARAM_GROUP],
        "description" : "Network used for AppScale instances",
        "IPv4Range" : "10.240.0.0/16"
      }
    )
    response = request.execute(http=auth_http)
    AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
    self.ensure_operation_succeeds(gce_service, auth_http, response,
      parameters[self.PARAM_PROJECT])
    return response['targetLink']

  def create_firewall(self, parameters, network_url):
    """ Creates a new firewall in Google Compute Engine with the specified name,
    bound to the specified network.

    Args:
      parameters: A dict with keys for each parameter needed to connect to
        Google Compute Engine, and an additional key indicating the name of the
        firewall that we should create.
      network_url: A str containing the URL of the network that this new
        firewall should be applied to.
    """
    gce_service, credentials = self.open_connection(parameters)
    http = httplib2.Http()
    auth_http = credentials.authorize(http)
    request = gce_service.firewalls().insert(
      project=parameters[self.PARAM_PROJECT],
      body={
        "name" : parameters[self.PARAM_GROUP],
        "description" : "Firewall used for AppScale instances",
        "network" : network_url,
        "sourceRanges" : ["0.0.0.0/0"],
        "allowed" : [
          {"IPProtocol" : "tcp", "ports": ["1-65535"]},
          {"IPProtocol" : "udp", "ports": ["1-65535"]},
          {"IPProtocol" : "icmp"}
        ]
      }
    )
    response = request.execute(http=auth_http)
    AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])
    self.ensure_operation_succeeds(gce_service, auth_http, response,
      parameters[self.PARAM_PROJECT])

  def get_optimal_spot_price(self, conn, instance_type, zone):
    """
    Returns the spot price for an EC2 instance of the specified instance type.
    The returned value is computed by averaging all the spot price history
    values returned by the back-end EC2 APIs and incrementing the average by
    extra 10%.

    Args:
      conn: A boto.EC2Connection that can be used to communicate with AWS.
      instance_type: A str representing the instance type whose prices we
        should speculate for.
      zone: A str representing the availability zone that the instance will
        be placed in.
    Returns:
      The estimated spot price for the specified instance type, in the
        specified availability zone.
    """
    end_time = datetime.datetime.now()
    start_time = end_time - datetime.timedelta(days=7)
    history = conn.get_spot_price_history(start_time=start_time.isoformat(),
      end_time=end_time.isoformat(), product_description='Linux/UNIX',
      instance_type=instance_type, availability_zone=zone)
    var_sum = 0.0
    for entry in history:
      var_sum += entry.price
    average = var_sum / len(history)
    bid_price = average * 1.10
    AppScaleLogger.log('The average spot instance price for a {0} machine is'\
        ' {1}, and 10% more is {2}'.format(instance_type, average, bid_price))
    return bid_price

  def handle_failure(self, msg):
    """ Log the specified error message and raise an AgentRuntimeException

    Args:
      msg: An error message to be logged and included in the raised exception.
    Raises:
      AgentRuntimeException Contains the input error message.
    """
    AppScaleLogger.log(msg)
    raise AgentRuntimeException(msg)

  def does_ssh_key_exist(self, parameters):
    """ Queries Google Compute Engine to see if the specified SSH key exists.

    Args:
      parameters: A dict with keys for each parameter needed to connect to
        Google Compute Engine. We don't have an additional key for the name of
        the SSH key, since we use the one in ~/.ssh.
    Returns:
      A tuple of two items. The first item is a bool that is True if
        our public key's contents are in GCE, and False otherwise, while
        the second item is the contents of all SSH keys stored in GCE.
    """
    our_public_ssh_key = None
    public_ssh_key_location = LocalState.LOCAL_APPSCALE_PATH + \
      parameters[self.PARAM_KEYNAME] + ".pub"
    with open(public_ssh_key_location) as file_handle:
      system_user = os.getenv('LOGNAME', default=pwd.getpwuid(os.getuid())[0])
      our_public_ssh_key = system_user + ":" + file_handle.read().rstrip()

    gce_service, credentials = self.open_connection(parameters)
    try:
      http = httplib2.Http()
      auth_http = credentials.authorize(http)
      request = gce_service.projects().get(
        project=parameters[self.PARAM_PROJECT])
      response = request.execute(http=auth_http)
      AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE])

      if not 'items' in response['commonInstanceMetadata']:
        return False, ""

      metadata = response['commonInstanceMetadata']['items']
      if not metadata:
        return False, ""

      all_ssh_keys = ""
      for item in metadata:
        if item['key'] != 'sshKeys':
          continue

        # Now that we know there's one or more SSH keys, just make sure that
        # ours is in this list.
        all_ssh_keys = item['value']
        if our_public_ssh_key in all_ssh_keys:
          return True, all_ssh_keys

      return False, all_ssh_keys
    except errors.HttpError:
      return False, ""

  def does_image_exist(self, parameters):
    """ Queries Amazon EC2 to see if the specified image exists.

    Args:
      parameters: A dict that contains the machine ID to check for existence.
    Returns:
      True if the machine ID exists, False otherwise.
    """
    try:
      conn = self.open_connection(parameters)
      image_id = parameters[self.PARAM_IMAGE_ID]
      conn.get_image(image_id)
      AppScaleLogger.log('Machine image {0} does exist'.format(image_id))
      return True
    except boto.exception.EC2ResponseError:
      AppScaleLogger.log('Machine image {0} does not exist'.format(image_id))
      return False

  def configure_instance_security(self, parameters):
    """
    Setup EC2 security keys and groups. Required input values are read from
    the parameters dictionary. More specifically, this method expects to
    find a 'keyname' parameter and a 'group' parameter in the parameters
    dictionary. Using these provided values, this method will create a new
    EC2 key-pair and a security group. Security group will be granted permission
    to access any port on the instantiated VMs. (Also see documentation for the
    BaseAgent class)

    Args:
      parameters: A dictionary of parameters.
    """
    keyname = parameters[self.PARAM_KEYNAME]
    group = parameters[self.PARAM_GROUP]

    AppScaleLogger.log("Verifying that keyname {0}".format(keyname) + \
      " is not already registered.")
    conn = self.open_connection(parameters)
    if conn.get_key_pair(keyname):
      self.handle_failure("SSH keyname {0} is already registered. Please " \
        "change the 'keyname' specified in your AppScalefile to a different " \
        "value, or erase it to have one automatically generated for you." \
        .format(keyname))

    security_groups = conn.get_all_security_groups()
    for security_group in security_groups:
      if security_group.name == group:
        self.handle_failure("Security group {0} is already registered. Please" \
          " change the 'group' specified in your AppScalefile to a different " \
          "value, or erase it to have one automatically generated for you." \
          .format(group))

    AppScaleLogger.log("Creating key pair: {0}".format(keyname))
    key_pair = conn.create_key_pair(keyname)
    ssh_key = '{0}{1}.key'.format(LocalState.LOCAL_APPSCALE_PATH, keyname)
    LocalState.write_key_file(ssh_key, key_pair.material)

    self.create_security_group(parameters, group)
    self.authorize_security_group(parameters, group, from_port=1, to_port=65535,
      ip_protocol='udp', cidr_ip='0.0.0.0/0')
    self.authorize_security_group(parameters, group, from_port=1, to_port=65535,
      ip_protocol='tcp', cidr_ip='0.0.0.0/0')
    self.authorize_security_group(parameters, group, from_port=-1, to_port=-1,
      ip_protocol='icmp', cidr_ip='0.0.0.0/0')
    return True

  def does_disk_exist(self, parameters, disk_name):
    """ Queries Amazon EC2 to see if the specified EBS volume exists.

    Args:
      parameters: A dict that contains the credentials needed to authenticate
        with AWS.
      disk_name: A str naming the EBS volume to check for existence.
    Returns:
      True if the named EBS volume exists, and False otherwise.
    """
    conn = self.open_connection(parameters)
    try:
      conn.get_all_volumes([disk_name])
      AppScaleLogger.log('EBS volume {0} does exist'.format(disk_name))
      return True
    except boto.exception.EC2ResponseError:
      AppScaleLogger.log('EBS volume {0} does not exist'.format(disk_name))
      return False

  def does_zone_exist(self, parameters):
    """ Queries Amazon EC2 to see if the specified availability zone exists.

    Args:
      parameters: A dict that contains the availability zone to check for
        existence.
    Returns:
      True if the availability zone exists, and False otherwise.
    """
    try:
      conn = self.open_connection(parameters)
      zone = parameters[self.PARAM_ZONE]
      conn.get_all_zones(zone)
      AppScaleLogger.log('Availability zone {0} does exist'.format(zone))
      return True
    except boto.exception.EC2ResponseError:
      AppScaleLogger.log('Availability zone {0} does not exist'.format(zone))
      return False

  def configure_instance_security(self, parameters):
    """ Creates a GCE network and firewall with the specified name, and opens
    the ports on that firewall as needed for AppScale.

    We expect both the network and the firewall to not exist before this point,
    to avoid accidentally placing AppScale instances from different deployments
    in the same network and firewall (thus enabling them to see each other's web
    traffic).

    Args:
      parameters: A dict with keys for each parameter needed to connect to
        Google Compute Engine, and an additional key indicating the name of the
        network and firewall that we should create in GCE.
    Returns:
      True, if the named network and firewall was created successfully.
    Raises:
      AgentRuntimeException: If the named network or firewall already exist in
      GCE.
    """
    AppScaleLogger.log("Verifying that SSH key exists locally")
    keyname = parameters[self.PARAM_KEYNAME]
    private_key = LocalState.LOCAL_APPSCALE_PATH + keyname
    public_key = private_key + ".pub"

    if os.path.exists(private_key) or os.path.exists(public_key):
      raise AgentRuntimeException("SSH key already found locally - please " +
        "use a different keyname")

    LocalState.generate_rsa_key(keyname, parameters[self.PARAM_VERBOSE])

    ssh_key_exists, all_ssh_keys = self.does_ssh_key_exist(parameters)
    if not ssh_key_exists:
      self.create_ssh_key(parameters, all_ssh_keys)

    if self.does_network_exist(parameters):
      raise AgentRuntimeException("Network already exists - please use a " + \
        "different group name.")

    if self.does_firewall_exist(parameters):
      raise AgentRuntimeException("Firewall already exists - please use a " + \
        "different group name.")

    network_url = self.create_network(parameters)
    self.create_firewall(parameters, network_url)

  def authorize_security_group(self, parameters, group, from_port, to_port,
    ip_protocol, cidr_ip):
    """Opens up traffic on the given port range for traffic of the named type.

    Args:
      parameters: A dict that contains the credentials necessary to authenticate
        with AWS.
      group: A str that names the group whose ports should be opened.
      from_port: An int that names the first port that access should be allowed
        on.
      to_port: An int that names the last port that access should be allowed on.
      ip_protocol: A str that indicates if TCP, UDP, or ICMP traffic should be
        allowed.
      cidr_ip: A str that names the IP range that traffic should be allowed
        from.
    Raises:
      AgentRuntimeException: If the ports could not be opened on the security
      group.
    """
    AppScaleLogger.log('Authorizing security group {0} for {1} traffic from ' \
      'port {2} to port {3}'.format(group, ip_protocol, from_port, to_port))
    conn = self.open_connection(parameters)
    retries_left = self.SECURITY_GROUP_RETRY_COUNT
    while retries_left:
      try:
        conn.authorize_security_group(group, from_port=from_port,
          to_port=to_port, ip_protocol=ip_protocol, cidr_ip=cidr_ip)
      except EC2ResponseError:
        pass
      try:
        group_info = conn.get_all_security_groups(group)[0]
        for rule in group_info.rules:
          if int(rule.from_port) == from_port and int(rule.to_port) == to_port \
            and rule.ip_protocol == ip_protocol:
            return
      except EC2ResponseError:
        pass
      time.sleep(self.SLEEP_TIME)
      retries_left -= 1

    raise AgentRuntimeException("Couldn't authorize {0} traffic from port " \
      "{1} to port {2} on CIDR IP {3}".format(ip_protocol, from_port, to_port,
      cidr_ip))