Python local_state.LocalState类代码示例

  def get_cloud_params(self, keyname):
    """Searches through the locations.json file with key
    'infrastructure_info' to build a dict containing the
    parameters necessary to interact with Amazon EC2.

    Args:
      keyname: The name of the SSH keypair that uniquely identifies this
        AppScale deployment.
    """
    params = {
      self.PARAM_CREDENTIALS : {},
      self.PARAM_GROUP : LocalState.get_group(keyname),
      self.PARAM_KEYNAME : keyname
    }

    zone = LocalState.get_zone(keyname)
    if zone:
      params[self.PARAM_REGION] = zone[:-1]
    else:
      params[self.PARAM_REGION] = self.DEFAULT_REGION


    for credential in self.REQUIRED_CREDENTIALS:
      if os.environ.get(credential):
        params[self.PARAM_CREDENTIALS][credential] = os.environ[credential]
      else:
        raise AgentConfigurationException("no " + credential)

    return params

  def test_start_all_nodes_reattach_changed_locations(self):
    self.node_layout = NodeLayout(self.reattach_options)

    fake_agent = FakeAgent()
    flexmock(factory.InfrastructureAgentFactory). \
      should_receive('create_agent'). \
      with_args('public cloud'). \
      and_return(fake_agent)

    LocalState.should_receive('get_host_with_role').and_return('0.0.0.1')

    node_info = [{ "public_ip": "0.0.0.0",
                   "private_ip": "0.0.0.0",
                   "instance_id": "i-APPSCALE1",
                   "roles": ['load_balancer', 'taskqueue', 'shadow',
                            'taskqueue_master'] },
                 { "public_ip": "0.0.0.0",
                   "private_ip": "0.0.0.0",
                   "instance_id": "i-APPSCALE2",
                   "roles": ['memcache', 'appengine'] },
                 { "public_ip": "0.0.0.0",
                   "private_ip": "0.0.0.0",
                   "instance_id": "i-APPSCALE3",
                   "roles": ['zookeeper', "appengine"] },
                 { "public_ip": "0.0.0.0",
                   "private_ip": "0.0.0.0",
                   "instance_id": "i-APPSCALE4",
                   "roles": ['db_master'] }
                 ]

    LocalState.should_receive('get_local_nodes_info').and_return(node_info)

    self.assertRaises(BadConfigurationException)

  def test_start_all_nodes_reattach_changed_asf(self):
    self.options = flexmock(
      infrastructure='public cloud',
      group='group',
      machine='vm image',
      instance_type='instance type',
      keyname='keyname',
      table='cassandra',
      verbose=False,
      test=False,
      use_spot_instances=False,
      zone='zone',
      static_ip=None,
      replication=None,
      appengine=None,
      autoscale=None,
      user_commands=[],
      flower_password='',
      max_memory='X',
      ips=THREE_NODE_CLOUD
    )

    self.node_layout = NodeLayout(self.options)

    fake_agent = FakeAgent()
    flexmock(factory.InfrastructureAgentFactory). \
      should_receive('create_agent'). \
      with_args('public cloud'). \
      and_return(fake_agent)

    LocalState.should_receive('get_local_nodes_info')\
      .and_return(self.reattach_node_info)

    self.assertRaises(BadConfigurationException)

  def test_remove_app_and_app_is_running(self):
    # mock out reading from stdin, and assume the user says 'YES'
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_receive('raw_input').and_return('YES')

    # mock out reading the secret key
    builtins.should_call('open')  # set the fall-through

    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)
    app_stats_data = {'apps': {'blargapp': {'http': 8080, 'language': 'python27',
      'total_reqs': 'no_change', 'appservers': 1, 'https': 4380, 'reqs_enqueued': None}}}


    # mock out the SOAP call to the AppController and assume it succeeded
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('status').with_args('the secret') \
      .and_return('Database is at public1')
    fake_appcontroller.should_receive('stop_app').with_args('blargapp',
      'the secret').and_return('OK')
    fake_appcontroller.should_receive('is_app_running').with_args('blargapp',
      'the secret').and_return(True).and_return(True).and_return(False)
    fake_appcontroller.should_receive('does_app_exist').with_args('blargapp',
      'the secret').and_return(True)
    fake_appcontroller.should_receive('get_all_stats').with_args(
      'the secret').and_return(json.dumps(app_stats_data))
    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
      .and_return(fake_appcontroller)

    # mock out reading the locations.json file, and slip in our own json
    flexmock(os.path)
    os.path.should_call('exists')  # set the fall-through
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return(True)

    fake_nodes_json = flexmock(name="fake_nodes_json")
    fake_nodes_json.should_receive('read').and_return(
      json.dumps({"node_info": [{
        "public_ip": "public1",
        "private_ip": "private1",
        "jobs": ["shadow", "login"]
      }]}))
    fake_nodes_json.should_receive('write').and_return()
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'r') \
      .and_return(fake_nodes_json)

    flexmock(RemoteHelper).should_receive('is_port_open').and_return(False)

    argv = [
      "--appname", "blargapp",
      "--keyname", self.keyname
    ]
    options = ParseArgs(argv, self.function).args
    AppScaleTools.remove_app(options)

  def open_connection(self, parameters):
    """ Connects to Google Compute Engine with the given credentials.

    Args:
      parameters: A dict that contains all the parameters necessary to
        authenticate this user with Google Compute Engine. We assume that the
        user has already authorized this account for use with GCE.
    Returns:
      An apiclient.discovery.Resource that is a connection valid for requests
      to Google Compute Engine for the given user, and a Credentials object that
      can be used to sign requests performed with that connection.
    Raises:
      AppScaleException if the user wants to abort.
    """
    is_autoscale_agent = parameters.get(self.PARAM_AUTOSCALE_AGENT, False)

    # Determine paths to credential files
    if is_autoscale_agent:
      client_secrets_path = self.CLIENT_SECRETS_LOCATION
      oauth2_storage_path = self.OAUTH2_STORAGE_LOCATION
    else:
      # Determine client secrets path
      client_secrets_path = LocalState.get_client_secrets_location(
        parameters[self.PARAM_KEYNAME])
      if not os.path.exists(client_secrets_path):
        client_secrets_path = parameters.get(self.PARAM_SECRETS, '')
      # Determine oauth2 storage
      oauth2_storage_path = parameters.get(self.PARAM_STORAGE)
      if not oauth2_storage_path or not os.path.exists(oauth2_storage_path):
        oauth2_storage_path = LocalState.get_oauth2_storage_location(
          parameters[self.PARAM_KEYNAME])

    if os.path.exists(client_secrets_path):
      # Attempt to perform authorization using Service account
      secrets_type = GCEAgent.get_secrets_type(client_secrets_path)
      if secrets_type == CredentialTypes.SERVICE:
        scopes = [GCPScopes.COMPUTE]
        credentials = ServiceAccountCredentials.from_json_keyfile_name(
          client_secrets_path, scopes=scopes)
        return discovery.build('compute', self.API_VERSION), credentials

    # Perform authorization using OAuth2 storage
    storage = oauth2client.file.Storage(oauth2_storage_path)
    credentials = storage.get()

    if not credentials or credentials.invalid:
      # If we couldn't get valid credentials from OAuth2 storage
      if not os.path.exists(client_secrets_path):
        raise AgentConfigurationException(
          'Couldn\'t find client secrets file at {}'.format(client_secrets_path)
        )
      # Run OAuth2 flow to get credentials
      flow = oauth2client.client.flow_from_clientsecrets(
        client_secrets_path, scope=self.GCE_SCOPE)
      flags = oauth2client.tools.argparser.parse_args(args=[])
      credentials = oauth2client.tools.run_flow(flow, storage, flags)

    # Build the service
    return discovery.build('compute', self.API_VERSION), credentials

  def test_describe_instances_with_two_nodes(self):
    # mock out writing the secret key to ~/.appscale, as well as reading it
    # later
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')  # set the fall-through

    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    fake_secret.should_receive('write').and_return()
    builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)

    # mock out the SOAP call to the AppController and assume it succeeded
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('get_all_public_ips').with_args('the secret') \
      .and_return(json.dumps(['public1', 'public2']))
    fake_appcontroller.should_receive('status').with_args('the secret') \
      .and_return('nothing interesting here') \
      .and_return('Database is at not-up-yet') \
      .and_return('Database is at 1.2.3.4')
    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
      .and_return(fake_appcontroller)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public2:17443') \
      .and_return(fake_appcontroller)

    # mock out reading the locations.json file, and slip in our own json
    flexmock(os.path)
    os.path.should_call('exists')  # set the fall-through
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return(True)

    fake_nodes_json = flexmock(name="fake_nodes_json")
    fake_nodes_json.should_receive('read').and_return(json.dumps(
      {"node_info": [{
        "public_ip": "public1",
        "private_ip": "private1",
        "jobs": ["shadow", "login"]
      },
        {
          "public_ip": "public2",
          "private_ip": "private2",
          "jobs": ["appengine"]
        },
      ]}))
    fake_nodes_json.should_receive('write').and_return()
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'r') \
      .and_return(fake_nodes_json)
    # assume that there are two machines running in our deployment

    argv = [
      "--keyname", self.keyname
    ]
    options = ParseArgs(argv, self.function).args
    AppScaleTools.describe_instances(options)

def main():
  """
  Execute appscale-describe-instances script.
  """
  options = ParseArgs(sys.argv[1:], "appscale-show-stats").args
  try:
    show_stats(options)
    sys.exit(0)
  except Exception, e:
    LocalState.generate_crash_log(e, traceback.format_exc())
    sys.exit(1)

  def configure_instance_security(self, parameters):
    """
    Setup EC2 security keys and groups. Required input values are read from
    the parameters dictionary. More specifically, this method expects to
    find a 'keyname' parameter and a 'group' parameter in the parameters
    dictionary. Using these provided values, this method will create a new
    EC2 key-pair and a security group. Security group will be granted permission
    to access any port on the instantiated VMs. (Also see documentation for the
    BaseAgent class)

    Args:
      parameters: A dictionary of parameters.
    """
    keyname = parameters[self.PARAM_KEYNAME]
    group = parameters[self.PARAM_GROUP]
    is_autoscale = parameters['autoscale_agent']

    AppScaleLogger.log("Verifying that keyname {0}".format(keyname) + \
      " is not already registered.")
    conn = self.open_connection(parameters)

    # While creating instances during autoscaling, we do not need to create a
    # new keypair or a security group. We just make use of the existing one.
    if is_autoscale in ['True', True]:
      return

    if conn.get_key_pair(keyname):
      self.handle_failure("SSH keyname {0} is already registered. Please " \
        "change the 'keyname' specified in your AppScalefile to a different " \
        "value, or erase it to have one automatically generated for you." \
        .format(keyname))

    security_groups = conn.get_all_security_groups()
    for security_group in security_groups:
      if security_group.name == group:
        self.handle_failure("Security group {0} is already registered. Please" \
          " change the 'group' specified in your AppScalefile to a different " \
          "value, or erase it to have one automatically generated for you." \
          .format(group))

    AppScaleLogger.log("Creating key pair: {0}".format(keyname))
    key_pair = conn.create_key_pair(keyname)
    ssh_key = '{0}{1}.key'.format(LocalState.LOCAL_APPSCALE_PATH, keyname)
    LocalState.write_key_file(ssh_key, key_pair.material)

    self.create_security_group(parameters, group)
    self.authorize_security_group(parameters, group, from_port=1, to_port=65535,
      ip_protocol='udp', cidr_ip='0.0.0.0/0')
    self.authorize_security_group(parameters, group, from_port=1, to_port=65535,
      ip_protocol='tcp', cidr_ip='0.0.0.0/0')
    self.authorize_security_group(parameters, group, from_port=-1, to_port=-1,
      ip_protocol='icmp', cidr_ip='0.0.0.0/0')
    return True

  def configure_instance_security(self, parameters):
    """ Creates a GCE network and firewall with the specified name, and opens
    the ports on that firewall as needed for AppScale.

    We expect both the network and the firewall to not exist before this point,
    to avoid accidentally placing AppScale instances from different deployments
    in the same network and firewall (thus enabling them to see each other's web
    traffic).

    Args:
      parameters: A dict with keys for each parameter needed to connect to
        Google Compute Engine, and an additional key indicating the name of the
        network and firewall that we should create in GCE.
    Returns:
      True, if the named network and firewall was created successfully.
    Raises:
      AgentRuntimeException: If the named network or firewall already exist in
      GCE.
    """
    is_autoscale_agent = parameters.get(self.PARAM_AUTOSCALE_AGENT, False)

    # While creating instances during autoscaling, we do not need to create a
    # new keypair or a network. We just make use of the existing one.
    if is_autoscale_agent:
      return

    AppScaleLogger.log("Verifying that SSH key exists locally")
    keyname = parameters[self.PARAM_KEYNAME]
    private_key = LocalState.LOCAL_APPSCALE_PATH + keyname
    public_key = private_key + ".pub"

    if os.path.exists(private_key) or os.path.exists(public_key):
      raise AgentRuntimeException("SSH key already found locally - please " +
        "use a different keyname")

    LocalState.generate_rsa_key(keyname, parameters[self.PARAM_VERBOSE])

    ssh_key_exists, all_ssh_keys = self.does_ssh_key_exist(parameters)
    if not ssh_key_exists:
      self.create_ssh_key(parameters, all_ssh_keys)

    if self.does_network_exist(parameters):
      raise AgentRuntimeException("Network already exists - please use a " + \
        "different group name.")

    if self.does_firewall_exist(parameters):
      raise AgentRuntimeException("Firewall already exists - please use a " + \
        "different group name.")

    network_url = self.create_network(parameters)
    self.create_firewall(parameters, network_url)

  def test_reset_password_for_user_that_exists(self):
    # put in a mock for reading the secret file
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')  # set the fall-through

    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)

    # mock out reading the username and new password from the user
    builtins.should_receive('raw_input').and_return('[email protected]')
    flexmock(getpass)
    getpass.should_receive('getpass').and_return('the password')

    # mock out finding the login node's IP address from the json file
    flexmock(os.path)
    os.path.should_call('exists')  # set the fall-through
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return(True)

    fake_nodes_json = flexmock(name="fake_secret")
    fake_nodes_json.should_receive('read').and_return(
      json.dumps({"node_info": [{
        'public_ip': 'public1',
        'private_ip': 'private1',
        'jobs': ['login', 'db_master']
      }]}))
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'r') \
      .and_return(fake_nodes_json)

    # mock out grabbing the userappserver ip from an appcontroller
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('status').with_args('the secret') \
      .and_return('nothing interesting here') \
      .and_return('Database is at not-up-yet') \
      .and_return('Database is at public1')
    fake_appcontroller.should_receive('reset_password').with_args(
      '[email protected]', str, 'the secret').and_return('true')

    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
      .and_return(fake_appcontroller)

    argv = [
      "--keyname", self.keyname
    ]
    options = ParseArgs(argv, self.function).args
    AppScaleTools.reset_password(options)

  def test_make_appscale_directory_creation(self):
    # let's say that our ~/.appscale directory
    # does not exist
    os.path.should_receive('exists') \
      .with_args(LocalState.LOCAL_APPSCALE_PATH) \
      .and_return(False) \
      .once()

    # thus, mock out making the appscale dir
    os.should_receive('mkdir') \
      .with_args(LocalState.LOCAL_APPSCALE_PATH) \
      .and_return()

    LocalState.make_appscale_directory()

  def test_remove_app_but_app_isnt_running(self):
    # mock out reading from stdin, and assume the user says 'yes'
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_receive('raw_input').and_return('yes')

    # mock out reading the secret key
    builtins.should_call('open')  # set the fall-through

    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)

    # mock out the SOAP call to the AppController and assume it succeeded
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('status').with_args('the secret') \
      .and_return('Database is at public1')
    fake_appcontroller.should_receive('is_app_running').with_args('blargapp',
      'the secret').and_return(False)
    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
      .and_return(fake_appcontroller)

    # mock out reading the locations.json file, and slip in our own json
    flexmock(os.path)
    os.path.should_call('exists')  # set the fall-through
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return(True)

    fake_nodes_json = flexmock(name="fake_nodes_json")
    fake_nodes_json.should_receive('read').and_return(
      json.dumps({"node_info": [{
        "public_ip": "public1",
        "private_ip": "private1",
        "jobs": ["shadow", "login"]
      }]}))
    fake_nodes_json.should_receive('write').and_return()
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'r') \
      .and_return(fake_nodes_json)

    argv = [
      "--appname", "blargapp",
      "--keyname", self.keyname
    ]
    options = ParseArgs(argv, self.function).args
    self.assertRaises(AppScaleException, AppScaleTools.remove_app, options)

  def test_get_property(self):
    # put in a mock for reading the secret file
    builtins = flexmock(sys.modules['__builtin__'])
    builtins.should_call('open')  # set the fall-through

    secret_key_location = LocalState.get_secret_key_location(self.keyname)
    fake_secret = flexmock(name="fake_secret")
    fake_secret.should_receive('read').and_return('the secret')
    builtins.should_receive('open').with_args(secret_key_location, 'r') \
      .and_return(fake_secret)

    # mock out finding the shadow node's IP address from the json file
    flexmock(os.path)
    os.path.should_call('exists')  # set the fall-through
    os.path.should_receive('exists').with_args(
      LocalState.get_locations_json_location(self.keyname)).and_return(True)

    fake_nodes_json = flexmock(name="fake_secret")
    fake_nodes_json.should_receive('read').and_return(
      json.dumps({"node_info": [{
        'public_ip': 'public1',
        'private_ip': 'private1',
        'roles': ['shadow']
      }]}))
    builtins.should_receive('open').with_args(
      LocalState.get_locations_json_location(self.keyname), 'r') \
      .and_return(fake_nodes_json)

    # mock out grabbing the userappserver ip from an appcontroller
    property_name = "name"
    property_value = "value"
    fake_appcontroller = flexmock(name='fake_appcontroller')
    fake_appcontroller.should_receive('set_property').with_args(property_name,
      property_value, 'the secret').and_return('OK')

    flexmock(SOAPpy)
    SOAPpy.should_receive('SOAPProxy').with_args('https://public1:17443') \
      .and_return(fake_appcontroller)

    argv = [
      "--keyname", self.keyname,
      "--property_name", property_name,
      "--property_value", property_value
    ]
    options = ParseArgs(argv, self.function).args

    result = AppScaleTools.set_property(options)
    self.assertEqual(None, result)

  def up(self):
    """ Starts an AppScale deployment with the configuration options from the
    AppScalefile in the current directory.

    Raises:
      AppScalefileException: If there is no AppScalefile in the current
      directory.
    """
    contents = self.read_appscalefile()

    # If running in a cluster environment, we first need to set up SSH keys
    contents_as_yaml = yaml.safe_load(contents)
    if not LocalState.ensure_appscalefile_is_up_to_date():
      contents = self.read_appscalefile()
      contents_as_yaml = yaml.safe_load(contents)

    # Construct a run-instances command from the file's contents
    command = []
    for key, value in contents_as_yaml.items():
      if key in self.DEPRECATED_ASF_ARGS:
        raise AppScalefileException(
          "'{0}' has been deprecated. Refer to {1} to see the full changes.".
            format(key, NodeLayout.APPSCALEFILE_INSTRUCTIONS ))

      if value is True:
        command.append(str("--%s" % key))
      elif value is False:
        pass
      else:
        if key == "ips_layout":
          command.append("--ips_layout")
          command.append(base64.b64encode(yaml.dump(value)))
        elif key == "disks":
          command.append("--disks")
          command.append(base64.b64encode(yaml.dump(value)))
        elif key == "user_commands":
          command.append("--user_commands")
          command.append(base64.b64encode(yaml.dump(value)))
        else:
          command.append(str("--%s" % key))
          command.append(str("%s" % value))

    run_instances_opts = ParseArgs(command, "appscale-run-instances").args

    if 'infrastructure' not in contents_as_yaml:
      # Generate a new keypair if necessary.
      if not self.valid_ssh_key(contents_as_yaml, run_instances_opts):
        add_keypair_command = []
        if 'keyname' in contents_as_yaml:
          add_keypair_command.append('--keyname')
          add_keypair_command.append(str(contents_as_yaml['keyname']))

        add_keypair_command.append('--ips_layout')
        add_keypair_command.append(
          base64.b64encode(yaml.dump(contents_as_yaml['ips_layout'])))
        add_keypair_opts = ParseArgs(
          add_keypair_command, 'appscale-add-keypair').args
        AppScaleTools.add_keypair(add_keypair_opts)

    AppScaleTools.run_instances(run_instances_opts)

  def test_start_all_nodes_reattach(self):
    self.node_layout = NodeLayout(self.reattach_options)
    self.assertNotEqual([], self.node_layout.nodes)
    fake_agent = FakeAgent()
    flexmock(factory.InfrastructureAgentFactory). \
      should_receive('create_agent'). \
      with_args('euca'). \
      and_return(fake_agent)

    LocalState.should_receive('get_host_with_role').and_return(IP_1)

    LocalState.should_receive('get_local_nodes_info') \
      .and_return(self.reattach_node_info)

    RemoteHelper.start_all_nodes(self.reattach_options,
                                 self.node_layout)