Python remote_helper.RemoteHelper类代码示例

    def terminate_instances(cls, options):
        """Stops all services running in an AppScale deployment, and in cloud
    deployments, also powers off the instances previously spawned.

    Raises:
      AppScaleException: If AppScale is not running, and thus can't be
      terminated.
    """
        if not os.path.exists(LocalState.get_secret_key_location(options.keyname)):
            raise AppScaleException("AppScale is not running with the keyname {0}".format(options.keyname))

        infrastructure = LocalState.get_infrastructure(options.keyname)

        # If the user is on a cloud deployment, and not backing their data to
        # persistent disks, warn them before shutting down AppScale.
        # Also, if we're in developer mode, skip the warning.
        if infrastructure != "xen" and not LocalState.are_disks_used(options.keyname) and not options.test:
            LocalState.ensure_user_wants_to_terminate()

        if infrastructure in InfrastructureAgentFactory.VALID_AGENTS:
            RemoteHelper.terminate_cloud_infrastructure(options.keyname, options.verbose)
        else:
            RemoteHelper.terminate_virtualized_cluster(options.keyname, options.verbose)

        LocalState.cleanup_appscale_files(options.keyname)
        AppScaleLogger.success("Successfully shut down your AppScale deployment.")

  def test_copy_deployment_credentials_in_cloud(self):
    options = flexmock(
      keyname='key1',
      infrastructure='ec2',
      verbose=True,
    )

    local_state = flexmock(LocalState)
    remote_helper = flexmock(RemoteHelper)
    local_state.should_receive('get_secret_key_location').and_return()
    local_state.should_receive('get_key_path_from_name').and_return()
    local_state.should_receive('get_certificate_location').and_return()
    local_state.should_receive('get_private_key_location').and_return()

    remote_helper.should_receive('scp').and_return()
    local_state.should_receive('generate_ssl_cert').and_return()
    popen_object = flexmock(communicate=lambda: ['hash_id'])
    flexmock(subprocess).should_receive('Popen').and_return(popen_object)
    remote_helper.should_receive('ssh').and_return()
    flexmock(AppScaleLogger).should_receive('log').and_return()

    RemoteHelper.copy_deployment_credentials('public1', options)

    flexmock(GCEAgent).should_receive('get_secrets_type').\
      and_return(CredentialTypes.OAUTH)
    flexmock(os.path).should_receive('exists').and_return(True)

    options = flexmock(
      keyname='key1',
      infrastructure='gce',
      verbose=True,
    )
    local_state.should_receive('get_oauth2_storage_location').and_return()

    RemoteHelper.copy_deployment_credentials('public1', options)

  def test_start_remote_appcontroller(self):
    # mock out removing the old json file
    local_state = flexmock(LocalState)
    local_state.should_receive('shell')\
      .with_args(re.compile('^ssh'),False,5,stdin=re.compile('rm -rf'))\
      .and_return()

    # assume we started monit on public1 fine
    local_state.should_receive('shell')\
      .with_args(re.compile('^ssh'), False, 5, stdin=re.compile('monit'))\
      .and_return()

    # also assume that we scp'ed over the god config file fine
    local_state.should_receive('shell')\
      .with_args(re.compile('scp .*controller-17443.cfg*'),False,5)\
      .and_return()

    # and assume we started the AppController on public1 fine
    local_state.should_receive('shell')\
      .with_args(re.compile('^ssh'), False, 5,
        stdin=re.compile('^monit start -g controller'))\
      .and_return()

    # finally, assume the appcontroller comes up after a few tries
    # assume that ssh comes up on the third attempt
    fake_socket = flexmock(name='fake_socket')
    fake_socket.should_receive('connect').with_args(('public1',
      AppControllerClient.PORT)).and_raise(Exception) \
      .and_raise(Exception).and_return(None)
    socket.should_receive('socket').and_return(fake_socket)

    RemoteHelper.start_remote_appcontroller('public1', 'bookey', False)

  def gather_logs(cls, options):
    """Collects logs from each machine in the currently running AppScale
    deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    # First, make sure that the place we want to store logs doesn't
    # already exist.
    if os.path.exists(options.location):
      raise AppScaleException("Can't gather logs, as the location you " + \
        "specified, {0}, already exists.".format(options.location))

    acc = AppControllerClient(LocalState.get_login_host(options.keyname),
      LocalState.get_secret_key(options.keyname))

    # do the mkdir after we get the secret key, so that a bad keyname will
    # cause the tool to crash and not create this directory
    os.mkdir(options.location)

    for ip in acc.get_all_public_ips():
      # Get the logs from each node, and store them in our local directory
      local_dir = "{0}/{1}".format(options.location, ip)
      os.mkdir(local_dir)
      RemoteHelper.scp_remote_to_local(ip, options.keyname, '/var/log/appscale',
        local_dir, options.verbose)
    AppScaleLogger.success("Successfully copied logs to {0}".format(
      options.location))

  def add_instances(cls, options):
    """Adds additional machines to an AppScale deployment.

    Args:
      options: A Namespace that has fields for each parameter that can be
        passed in via the command-line interface.
    """
    if 'master' in options.ips.keys():
      raise BadConfigurationException("Cannot add master nodes to an " + \
        "already running AppScale deployment.")

    # Skip checking for -n (replication) because we don't allow the user
    # to specify it here (only allowed in run-instances).
    additional_nodes_layout = NodeLayout(options)

    # In virtualized cluster deployments, we need to make sure that the user
    # has already set up SSH keys.
    if LocalState.get_from_yaml(options.keyname, 'infrastructure') == "xen":
      for ip in options.ips.values():
        # throws a ShellException if the SSH key doesn't work
        RemoteHelper.ssh(ip, options.keyname, "ls", options.verbose)

    # Finally, find an AppController and send it a message to add
    # the given nodes with the new roles.
    AppScaleLogger.log("Sending request to add instances")
    login_ip = LocalState.get_login_host(options.keyname)
    acc = AppControllerClient(login_ip, LocalState.get_secret_key(
      options.keyname))
    acc.start_roles_on_nodes(json.dumps(options.ips))

    # TODO(cgb): Should we wait for the new instances to come up and get
    # initialized?
    AppScaleLogger.success("Successfully sent request to add instances " + \
      "to this AppScale deployment.")

  def add(self, argv):
    parser = argparse.ArgumentParser(usage=self.SUPPORT_CMDS['add'])
    parser.add_argument("policy_name")
    parser.add_argument("policy_file", nargs="?", type=argparse.FileType('r'), default=sys.stdin)
    parser.add_argument("-inactive", action='store_true')
    options = parser.parse_args(argv)
    content = options.policy_file.read()
    if not options.policy_file == sys.stdin:
	    options.policy_file.close()

    if self.eager:
	    res = self.eager.add_policy(options.policy_name, content, not options.inactive)
	    if res[0] == 0:
		    print res[1]
    else: 
      if self.remote_exist(options.policy_name, 'all'):
        print "Error: Policy {0} already exists!".format(options.policy_name)
        return
      if options.inactive:
        s = '.i.py'
      else:
        s = '.a.py'
      s = options.policy_name + s
      tmp = open(s, "w")
      tmp.write(content)
      tmp.close()
      RemoteHelper.scp(self.service_host, self.key_name, s, self.POLICY_DIR, False)

    def terminate_instances(cls, options):
        """Stops all services running in an AppScale deployment, and in cloud
    deployments, also powers off the instances previously spawned.

    Raises:
      AppScaleException: If AppScale is not running, and thus can't be
      terminated.
    """
        try:
            infrastructure = LocalState.get_infrastructure(options.keyname)
        except IOError:
            raise AppScaleException("Cannot find AppScale's configuration for keyname {0}".format(options.keyname))

        if infrastructure == "xen" and options.terminate:
            raise AppScaleException("Terminate option is invalid for cluster mode.")

        if infrastructure == "xen" or not options.terminate:
            # We are in cluster mode: let's check if AppScale is running.
            if not os.path.exists(LocalState.get_secret_key_location(options.keyname)):
                raise AppScaleException("AppScale is not running with the keyname {0}".format(options.keyname))

        # Stop gracefully the AppScale deployment.
        try:
            RemoteHelper.terminate_virtualized_cluster(options.keyname, options.verbose)
        except (IOError, AppScaleException):
            # Don't fail if we cannot find the configuration.
            pass

        # And if we are on a cloud infrastructure, terminate instances if
        # asked.
        if infrastructure in InfrastructureAgentFactory.VALID_AGENTS and options.terminate:
            RemoteHelper.terminate_cloud_infrastructure(options.keyname, options.verbose)

    def test_wait_for_machines_to_finish_loading(self):
        # mock out reading the secret key
        builtins = flexmock(sys.modules["__builtin__"])
        builtins.should_call("open")  # set the fall-through

        secret_key_location = LocalState.LOCAL_APPSCALE_PATH + "bookey.secret"
        fake_secret = flexmock(name="fake_secret")
        fake_secret.should_receive("read").and_return("the secret")
        builtins.should_receive("open").with_args(secret_key_location, "r").and_return(fake_secret)

        # mock out getting all the ips in the deployment from the head node
        fake_soap = flexmock(name="fake_soap")
        fake_soap.should_receive("get_all_public_ips").with_args("the secret").and_return(
            json.dumps(["public1", "public2"])
        )
        role_info = [
            {"public_ip": "public1", "private_ip": "private1", "jobs": ["shadow", "db_master"]},
            {"public_ip": "public2", "private_ip": "private2", "jobs": ["appengine"]},
        ]
        fake_soap.should_receive("get_role_info").with_args("the secret").and_return(json.dumps(role_info))

        # also, let's say that our machines aren't running the first time we ask,
        # but that they are the second time
        fake_soap.should_receive("is_done_initializing").with_args("the secret").and_return(False).and_return(True)

        flexmock(SOAPpy)
        SOAPpy.should_receive("SOAPProxy").with_args("https://public1:17443").and_return(fake_soap)
        SOAPpy.should_receive("SOAPProxy").with_args("https://public2:17443").and_return(fake_soap)

        RemoteHelper.wait_for_machines_to_finish_loading("public1", "bookey")

    def test_start_remote_appcontroller(self):
        # mock out removing the old json file
        local_state = flexmock(LocalState)
        local_state.should_receive("shell").with_args(
            re.compile("^ssh"), False, 5, stdin=re.compile("rm -rf")
        ).and_return().ordered()

        # assume we started god on public1 fine
        local_state.should_receive("shell").with_args(
            re.compile("^ssh"), False, 5, stdin=re.compile("nohup god")
        ).and_return().ordered()

        # also assume that we scp'ed over the god config file fine
        local_state.should_receive("shell").with_args(
            re.compile("scp .*appcontroller\.god.*"), False, 5
        ).and_return().ordered()

        # and assume we started the AppController on public1 fine
        local_state.should_receive("shell").with_args(
            re.compile("^ssh"), False, 5, stdin=re.compile("^god load .*appcontroller\.god")
        ).and_return().ordered()

        # finally, assume the appcontroller comes up after a few tries
        # assume that ssh comes up on the third attempt
        fake_socket = flexmock(name="fake_socket")
        fake_socket.should_receive("connect").with_args(("public1", AppControllerClient.PORT)).and_raise(
            Exception
        ).and_raise(Exception).and_return(None)
        socket.should_receive("socket").and_return(fake_socket)

        RemoteHelper.start_remote_appcontroller("public1", "bookey", False)

    def test_copy_deployment_credentials_in_cloud(self):
        # mock out the scp'ing to public1 and assume they succeed
        local_state = flexmock(LocalState)
        local_state.should_receive("shell").with_args(re.compile("^scp .*secret.key"), True, 5).and_return().ordered()

        local_state.should_receive("shell").with_args(re.compile("^scp .*ssh.key"), True, 5).and_return().ordered()

        # mock out generating the private key
        local_state = flexmock(LocalState)
        local_state.should_receive("shell").with_args(re.compile("^openssl"), True, stdin=None).and_return().ordered()

        local_state.should_receive("shell").with_args(re.compile("^scp .*mycert.pem"), True, 5).and_return().ordered()

        local_state.should_receive("shell").with_args(re.compile("^scp .*mykey.pem"), True, 5).and_return().ordered()

        # next, mock out copying the private key and certificate
        local_state.should_receive("shell").with_args(
            re.compile("^ssh"), True, 5, stdin=re.compile("^mkdir -p")
        ).and_return().ordered()

        local_state.should_receive("shell").with_args(
            re.compile("^scp .*cloud1/mycert.pem"), True, 5
        ).and_return().ordered()

        local_state.should_receive("shell").with_args(
            re.compile("^scp .*cloud1/mykey.pem"), True, 5
        ).and_return().ordered()

        options = flexmock(name="options", keyname="bookey", infrastructure="ec2", verbose=True)
        RemoteHelper.copy_deployment_credentials("public1", options)

  def testCleanInClusterDeployment(self):
    # calling 'appscale clean' in a cluster deployment should ssh into each of
    # the boxes specified in the ips_layout and run the terminate script

    # Mock out the actual file reading itself, and slip in a YAML-dumped
    # file
    contents = {
      'ips_layout' : {
        'controller': 'public1',
        'servers': ['public2', 'public3']
      },
      'test' : True
    }
    yaml_dumped_contents = yaml.dump(contents)

    flexmock(RemoteHelper)
    RemoteHelper.should_receive('ssh') \
      .with_args(re.compile('public[123]'), 'appscale', str, False)

    flexmock(LocalState)
    LocalState.should_receive('cleanup_appscale_files').with_args('appscale')

    appscale = AppScale()
    self.addMockForAppScalefile(appscale, yaml_dumped_contents)
    expected = ['public1', 'public2', 'public3']
    self.assertEquals(expected, appscale.clean())

  def test_start_remote_appcontroller(self):
    # mock out removing the old json file
    subprocess.should_receive('Popen').with_args(re.compile('rm -rf'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # assume we started god on public1 fine
    subprocess.should_receive('Popen').with_args(re.compile('god &'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # also assume that we scp'ed over the god config file fine
    subprocess.should_receive('Popen').with_args(re.compile('appcontroller'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # and assume we started the AppController on public1 fine
    subprocess.should_receive('Popen').with_args(re.compile('god load'),
      shell=True, stdout=self.fake_temp_file, stderr=subprocess.STDOUT) \
      .and_return(self.success)

    # finally, assume the appcontroller comes up after a few tries
    # assume that ssh comes up on the third attempt
    fake_socket = flexmock(name='fake_socket')
    fake_socket.should_receive('connect').with_args(('public1',
      AppControllerClient.PORT)).and_raise(Exception) \
      .and_raise(Exception).and_return(None)
    socket.should_receive('socket').and_return(fake_socket)

    RemoteHelper.start_remote_appcontroller('public1', 'bookey', False)

 def test_copy_deployment_credentials_in_cloud(self):
   # mock out the scp'ing to public1 and assume they succeed
   local_state = flexmock(LocalState)
   local_state.should_receive('shell').and_return().ordered()
 
   options = flexmock(name='options', keyname='bookey', infrastructure='ec2',
     verbose=True)
   RemoteHelper.copy_deployment_credentials('public1', options)

  def valid_ssh_key(self, config, run_instances_opts):
    """ Checks if the tools can log into the head node with the current key.

    Args:
      config: A dictionary that includes the IPs layout (which itself is a dict
        mapping role names to IPs) and, optionally, the keyname to use.
      run_instances_opts: The arguments parsed from the appscale-run-instances
        command.

    Returns:
      A bool indicating whether or not the specified keyname can be used to log
      into the head node.

    Raises:
      BadConfigurationException: If the IPs layout was not a dictionary.
    """
    keyname = config['keyname']
    verbose = config.get('verbose', False)

    if not isinstance(config['ips_layout'], dict):
      raise BadConfigurationException(
        'ips_layout should be a dictionary. Please fix it and try again.')

    ssh_key_location = self.APPSCALE_DIRECTORY + keyname + ".key"
    if not os.path.exists(ssh_key_location):
      return False

    all_ips = LocalState.get_all_public_ips(keyname)

    # If a login node is defined, use that to communicate with other nodes.
    node_layout = NodeLayout(run_instances_opts)
    head_node = node_layout.head_node()
    if head_node is not None:
      remote_key = '{}/ssh.key'.format(RemoteHelper.CONFIG_DIR)
      try:
        RemoteHelper.scp(
          head_node.public_ip, keyname, ssh_key_location, remote_key, verbose)
      except ShellException:
        return False

      for ip in all_ips:
        ssh_to_ip = 'ssh -i {key} -o StrictHostkeyChecking=no [email protected]{ip} true'\
          .format(key=remote_key, ip=ip)
        try:
          RemoteHelper.ssh(
            head_node.public_ip, keyname, ssh_to_ip, verbose, user='root')
        except ShellException:
          return False
      return True

    for ip in all_ips:
      if not self.can_ssh_to_ip(ip, keyname, verbose):
        return False

    return True

    def test_rsync_files_from_dir_that_does_exist(self):
        # if the user specifies that we should copy from a directory that does
        # exist, and has all the right directories in it, we should succeed
        flexmock(os.path)
        os.path.should_receive("exists").with_args(re.compile("/tmp/booscale-local/")).and_return(True)

        # assume the rsyncs succeed
        local_state = flexmock(LocalState)
        local_state.should_receive("shell").with_args(re.compile("^rsync"), False).and_return().ordered()

        RemoteHelper.rsync_files("public1", "booscale", "/tmp/booscale-local", False)