Python AmbariConfig.AmbariConfig类代码示例

  def test_build(self, read_actual_component_mock):
    for component in LiveStatus.COMPONENTS:
      config = AmbariConfig().getConfig()
      config.set('agent', 'prefix', "ambari_agent" + os.sep + "dummy_files")
      livestatus = LiveStatus('', component['serviceName'], component['componentName'], {}, config, {})
      livestatus.versionsHandler.versionsFilePath = "ambari_agent" + os.sep + "dummy_files" + os.sep + "dummy_current_stack"
      result = livestatus.build()
      print "LiveStatus of {0}: {1}".format(component['serviceName'], str(result))
      self.assertEquals(len(result) > 0, True, 'Livestatus should not be empty')
      if component['componentName'] == 'GANGLIA_SERVER':
        self.assertEquals(result['stackVersion'],'{"stackName":"HDP","stackVersion":"1.2.2"}',
                      'Livestatus should contain component stack version')

    # Test build status for CLIENT component (in LiveStatus.CLIENT_COMPONENTS)
    read_actual_component_mock.return_value = "some tags"
    livestatus = LiveStatus('c1', 'HDFS', 'HDFS_CLIENT', { }, config, {})
    result = livestatus.build()
    self.assertTrue(len(result) > 0, 'Livestatus should not be empty')
    self.assertTrue(result.has_key('configurationTags'))
    # Test build status with forsed_component_status
    ## Alive
    livestatus = LiveStatus('c1', 'HDFS', 'HDFS_CLIENT', { }, config, {})
    result = livestatus.build(forsed_component_status = LiveStatus.LIVE_STATUS)
    self.assertTrue(len(result) > 0, 'Livestatus should not be empty')
    self.assertTrue(result['status'], LiveStatus.LIVE_STATUS)
    ## Dead
    livestatus = LiveStatus('c1', 'HDFS', 'HDFS_CLIENT', { }, config, {})
    result = livestatus.build(forsed_component_status = LiveStatus.DEAD_STATUS)
    self.assertTrue(len(result) > 0, 'Livestatus should not be empty')
    self.assertTrue(result['status'], LiveStatus.DEAD_STATUS)

    livestatus = LiveStatus('c1', 'TEZ', 'TEZ_CLIENT', { }, config, {})
    result = livestatus.build(forsed_component_status = LiveStatus.LIVE_STATUS)
    self.assertTrue(len(result) > 0, 'Livestatus should not be empty')
    self.assertTrue(result['status'], LiveStatus.LIVE_STATUS)

 def test_status_command_without_globals_section(self, stopped_method,
                                                 read_stack_version_method):
   config = AmbariConfig().getConfig()
   config.set('agent', 'prefix', TestStackVersionsFileHandler.dummyVersionsFile)
   queue = ActionQueue(config)
   statusCommand = {
     "serviceName" : 'HDFS',
     "commandType" : "STATUS_COMMAND",
     "clusterName" : "",
     "componentName" : "DATANODE",
     'configurations':{}
   }
   queue.stopped = stopped_method
   stopped_method.side_effect = [False, False, True, True, True]
   read_stack_version_method.return_value="1.3.0"
   queue.IDLE_SLEEP_TIME = 0.001
   queue.put(statusCommand)
   queue.run()
   returned_result = queue.resultQueue.get()
   returned_result[1]['status'] = 'INSTALLED' # Patch live value
   self.assertEquals(returned_result, ('STATUS_COMMAND',
                                       {'clusterName': '',
                                        'componentName': 'DATANODE',
                                        'msg': '',
                                        'serviceName': 'HDFS',
                                        'stackVersion': '1.3.0',
                                        'status': 'INSTALLED'}))

  def test_read_write_component(self):
    config = AmbariConfig().getConfig()
    tmpdir = tempfile.gettempdir()
    config.set('agent', 'prefix', tmpdir)

    tags1 = { "global": "version1", "core-site": "version2" }
    handler = ActualConfigHandler(config, {})
    handler.write_actual(tags1)
    handler.write_actual_component('FOO', tags1)

    output1 = handler.read_actual_component('FOO')
    output2 = handler.read_actual_component('GOO')

    self.assertEquals(tags1, output1)
    self.assertEquals(None, output2)
    
    tags2 = { "global": "version1", "core-site": "version2" }
    handler.write_actual(tags2)

    output3 = handler.read_actual()
    output4 = handler.read_actual_component('FOO')
    self.assertEquals(tags2, output3)
    self.assertEquals(tags1, output4)
    os.remove(os.path.join(tmpdir, "FOO_" + ActualConfigHandler.CONFIG_NAME))
    os.remove(os.path.join(tmpdir, ActualConfigHandler.CONFIG_NAME))

  def test_registration_build(self, get_os_version_mock, get_os_type_mock, run_os_cmd_mock):
    config = AmbariConfig().getConfig()
    tmpdir = tempfile.gettempdir()
    config.set('agent', 'prefix', tmpdir)
    config.set('agent', 'current_ping_port', '33777')
    get_os_type_mock.return_value = "suse"
    get_os_version_mock.return_value = "11"
    run_os_cmd_mock.return_value = (3, "", "")
    ver_file = os.path.join(tmpdir, "version")
    with open(ver_file, "w") as text_file:
      text_file.write("1.3.0")

    register = Register(config)
    data = register.build(1)
    #print ("Register: " + pprint.pformat(data))
    self.assertEquals(len(data['hardwareProfile']) > 0, True, "hardwareProfile should contain content")
    self.assertEquals(data['hostname'] != "", True, "hostname should not be empty")
    self.assertEquals(data['publicHostname'] != "", True, "publicHostname should not be empty")
    self.assertEquals(data['responseId'], 1)
    self.assertEquals(data['timestamp'] > 1353678475465L, True, "timestamp should not be empty")
    self.assertEquals(len(data['agentEnv']) > 0, True, "agentEnv should not be empty")
    self.assertEquals(data['agentVersion'], '1.3.0', "agentVersion should not be empty")
    print data['agentEnv']['umask']
    self.assertEquals(not data['agentEnv']['umask']== "", True, "agents umask should not be empty")
    self.assertEquals(data['currentPingPort'] == 33777, True, "current ping port should be 33777")
    self.assertEquals(len(data), 8)

    os.remove(ver_file)

  def test_RetryAction(self):
    action={'id' : 'tttt'}
    config = AmbariConfig().getConfig()
    actionQueue = ActionQueue(config)
    path = actionQueue.getInstallFilename(action['id'])
    configFile = {
      "data"       : "test",
      "owner"      : os.getuid(),
      "group"      : os.getgid() ,
      "permission" : 0700,
      "path"       : path,
      "umask"      : 022
    }

    #note that the command in the action is just a listing of the path created
    #we just want to ensure that 'ls' can run on the data file (in the actual world
    #this 'ls' would be a puppet or a chef command that would work on a data
    #file
    badAction = {
      'id' : 'tttt',
      'kind' : 'INSTALL_AND_CONFIG_ACTION',
      'workDirComponent' : 'abc-hdfs',
      'file' : configFile,
      'clusterDefinitionRevision' : 12,
      'command' : ['/bin/ls',"/foo/bar/badPath1234"]
    }
    path=getFilePath(action,path)
    goodAction = {
      'id' : 'tttt',
      'kind' : 'INSTALL_AND_CONFIG_ACTION',
      'workDirComponent' : 'abc-hdfs',
      'file' : configFile,
      'clusterDefinitionRevision' : 12,
      'command' : ['/bin/ls',path]
    }
    actionQueue.start()
    response = {'actions' : [badAction,goodAction]}
    actionQueue.maxRetries = 2
    actionQueue.sleepInterval = 1
    result = actionQueue.put(response)
    results = actionQueue.result()
    sleptCount = 1
    while (len(results) < 2 and sleptCount < 15):
        time.sleep(1)
        sleptCount += 1
        results = actionQueue.result()
    actionQueue.stop()
    actionQueue.join()
    self.assertEqual(len(results), 2, 'Number of results is not 2.')
    result = results[0]
    maxretries = config.get('command', 'maxretries')
    self.assertEqual(int(result['retryActionCount']), 
                     int(maxretries),
                     "Number of retries is %d and not %d" % 
                     (int(result['retryActionCount']), int(str(maxretries))))
    result = results[1]
    self.assertEqual(int(result['retryActionCount']), 
                     1,
                     "Number of retries is %d and not %d" % 
                     (int(result['retryActionCount']), 1))        

def setup(options):
  config = AmbariConfig()
  configFile = config.getConfigFile()

  updateConfigServerHostname(configFile, options.host_name)

  AmbariAgentService.set_ctrl_c_handler(ctrlHandler)
  AmbariAgentService.Install()

 def test_add_reg_listener_to_controller(self, FileCache_mock):
   FileCache_mock.return_value = None
   dummy_controller = MagicMock()
   config = AmbariConfig().getConfig()
   tempdir = tempfile.gettempdir()
   config.set('agent', 'prefix', tempdir)
   CustomServiceOrchestrator(config, dummy_controller)
   self.assertTrue(dummy_controller.registration_listeners.append.called)

 def test_dump_command_to_json(self, FileCache_mock, unlink_mock,
                               isfile_mock, hostname_mock,
                               decompress_cluster_host_info_mock):
   FileCache_mock.return_value = None
   hostname_mock.return_value = "test.hst"
   command = {
     'commandType': 'EXECUTION_COMMAND',
     'role': u'DATANODE',
     'roleCommand': u'INSTALL',
     'commandId': '1-1',
     'taskId': 3,
     'clusterName': u'cc',
     'serviceName': u'HDFS',
     'configurations':{'global' : {}},
     'configurationTags':{'global' : { 'tag': 'v1' }},
     'clusterHostInfo':{'namenode_host' : ['1'],
                        'slave_hosts'   : ['0', '1'],
                        'all_hosts'     : ['h1.hortonworks.com', 'h2.hortonworks.com'],
                        'all_ping_ports': ['8670:0,1']},
     'hostLevelParams':{}
   }
   
   decompress_cluster_host_info_mock.return_value = {'namenode_host' : ['h2.hortonworks.com'],
                        'slave_hosts'   : ['h1.hortonworks.com', 'h2.hortonworks.com'],
                        'all_hosts'     : ['h1.hortonworks.com', 'h2.hortonworks.com'],
                        'all_ping_ports': ['8670', '8670']}
   
   config = AmbariConfig()
   tempdir = tempfile.gettempdir()
   config.set('agent', 'prefix', tempdir)
   dummy_controller = MagicMock()
   orchestrator = CustomServiceOrchestrator(config, dummy_controller)
   isfile_mock.return_value = True
   # Test dumping EXECUTION_COMMAND
   json_file = orchestrator.dump_command_to_json(command)
   self.assertTrue(os.path.exists(json_file))
   self.assertTrue(os.path.getsize(json_file) > 0)
   if get_platform() != PLATFORM_WINDOWS:
     self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
   self.assertTrue(json_file.endswith("command-3.json"))
   self.assertTrue(decompress_cluster_host_info_mock.called)
   os.unlink(json_file)
   # Test dumping STATUS_COMMAND
   command['commandType']='STATUS_COMMAND'
   decompress_cluster_host_info_mock.reset_mock()
   json_file = orchestrator.dump_command_to_json(command)
   self.assertTrue(os.path.exists(json_file))
   self.assertTrue(os.path.getsize(json_file) > 0)
   if get_platform() != PLATFORM_WINDOWS:
     self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
   self.assertTrue(json_file.endswith("status_command.json"))
   self.assertFalse(decompress_cluster_host_info_mock.called)
   os.unlink(json_file)
   # Testing side effect of dump_command_to_json
   self.assertEquals(command['public_hostname'], "test.hst")
   self.assertEquals(command['agentConfigParams']['agent']['parallel_execution'], 0)
   self.assertTrue(unlink_mock.called)

def run_simulation():
  Controller.logger = MagicMock()
  sendRequest_method = MagicMock()

  tmpfile = tempfile.gettempdir()

  config = AmbariConfig().getConfig()
  config.set('agent', 'prefix', tmpfile)

  scriptsDir = os.path.join(os.getcwd(), os.pardir,os.pardir,
    os.pardir, 'main', 'upgrade_stack')
  config.set('stack', 'upgradeScriptsDir', scriptsDir)

  ver_file = os.path.join(tmpfile, "version")

  with open(ver_file, "w") as text_file:
      text_file.write(agent_version)

  controller = Controller.Controller(config)
  controller.sendRequest = sendRequest_method
  controller.netutil.HEARTBEAT_IDDLE_INTERVAL_SEC = 0.1
  controller.netutil.HEARTBEAT_NOT_IDDLE_INTERVAL_SEC = 0.1
  controller.range = 1

  for responce in responces:
    queue.put(responce)

  def send_stub(url, data):
    logger.info("Controller sends data to %s :" % url)
    logger.info(pprint.pformat(data))
    if not queue.empty():
      responce = queue.get()
    else:
      responce = responces[-1]
      logger.info("There is no predefined responce available, sleeping for 30 sec")
      time.sleep(30)
    responce = json.loads(responce)
    responseId.inc()
    responce["responseId"] = responseId.val()
    responce = json.dumps(responce)
    logger.info("Returning data to Controller:" + responce)
    return responce

  sendRequest_method.side_effect = send_stub

  logger.setLevel(logging.DEBUG)
  formatter = logging.Formatter("%(asctime)s %(filename)s:%(lineno)d - \
        %(message)s")
  stream_handler = logging.StreamHandler()
  stream_handler.setFormatter(formatter)
  logger.addHandler(stream_handler)
  logger.info("Starting")

  controller.start()
  controller.actionQueue.IDLE_SLEEP_TIME = 0.1
  controller.run()

  def test_update_log_level(self, basicConfig_mock, setLevel_mock):
    config = AmbariConfig().getConfig()

    # Testing with default setup (config file does not contain loglevel entry)
    # Log level should not be changed
    config.set('agent', 'loglevel', None)
    main.update_log_level(config)
    self.assertFalse(setLevel_mock.called)

    setLevel_mock.reset_mock()

    # Testing debug mode
    config.set('agent', 'loglevel', 'DEBUG')
    main.update_log_level(config)
    setLevel_mock.assert_called_with(logging.DEBUG)
    setLevel_mock.reset_mock()

    # Testing any other mode
    config.set('agent', 'loglevel', 'INFO')
    main.update_log_level(config)
    setLevel_mock.assert_called_with(logging.INFO)

    setLevel_mock.reset_mock()

    config.set('agent', 'loglevel', 'WRONG')
    main.update_log_level(config)
    setLevel_mock.assert_called_with(logging.INFO)

  def test_process_command(self, execute_status_command_mock,
                           execute_command_mock, print_exc_mock):
    dummy_controller = MagicMock()
    config = AmbariConfig()
    config.set('agent', 'tolerate_download_failures', "true")
    actionQueue = ActionQueue(config, dummy_controller)
    execution_command = {
      'commandType' : ActionQueue.EXECUTION_COMMAND,
    }
    status_command = {
      'commandType' : ActionQueue.STATUS_COMMAND,
    }
    wrong_command = {
      'commandType' : "SOME_WRONG_COMMAND",
    }
    # Try wrong command
    actionQueue.process_command(wrong_command)
    self.assertFalse(execute_command_mock.called)
    self.assertFalse(execute_status_command_mock.called)
    self.assertFalse(print_exc_mock.called)

    execute_command_mock.reset_mock()
    execute_status_command_mock.reset_mock()
    print_exc_mock.reset_mock()
    # Try normal execution
    actionQueue.process_command(execution_command)
    self.assertTrue(execute_command_mock.called)
    self.assertFalse(execute_status_command_mock.called)
    self.assertFalse(print_exc_mock.called)

    execute_command_mock.reset_mock()
    execute_status_command_mock.reset_mock()
    print_exc_mock.reset_mock()

    actionQueue.process_command(status_command)
    self.assertFalse(execute_command_mock.called)
    self.assertTrue(execute_status_command_mock.called)
    self.assertFalse(print_exc_mock.called)

    execute_command_mock.reset_mock()
    execute_status_command_mock.reset_mock()
    print_exc_mock.reset_mock()

    # Try exception to check proper logging
    def side_effect(self):
      raise Exception("TerribleException")
    execute_command_mock.side_effect = side_effect
    actionQueue.process_command(execution_command)
    self.assertTrue(print_exc_mock.called)

    print_exc_mock.reset_mock()

    execute_status_command_mock.side_effect = side_effect
    actionQueue.process_command(execution_command)
    self.assertTrue(print_exc_mock.called)

  def test_read_write(self):
    config = AmbariConfig().getConfig()
    tmpdir = tempfile.gettempdir()
    config.set('agent', 'prefix', tmpdir)

    tags = { "global": "version1", "core-site": "version2" }
    handler = ActualConfigHandler(config, tags)
    handler.write_actual(tags)
    output = handler.read_actual()
    self.assertEquals(tags, output)
    os.remove(os.path.join(tmpdir, ActualConfigHandler.CONFIG_NAME))

 def test_read_agent_version(self, get_os_version_mock, get_os_type_mock):
   config = AmbariConfig().getConfig()
   tmpdir = tempfile.gettempdir()
   config.set('agent', 'prefix', tmpdir)
   config.set('agent', 'current_ping_port', '33777')
   ver_file = os.path.join(tmpdir, "version")
   reference_version = "1.3.0"
   with open(ver_file, "w") as text_file:
     text_file.write(reference_version)
   version = self.controller.read_agent_version(config)
   os.remove(ver_file)
   self.assertEqual(reference_version, version)

  def test_command_in_progress(self):
    config = AmbariConfig().getConfig()
    tmpfile = tempfile.gettempdir()
    config.set('agent', 'prefix', tmpfile)
    actionQueue = ActionQueue(config)
    actionQueue.IDLE_SLEEP_TIME = 0.01
    executor_started_event = threading.Event()
    end_executor_event = threading.Event()
    actionQueue.puppetExecutor = FakeExecutor(executor_started_event, end_executor_event)
    before_start_result = actionQueue.result()

    command = {
      'commandId': 17,
      'role' : "role",
      'taskId' : "taskId",
      'clusterName' : "clusterName",
      'serviceName' : "serviceName",
      'status' : 'IN_PROGRESS',
      'hostname' : "localhost.localdomain",
      'hostLevelParams': "hostLevelParams",
      'clusterHostInfo': "clusterHostInfo",
      'roleCommand': "roleCommand",
      'configurations': "configurations",
      'commandType': "EXECUTION_COMMAND",
      'configurations':{'global' : {}}
    }
    actionQueue.put(command)

    actionQueue.start()
    executor_started_event.wait()
    #print ("ii: " + pprint.pformat(actionQueue.commandInProgress))
    in_progress_result = actionQueue.result()
    end_executor_event.set()
    actionQueue.stop()
    actionQueue.join()
    after_start_result = actionQueue.result()

    self.assertEquals(len(before_start_result['componentStatus']), 0)
    self.assertEquals(len(before_start_result['reports']), 0)

    self.assertEquals(len(in_progress_result['componentStatus']), 0)
    self.assertEquals(len(in_progress_result['reports']), 1)
    self.assertEquals(in_progress_result['reports'][0]['status'], "IN_PROGRESS")
    self.assertEquals(in_progress_result['reports'][0]['stdout'], "Dummy output")
    self.assertEquals(in_progress_result['reports'][0]['exitCode'], 777)
    self.assertEquals(in_progress_result['reports'][0]['stderr'], 'Dummy err')

    self.assertEquals(len(after_start_result['componentStatus']), 0)
    self.assertEquals(len(after_start_result['reports']), 1)
    self.assertEquals(after_start_result['reports'][0]['status'], "COMPLETED")
    self.assertEquals(after_start_result['reports'][0]['stdout'], "returned stdout")
    self.assertEquals(after_start_result['reports'][0]['exitCode'], 0)
    self.assertEquals(after_start_result['reports'][0]['stderr'], 'returned stderr')

 def test_build(self):
   for component in LiveStatus.COMPONENTS:
     config = AmbariConfig().getConfig()
     config.set('agent', 'prefix', "dummy_files")
     livestatus = LiveStatus('', component['serviceName'], component['componentName'], {}, config)
     livestatus.versionsHandler.versionsFilePath = os.path.join("dummy_files","dummy_current_stack")
     result = livestatus.build()
     print "LiveStatus of {0}: {1}".format(component['serviceName'], str(result))
     self.assertEquals(len(result) > 0, True, 'Livestatus should not be empty')
     if component['componentName'] == 'GANGLIA_SERVER':
       self.assertEquals(result['stackVersion'],'{"stackName":"HDP","stackVersion":"1.2.2"}',
                     'Livestatus should contain component stack version')