Python CustomServiceOrchestrator.CustomServiceOrchestrator类代码示例

  def test_runCommand_background_action(self, get_custom_actions_base_dir_mock,
                                    FileCache_mock,
                                    dump_command_to_json_mock):
    FileCache_mock.return_value = None
    get_custom_actions_base_dir_mock.return_value = "some path"
    _, script = tempfile.mkstemp()
    command = {
      'role' : 'any',
      'commandParams': {
        'script_type': 'PYTHON',
        'script': 'some_custom_action.py',
        'command_timeout': '600',
        'jdk_location' : 'some_location'
      },
      'taskId' : '13',
      'roleCommand': 'ACTIONEXECUTE',
      'commandType': 'BACKGROUND_EXECUTION_COMMAND',
      '__handle': BackgroundCommandExecutionHandle({'taskId': '13'}, 13,
                                                   MagicMock(), MagicMock())
    }
    dummy_controller = MagicMock()
    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)

    import TestActionQueue
    TestActionQueue.patch_output_file(orchestrator.python_executor)
    orchestrator.python_executor.condenseOutput = MagicMock()
    orchestrator.dump_command_to_json = MagicMock()

    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
    self.assertEqual(ret['exitcode'], 777)

  def test_requestComponentSecurityState(self, FileCache_mock, runCommand_mock):
    FileCache_mock.return_value = None
    status_command = {
      "serviceName" : 'HDFS',
      "commandType" : "STATUS_COMMAND",
      "clusterName" : "",
      "componentName" : "DATANODE",
      'configurations':{}
    }
    dummy_controller = MagicMock()
    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
    # Test securityState
    runCommand_mock.return_value = {
      'exitcode' : 0,
      'structuredOut' : {'securityState': 'UNSECURED'}
    }

    status = orchestrator.requestComponentSecurityState(status_command)
    self.assertEqual('UNSECURED', status)

    # Test case where exit code indicates failure
    runCommand_mock.return_value = {
      "exitcode" : 1
    }
    status = orchestrator.requestComponentSecurityState(status_command)
    self.assertEqual('UNKNOWN', status)

  def test_parse_allowed_port_values(self):
    dummy_controller = MagicMock()
    tempdir = tempfile.gettempdir()
    tempWorkDir = tempdir + "W"
    config = MagicMock()
    config.get.return_value = "something"
    config.getResolvedPath.return_value = tempdir
    config.getWorkRootPath.return_value = tempWorkDir
    config.getLogPath.return_value = tempdir

    orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
    port_range = "48000-48005"
    port_range_full_list = [48000, 48001, 48002, 48003, 48004, 48005]
    allowed_ports = orchestrator.get_allowed_port_list(port_range, 3)
    self.assertTrue(set(allowed_ports).issubset(port_range_full_list))

    port_range = "48000 , 48005"
    port_range_full_list = [48000, 48005]
    allowed_ports = orchestrator.get_allowed_port_list(port_range, 1)
    self.assertTrue(set(allowed_ports).issubset(port_range_full_list))

    port_range = "48000 , 48004-48005"
    port_range_full_list = [48000, 48004, 48005]
    allowed_ports = orchestrator.get_allowed_port_list(port_range, 2)
    self.assertTrue(set(allowed_ports).issubset(port_range_full_list))

  def test_cancel_backgound_command(self, read_stack_version_mock, resolve_hook_script_path_mock, resolve_script_path_mock, FileCache_mock,
                                      kill_process_with_children_mock):
    FileCache_mock.return_value = None
    FileCache_mock.cache_dir = MagicMock()
    resolve_hook_script_path_mock.return_value = None
#     shell.kill_process_with_children = MagicMock()
    dummy_controller = MagicMock()
    cfg = AmbariConfig().getConfig()
    cfg.set('agent', 'tolerate_download_failures', 'true')
    cfg.set('agent', 'prefix', '.')
    cfg.set('agent', 'cache_dir', 'background_tasks')

    actionQueue = ActionQueue(cfg, dummy_controller)

    dummy_controller.actionQueue = actionQueue
    orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
    orchestrator.file_cache = MagicMock()
    def f (a, b):
      return ""
    orchestrator.file_cache.get_service_base_dir = f
    actionQueue.customServiceOrchestrator = orchestrator

    import TestActionQueue
    import copy

    TestActionQueue.patch_output_file(orchestrator.python_executor)
    orchestrator.python_executor.prepare_process_result = MagicMock()
    orchestrator.dump_command_to_json = MagicMock()

    lock = threading.RLock()
    complete_done = threading.Condition(lock)

    complete_was_called = {}
    def command_complete_w(process_condenced_result, handle):
      with lock:
        complete_was_called['visited']= ''
        complete_done.wait(3)

    actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(actionQueue.on_background_command_complete_callback, command_complete_w, None)
    execute_command = copy.deepcopy(TestActionQueue.TestActionQueue.background_command)
    actionQueue.put([execute_command])
    actionQueue.processBackgroundQueueSafeEmpty()

    time.sleep(.1)

    orchestrator.cancel_command(19,'')
    self.assertTrue(kill_process_with_children_mock.called)
    kill_process_with_children_mock.assert_called_with(33)

    with lock:
      complete_done.notifyAll()

    with lock:
      self.assertTrue(complete_was_called.has_key('visited'))

    time.sleep(.1)

    runningCommand = actionQueue.commandStatuses.get_command_status(19)
    self.assertTrue(runningCommand is not None)
    self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)

 def test_runCommand_custom_action(self, get_custom_actions_base_dir_mock,
                                   FileCache_mock,
                                   run_file_mock, dump_command_to_json_mock):
   FileCache_mock.return_value = None
   get_custom_actions_base_dir_mock.return_value = "some path"
   _, script = tempfile.mkstemp()
   command = {
     'role' : 'any',
     'commandParams': {
       'script_type': 'PYTHON',
       'script': 'some_custom_action.py',
       'command_timeout': '600',
       'jdk_location' : 'some_location'
     },
     'taskId' : '3',
     'roleCommand': 'ACTIONEXECUTE'
   }
   dummy_controller = MagicMock()
   orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
   # normal run case
   run_file_mock.return_value = {
     'stdout' : 'sss',
     'stderr' : 'eee',
     'exitcode': 0,
     }
   ret = orchestrator.runCommand(command, "out.txt", "err.txt")
   self.assertEqual(ret['exitcode'], 0)
   self.assertTrue(run_file_mock.called)
   # Hoooks are not supported for custom actions,
   # that's why run_file() should be called only once
   self.assertEqual(run_file_mock.call_count, 1)

  def test_finalize_command(self):
    dummy_controller = MagicMock()
    tempdir = tempfile.gettempdir()
    tempWorkDir = tempdir + "W"
    config = MagicMock()
    config.get.return_value = "something"
    config.getResolvedPath.return_value = tempdir
    config.getWorkRootPath.return_value = tempWorkDir
    config.getLogPath.return_value = tempdir

    orchestrator = CustomServiceOrchestrator(config, dummy_controller)
    command = {}
    command['configurations'] = {}
    command['configurations']['hbase-site'] = {}
    command['configurations']['hbase-site']['a'] = 'b'
    command['configurations']['hbase-site']['work_root'] = "${AGENT_WORK_ROOT}"
    command['configurations']['hbase-site']['log_root'] = "${AGENT_LOG_ROOT}/log"
    command['configurations']['hbase-site']['blog_root'] = "/b/${AGENT_LOG_ROOT}/log"
    command['configurations']['oozie-site'] = {}
    command['configurations']['oozie-site']['log_root'] = "${AGENT_LOG_ROOT}"

    orchestrator.finalize_command(command, False)
    self.assertEqual(command['configurations']['hbase-site']['work_root'], tempWorkDir)
    self.assertEqual(command['configurations']['oozie-site']['log_root'], tempdir)
    self.assertEqual(orchestrator.applied_configs, {})

    command['configurations']['hbase-site']['work_root'] = "${AGENT_WORK_ROOT}"
    command['configurations']['hbase-site']['log_root'] = "${AGENT_LOG_ROOT}/log"
    command['configurations']['hbase-site']['blog_root'] = "/b/${AGENT_LOG_ROOT}/log"
    command['configurations']['oozie-site']['log_root'] = "${AGENT_LOG_ROOT}"

    orchestrator.finalize_command(command, True)
    self.assertEqual(command['configurations']['hbase-site']['log_root'], tempdir + "/log")
    self.assertEqual(command['configurations']['hbase-site']['blog_root'], "/b/" + tempdir + "/log")
    self.assertEqual(orchestrator.applied_configs, command['configurations'])

  def test_requestComponentStatus(self, runCommand_mock):
    status_command = {
      "serviceName": 'HDFS',
      "commandType": "STATUS_COMMAND",
      "clusterName": "",
      "componentName": "DATANODE",
      'configurations': {},
      'roleCommand': "STATUS"
    }
    dummy_controller = MagicMock()

    tempdir = tempfile.gettempdir()
    config = MagicMock()
    config.get.return_value = "something"
    config.getResolvedPath.return_value = tempdir
    config.getWorkRootPath.return_value = tempdir
    config.getLogPath.return_value = tempdir

    orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
    # Test alive case
    runCommand_mock.return_value = {
      "exitcode": 0
    }
    status = orchestrator.requestComponentStatus(status_command)
    self.assertEqual(CustomServiceOrchestrator.LIVE_STATUS, status['exitcode'])

    # Test dead case
    runCommand_mock.return_value = {
      "exitcode": 1
    }
    status = orchestrator.requestComponentStatus(status_command)
    self.assertEqual(CustomServiceOrchestrator.DEAD_STATUS, status['exitcode'])

  def test_requestComponentStatus(self, FileCache_mock, runCommand_mock):
    FileCache_mock.return_value = None
    status_command = {
      "serviceName" : 'HDFS',
      "commandType" : "STATUS_COMMAND",
      "clusterName" : "",
      "componentName" : "DATANODE",
      'configurations':{}
    }
    dummy_controller = MagicMock()
    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
    # Test alive case
    runCommand_mock.return_value = {
      "exitcode" : 0
    }

    status = orchestrator.requestComponentStatus(status_command)
    self.assertEqual(runCommand_mock.return_value, status)

    # Test dead case
    runCommand_mock.return_value = {
      "exitcode" : 1
    }
    status = orchestrator.requestComponentStatus(status_command)
    self.assertEqual(runCommand_mock.return_value, status)

 def test_dump_command_to_json(self, FileCache_mock, unlink_mock,
                               isfile_mock, hostname_mock,
                               decompress_cluster_host_info_mock):
   FileCache_mock.return_value = None
   hostname_mock.return_value = "test.hst"
   command = {
     'commandType': 'EXECUTION_COMMAND',
     'role': u'DATANODE',
     'roleCommand': u'INSTALL',
     'commandId': '1-1',
     'taskId': 3,
     'clusterName': u'cc',
     'serviceName': u'HDFS',
     'configurations':{'global' : {}},
     'configurationTags':{'global' : { 'tag': 'v1' }},
     'clusterHostInfo':{'namenode_host' : ['1'],
                        'slave_hosts'   : ['0', '1'],
                        'all_hosts'     : ['h1.hortonworks.com', 'h2.hortonworks.com'],
                        'all_ping_ports': ['8670:0,1']},
     'hostLevelParams':{}
   }
   
   decompress_cluster_host_info_mock.return_value = {'namenode_host' : ['h2.hortonworks.com'],
                        'slave_hosts'   : ['h1.hortonworks.com', 'h2.hortonworks.com'],
                        'all_hosts'     : ['h1.hortonworks.com', 'h2.hortonworks.com'],
                        'all_ping_ports': ['8670', '8670']}
   
   config = AmbariConfig().getConfig()
   tempdir = tempfile.gettempdir()
   config.set('agent', 'prefix', tempdir)
   dummy_controller = MagicMock()
   orchestrator = CustomServiceOrchestrator(config, dummy_controller)
   isfile_mock.return_value = True
   # Test dumping EXECUTION_COMMAND
   json_file = orchestrator.dump_command_to_json(command)
   self.assertTrue(os.path.exists(json_file))
   self.assertTrue(os.path.getsize(json_file) > 0)
   if get_platform() != PLATFORM_WINDOWS:
     self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
   self.assertTrue(json_file.endswith("command-3.json"))
   self.assertTrue(decompress_cluster_host_info_mock.called)
   os.unlink(json_file)
   # Test dumping STATUS_COMMAND
   command['commandType']='STATUS_COMMAND'
   decompress_cluster_host_info_mock.reset_mock()
   json_file = orchestrator.dump_command_to_json(command)
   self.assertTrue(os.path.exists(json_file))
   self.assertTrue(os.path.getsize(json_file) > 0)
   if get_platform() != PLATFORM_WINDOWS:
     self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
   self.assertTrue(json_file.endswith("status_command.json"))
   self.assertFalse(decompress_cluster_host_info_mock.called)
   os.unlink(json_file)
   # Testing side effect of dump_command_to_json
   self.assertEquals(command['public_hostname'], "test.hst")
   self.assertTrue(unlink_mock.called)

  def test_finalize_command(self, mock_allocate_ports):
    dummy_controller = MagicMock()
    tempdir = tempfile.gettempdir()
    tempWorkDir = tempdir + "W"
    config = MagicMock()
    config.get.return_value = "something"
    config.getResolvedPath.return_value = tempdir
    config.getWorkRootPath.return_value = tempWorkDir
    config.getLogPath.return_value = tempdir
    mock_allocate_ports.return_value = "10023"

    orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
    command = {}
    command['componentName'] = "HBASE_MASTER"
    command['configurations'] = {}
    command['configurations']['hbase-site'] = {}
    command['configurations']['hbase-site']['a'] = 'b'
    command['configurations']['hbase-site']['work_root'] = "${AGENT_WORK_ROOT}"
    command['configurations']['hbase-site']['log_root'] = "${AGENT_LOG_ROOT}/log"
    command['configurations']['hbase-site']['blog_root'] = "/b/${AGENT_LOG_ROOT}/log"
    command['configurations']['oozie-site'] = {}
    command['configurations']['oozie-site']['log_root'] = "${AGENT_LOG_ROOT}"
    command['configurations']['oozie-site']['a_port'] = "${HBASE_MASTER.ALLOCATED_PORT}"
    command['configurations']['oozie-site']['ignore_port1'] = "[${HBASE_RS.ALLOCATED_PORT}]"
    command['configurations']['oozie-site']['ignore_port2'] = "[${HBASE_RS.ALLOCATED_PORT},${HBASE_REST.ALLOCATED_PORT}{PER_CONTAINER}]"
    command['configurations']['oozie-site']['ignore_port3'] = "[${HBASE_RS.ALLOCATED_PORT}{a}{b}{c},${A.ALLOCATED_PORT}{PER_CONTAINER},${A.ALLOCATED_PORT}{DEFAULT_3}{PER_CONTAINER}]"
    command['configurations']['oozie-site']['ignore_port4'] = "${HBASE_RS}{a}{b}{c}"

    allocated_ports = {}
    orchestrator.finalize_command(command, False, allocated_ports)
    self.assertEqual(command['configurations']['hbase-site']['work_root'], tempWorkDir)
    self.assertEqual(command['configurations']['oozie-site']['log_root'], tempdir)
    self.assertEqual(command['configurations']['oozie-site']['a_port'], "10023")
    self.assertEqual(command['configurations']['oozie-site']['ignore_port1'], "[0]")
    self.assertEqual(command['configurations']['oozie-site']['ignore_port2'], "[0,0]")
    self.assertEqual(command['configurations']['oozie-site']['ignore_port3'], "[0,0,0]")
    self.assertEqual(command['configurations']['oozie-site']['ignore_port4'], "${HBASE_RS}{a}{b}{c}")
    self.assertEqual(orchestrator.stored_command, {})
    self.assertEqual(len(allocated_ports), 1)
    self.assertTrue('oozie-site.a_port' in allocated_ports)
    self.assertEqual(allocated_ports['oozie-site.a_port'], '10023')

    command['configurations']['hbase-site']['work_root'] = "${AGENT_WORK_ROOT}"
    command['configurations']['hbase-site']['log_root'] = "${AGENT_LOG_ROOT}/log"
    command['configurations']['hbase-site']['blog_root'] = "/b/${AGENT_LOG_ROOT}/log"
    command['configurations']['oozie-site']['log_root'] = "${AGENT_LOG_ROOT}"
    command['configurations']['oozie-site']['b_port'] = "${HBASE_REGIONSERVER.ALLOCATED_PORT}"

    orchestrator.finalize_command(command, True, {})
    self.assertEqual(command['configurations']['hbase-site']['log_root'], tempdir + "/log")
    self.assertEqual(command['configurations']['hbase-site']['blog_root'], "/b/" + tempdir + "/log")
    self.assertEqual(command['configurations']['oozie-site']['b_port'], "0")
    self.assertEqual(orchestrator.stored_command, command)

  def test_allocate_port_no_def(self, socket_bind_mock, socket_getsockname_mock):
    tempdir = tempfile.gettempdir()
    config = MagicMock()
    config.get.return_value = "something"
    config.getResolvedPath.return_value = tempdir
    config.getWorkRootPath.return_value = tempdir
    config.getLogPath.return_value = tempdir

    dummy_controller = MagicMock()
    orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
    socket_getsockname_mock.return_value = [100, 102]
    ret = orchestrator.allocate_port()
    self.assertEqual(ret, 102)

  def test_allocate_port_def(self, socket_connect_mock, socket_close_mock):
    e = OSError()
    socket_connect_mock.side_effect = e
    tempdir = tempfile.gettempdir()
    config = MagicMock()
    config.get.return_value = "something"
    config.getResolvedPath.return_value = tempdir
    config.getWorkRootPath.return_value = tempdir
    config.getLogPath.return_value = tempdir

    dummy_controller = MagicMock()
    orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
    ret = orchestrator.allocate_port(10)
    self.assertEqual(ret, 10)

  def test_port_allocation(self):
    dummy_controller = MagicMock()
    tempdir = tempfile.gettempdir()
    tempWorkDir = tempdir + "W"
    config = MagicMock()
    config.get.return_value = "something"
    config.getResolvedPath.return_value = tempdir
    config.getWorkRootPath.return_value = tempWorkDir
    config.getLogPath.return_value = tempdir

    orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
    port = orchestrator.allocate_port()
    self.assertFalse(port == -1)
    self.assertTrue(port > 0)

  def test_runCommand_get_port(self,
                               run_file_mock,
                               resolve_script_path_mock,
                               allocate_ports_mock):
    command = {
      'role': 'HBASE_REGIONSERVER',
      'hostLevelParams': {
        'stack_name': 'HDP',
        'stack_version': '2.0.7',
        'jdk_location': 'some_location'
      },
      'commandParams': {
        'script_type': 'PYTHON',
        'script': 'scripts/hbase_regionserver.py',
        'command_timeout': '600',
        'service_package_folder': 'HBASE'
      },
      'taskId': '3',
      'roleCommand': 'INSTALL',
      'commandType': 'EXECUTE',
      'componentName': 'HBASE_REGIONSERVER',
      'configurations': {'a': {'a.port': '${HBASE_REGIONSERVER.ALLOCATED_PORT}'}}
    }

    tempdir = tempfile.gettempdir()
    config = MagicMock()
    config.get.return_value = "something"
    config.getResolvedPath.return_value = tempdir
    config.getWorkRootPath.return_value = tempdir
    config.getLogPath.return_value = tempdir

    allocate_ports_mock.return_value = str(10233)

    resolve_script_path_mock.return_value = "/basedir/scriptpath"
    dummy_controller = MagicMock()
    orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
    # normal run case
    run_file_mock.return_value = {
      'stdout': 'sss',
      'stderr': 'eee',
      'exitcode': 0
    }
    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
    self.assertEqual(ret['exitcode'], 0)
    self.assertEqual(ret['allocated_ports'], {'a.a.port': '10233'})
    self.assertTrue(run_file_mock.called)
    self.assertEqual(run_file_mock.call_count, 1)
    self.assertEqual(orchestrator.allocated_ports, {'a.a.port': '10233'})
    self.assertEqual(orchestrator.stored_command, {})

  def test_dump_command_to_json(self, unlink_mock,
                                isfile_mock, hostname_mock):
    hostname_mock.return_value = "test.hst"
    command = {
      'commandType': 'EXECUTION_COMMAND',
      'componentName': 'NAMENODE',
      'role': u'DATANODE',
      'roleCommand': u'INSTALL',
      'commandId': '1-1',
      'taskId': 3,
      'clusterName': u'cc',
      'serviceName': u'HDFS',
      'configurations': {'global': {}},
      'configurationTags': {'global': {'tag': 'v1'}},
      'clusterHostInfo': {'namenode_host': ['1'],
                          'slave_hosts': ['0', '1'],
                          'all_hosts': ['h1.hortonworks.com', 'h2.hortonworks.com'],
                          'all_ping_ports': ['8670:0,1']}
    }

    tempdir = tempfile.gettempdir()
    config = MagicMock()
    config.get.return_value = "something"
    config.getResolvedPath.return_value = tempdir
    config.getWorkRootPath.return_value = tempdir
    config.getLogPath.return_value = tempdir

    dummy_controller = MagicMock()
    orchestrator = CustomServiceOrchestrator(config, dummy_controller)
    isfile_mock.return_value = True
    # Test dumping EXECUTION_COMMAND
    json_file = orchestrator.dump_command_to_json(command, {})
    self.assertTrue(os.path.exists(json_file))
    self.assertTrue(os.path.getsize(json_file) > 0)
    self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
    self.assertTrue(json_file.endswith("command-3.json"))
    os.unlink(json_file)
    # Test dumping STATUS_COMMAND
    command['commandType'] = 'STATUS_COMMAND'
    json_file = orchestrator.dump_command_to_json(command, {})
    self.assertTrue(os.path.exists(json_file))
    self.assertTrue(os.path.getsize(json_file) > 0)
    self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
    self.assertTrue(json_file.endswith("status_command.json"))
    os.unlink(json_file)
    # Testing side effect of dump_command_to_json
    self.assertEquals(command['public_hostname'], "test.hst")
    self.assertTrue(unlink_mock.called)