Python ActionQueue.put方法代码示例

# 需要导入模块: from ambari_agent.ActionQueue import ActionQueue [as 别名]
# 或者: from ambari_agent.ActionQueue.ActionQueue import put [as 别名]
  def test_heartbeat_with_status_multiple(self, read_stack_version_method):
    actionQueue = ActionQueue(AmbariConfig.AmbariConfig().getConfig())
    actionQueue.IDLE_SLEEP_TIME = 0.01
    read_stack_version_method.return_value="1.3.0"
    heartbeat = Heartbeat(actionQueue)
    actionQueue.start()
    max_number_of_status_entries = 0
    for i in range(1,5):
      statusCommand = {
        "serviceName" : 'HDFS',
        "commandType" : "STATUS_COMMAND",
        "clusterName" : "",
        "componentName" : "DATANODE",
        'configurations':{'global' : {}}
      }
      actionQueue.put(statusCommand)
      time.sleep(0.1)
      result = heartbeat.build(101)
      number_of_status_entries = len(result['componentStatus'])
#      print "Heartbeat with status: " + str(result) + " XXX " + str(number_of_status_entries)
      if max_number_of_status_entries < number_of_status_entries:
        max_number_of_status_entries = number_of_status_entries
    actionQueue.stop()
    actionQueue.join()

    NUMBER_OF_COMPONENTS = 1
    self.assertEquals(max_number_of_status_entries == NUMBER_OF_COMPONENTS, True)

# 需要导入模块: from ambari_agent.ActionQueue import ActionQueue [as 别名]
# 或者: from ambari_agent.ActionQueue.ActionQueue import put [as 别名]
  def test_cancel_backgound_command(self, read_stack_version_mock, resolve_hook_script_path_mock, resolve_script_path_mock, FileCache_mock,
                                      kill_process_with_children_mock):
    FileCache_mock.return_value = None
    FileCache_mock.cache_dir = MagicMock()
    resolve_hook_script_path_mock.return_value = None
#     shell.kill_process_with_children = MagicMock()
    dummy_controller = MagicMock()
    cfg = AmbariConfig().getConfig()
    cfg.set('agent', 'tolerate_download_failures', 'true')
    cfg.set('agent', 'prefix', '.')
    cfg.set('agent', 'cache_dir', 'background_tasks')

    actionQueue = ActionQueue(cfg, dummy_controller)

    dummy_controller.actionQueue = actionQueue
    orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
    orchestrator.file_cache = MagicMock()
    def f (a, b):
      return ""
    orchestrator.file_cache.get_service_base_dir = f
    actionQueue.customServiceOrchestrator = orchestrator

    import TestActionQueue
    import copy

    TestActionQueue.patch_output_file(orchestrator.python_executor)
    orchestrator.python_executor.prepare_process_result = MagicMock()
    orchestrator.dump_command_to_json = MagicMock()

    lock = threading.RLock()
    complete_done = threading.Condition(lock)

    complete_was_called = {}
    def command_complete_w(process_condenced_result, handle):
      with lock:
        complete_was_called['visited']= ''
        complete_done.wait(3)

    actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(actionQueue.on_background_command_complete_callback, command_complete_w, None)
    execute_command = copy.deepcopy(TestActionQueue.TestActionQueue.background_command)
    actionQueue.put([execute_command])
    actionQueue.processBackgroundQueueSafeEmpty()

    time.sleep(.1)

    orchestrator.cancel_command(19,'')
    self.assertTrue(kill_process_with_children_mock.called)
    kill_process_with_children_mock.assert_called_with(33)

    with lock:
      complete_done.notifyAll()

    with lock:
      self.assertTrue(complete_was_called.has_key('visited'))

    time.sleep(.1)

    runningCommand = actionQueue.commandStatuses.get_command_status(19)
    self.assertTrue(runningCommand is not None)
    self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)

# 需要导入模块: from ambari_agent.ActionQueue import ActionQueue [as 别名]
# 或者: from ambari_agent.ActionQueue.ActionQueue import put [as 别名]
 def test_status_command_without_globals_section(self, stopped_method,
                                                 read_stack_version_method):
   config = AmbariConfig().getConfig()
   config.set('agent', 'prefix', TestStackVersionsFileHandler.dummyVersionsFile)
   queue = ActionQueue(config)
   statusCommand = {
     "serviceName" : 'HDFS',
     "commandType" : "STATUS_COMMAND",
     "clusterName" : "",
     "componentName" : "DATANODE",
     'configurations':{}
   }
   queue.stopped = stopped_method
   stopped_method.side_effect = [False, False, True, True, True]
   read_stack_version_method.return_value="1.3.0"
   queue.IDLE_SLEEP_TIME = 0.001
   queue.put(statusCommand)
   queue.run()
   returned_result = queue.resultQueue.get()
   returned_result[1]['status'] = 'INSTALLED' # Patch live value
   self.assertEquals(returned_result, ('STATUS_COMMAND',
                                       {'clusterName': '',
                                        'componentName': 'DATANODE',
                                        'msg': '',
                                        'serviceName': 'HDFS',
                                        'stackVersion': '1.3.0',
                                        'status': 'INSTALLED'}))

# 需要导入模块: from ambari_agent.ActionQueue import ActionQueue [as 别名]
# 或者: from ambari_agent.ActionQueue.ActionQueue import put [as 别名]
  def test_command_in_progress(self):
    config = AmbariConfig().getConfig()
    tmpfile = tempfile.gettempdir()
    config.set('agent', 'prefix', tmpfile)
    actionQueue = ActionQueue(config)
    actionQueue.IDLE_SLEEP_TIME = 0.01
    executor_started_event = threading.Event()
    end_executor_event = threading.Event()
    actionQueue.puppetExecutor = FakeExecutor(executor_started_event, end_executor_event)
    before_start_result = actionQueue.result()

    command = {
      'commandId': 17,
      'role' : "role",
      'taskId' : "taskId",
      'clusterName' : "clusterName",
      'serviceName' : "serviceName",
      'status' : 'IN_PROGRESS',
      'hostname' : "localhost.localdomain",
      'hostLevelParams': "hostLevelParams",
      'clusterHostInfo': "clusterHostInfo",
      'roleCommand': "roleCommand",
      'configurations': "configurations",
      'commandType': "EXECUTION_COMMAND",
      'configurations':{'global' : {}}
    }
    actionQueue.put(command)

    actionQueue.start()
    executor_started_event.wait()
    #print ("ii: " + pprint.pformat(actionQueue.commandInProgress))
    in_progress_result = actionQueue.result()
    end_executor_event.set()
    actionQueue.stop()
    actionQueue.join()
    after_start_result = actionQueue.result()

    self.assertEquals(len(before_start_result['componentStatus']), 0)
    self.assertEquals(len(before_start_result['reports']), 0)

    self.assertEquals(len(in_progress_result['componentStatus']), 0)
    self.assertEquals(len(in_progress_result['reports']), 1)
    self.assertEquals(in_progress_result['reports'][0]['status'], "IN_PROGRESS")
    self.assertEquals(in_progress_result['reports'][0]['stdout'], "Dummy output")
    self.assertEquals(in_progress_result['reports'][0]['exitCode'], 777)
    self.assertEquals(in_progress_result['reports'][0]['stderr'], 'Dummy err')

    self.assertEquals(len(after_start_result['componentStatus']), 0)
    self.assertEquals(len(after_start_result['reports']), 1)
    self.assertEquals(after_start_result['reports'][0]['status'], "COMPLETED")
    self.assertEquals(after_start_result['reports'][0]['stdout'], "returned stdout")
    self.assertEquals(after_start_result['reports'][0]['exitCode'], 0)
    self.assertEquals(after_start_result['reports'][0]['stderr'], 'returned stderr')

# 需要导入模块: from ambari_agent.ActionQueue import ActionQueue [as 别名]
# 或者: from ambari_agent.ActionQueue.ActionQueue import put [as 别名]
  def test_execute_python_executor(self, read_stack_version_mock, resolve_script_path_mock,
                                   get_py_executor_mock):
    
    dummy_controller = MagicMock()
    cfg = AmbariConfig()
    cfg.set('agent', 'tolerate_download_failures', 'true')
    cfg.set('agent', 'prefix', '.')
    cfg.set('agent', 'cache_dir', 'background_tasks')
    
    actionQueue = ActionQueue(cfg, dummy_controller)
    pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir, actionQueue.customServiceOrchestrator.config)
    patch_output_file(pyex)
    get_py_executor_mock.return_value = pyex
    actionQueue.customServiceOrchestrator.dump_command_to_json = MagicMock()
   
    result = {}
    lock = threading.RLock()
    complete_done = threading.Condition(lock)
    
    def command_complete_w(process_condensed_result, handle):
      with lock:
        result['command_complete'] = {'condensed_result' : copy.copy(process_condensed_result),
                                      'handle' : copy.copy(handle),
                                      'command_status' : actionQueue.commandStatuses.get_command_status(handle.command['taskId'])
                                      }
        complete_done.notifyAll()

    actionQueue.on_background_command_complete_callback = wraped(actionQueue.on_background_command_complete_callback,
                                                                 None, command_complete_w)
    actionQueue.put([self.background_command])
    actionQueue.processBackgroundQueueSafeEmpty();
    actionQueue.processStatusCommandQueueSafeEmpty();
    
    with lock:
      complete_done.wait(0.1)
      
      finished_status = result['command_complete']['command_status']
      self.assertEqual(finished_status['status'], ActionQueue.COMPLETED_STATUS)
      self.assertEqual(finished_status['stdout'], 'process_out')
      self.assertEqual(finished_status['stderr'], 'process_err')
      self.assertEqual(finished_status['exitCode'], 0)
      
    
    runningCommand = actionQueue.commandStatuses.current_state.get(self.background_command['taskId'])
    self.assertTrue(runningCommand is not None)
    
    report = actionQueue.result()
    self.assertEqual(len(report['reports']),1)
    self.assertEqual(report['reports'][0]['stdout'],'process_out')

# 需要导入模块: from ambari_agent.ActionQueue import ActionQueue [as 别名]
# 或者: from ambari_agent.ActionQueue.ActionQueue import put [as 别名]
 def test_cancel(self, CustomServiceOrchestrator_mock,
                      get_mock, process_command_mock):
   CustomServiceOrchestrator_mock.return_value = None
   dummy_controller = MagicMock()
   config = MagicMock()
   actionQueue = ActionQueue(config, dummy_controller)
   actionQueue.start()
   actionQueue.put([self.datanode_install_command, self.hbase_install_command])
   self.assertEqual(2, actionQueue.commandQueue.qsize())
   actionQueue.reset()
   self.assertTrue(actionQueue.commandQueue.empty())
   time.sleep(0.1)
   actionQueue.stop()
   actionQueue.join()
   self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.')

# 需要导入模块: from ambari_agent.ActionQueue import ActionQueue [as 别名]
# 或者: from ambari_agent.ActionQueue.ActionQueue import put [as 别名]
  def test_configtags(self):
    config = AmbariConfig().getConfig()
    tmpfile = tempfile.gettempdir()
    config.set('agent', 'prefix', tmpfile)
    actionQueue = ActionQueue(config)
    actionQueue.IDLE_SLEEP_TIME = 0.01
    executor_started_event = threading.Event()
    end_executor_event = threading.Event()
    actionQueue.puppetExecutor = FakeExecutor(executor_started_event, end_executor_event)

    command = {
      'commandId': 17,
      'role' : "role",
      'taskId' : "taskId",
      'clusterName' : "clusterName",
      'serviceName' : "serviceName",
      'status' : 'IN_PROGRESS',
      'hostname' : "localhost.localdomain",
      'hostLevelParams': "hostLevelParams",
      'clusterHostInfo': "clusterHostInfo",
      'roleCommand': "roleCommand",
      'configurations': "configurations",
      'commandType': "EXECUTION_COMMAND",
      'configurations':{'global' : {}},
      'configurationTags':{'global' : { 'tag': 'v1' }}
    }
    actionQueue.put(command)

    actionQueue.start()
    executor_started_event.wait()

    end_executor_event.set()
    actionQueue.stop()
    actionQueue.join()
    after_start_result = actionQueue.result()

    configname = os.path.join(tmpfile, 'config.json')

    self.assertEquals(len(after_start_result['componentStatus']), 0)
    self.assertEquals(len(after_start_result['reports']), 1)
    self.assertEquals(after_start_result['reports'][0]['status'], "COMPLETED")
    self.assertEquals(after_start_result['reports'][0]['stdout'], "returned stdout")
    self.assertEquals(after_start_result['reports'][0]['exitCode'], 0)
    self.assertEquals(after_start_result['reports'][0]['stderr'], 'returned stderr')
    self.assertEquals(len(after_start_result['reports'][0]['configurationTags']), 1)
    self.assertEquals(True, os.path.isfile(configname))

    os.remove(configname)

# 需要导入模块: from ambari_agent.ActionQueue import ActionQueue [as 别名]
# 或者: from ambari_agent.ActionQueue.ActionQueue import put [as 别名]
  def test_RetryAction(self):
    action={'id' : 'tttt'}
    config = AmbariConfig().getConfig()
    actionQueue = ActionQueue(config)
    path = actionQueue.getInstallFilename(action['id'])
    configFile = {
      "data"       : "test",
      "owner"      : os.getuid(),
      "group"      : os.getgid() ,
      "permission" : 0700,
      "path"       : path,
      "umask"      : 022
    }

    #note that the command in the action is just a listing of the path created
    #we just want to ensure that 'ls' can run on the data file (in the actual world
    #this 'ls' would be a puppet or a chef command that would work on a data
    #file
    badAction = {
      'id' : 'tttt',
      'kind' : 'INSTALL_AND_CONFIG_ACTION',
      'workDirComponent' : 'abc-hdfs',
      'file' : configFile,
      'clusterDefinitionRevision' : 12,
      'command' : ['/bin/ls',"/foo/bar/badPath1234"]
    }
    path=getFilePath(action,path)
    goodAction = {
      'id' : 'tttt',
      'kind' : 'INSTALL_AND_CONFIG_ACTION',
      'workDirComponent' : 'abc-hdfs',
      'file' : configFile,
      'clusterDefinitionRevision' : 12,
      'command' : ['/bin/ls',path]
    }
    actionQueue.start()
    response = {'actions' : [badAction,goodAction]}
    actionQueue.maxRetries = 2
    actionQueue.sleepInterval = 1
    result = actionQueue.put(response)
    results = actionQueue.result()
    sleptCount = 1
    while (len(results) < 2 and sleptCount < 15):
        time.sleep(1)
        sleptCount += 1
        results = actionQueue.result()
    actionQueue.stop()
    actionQueue.join()
    self.assertEqual(len(results), 2, 'Number of results is not 2.')
    result = results[0]
    maxretries = config.get('command', 'maxretries')
    self.assertEqual(int(result['retryActionCount']), 
                     int(maxretries),
                     "Number of retries is %d and not %d" % 
                     (int(result['retryActionCount']), int(str(maxretries))))
    result = results[1]
    self.assertEqual(int(result['retryActionCount']), 
                     1,
                     "Number of retries is %d and not %d" % 
                     (int(result['retryActionCount']), 1))        

# 需要导入模块: from ambari_agent.ActionQueue import ActionQueue [as 别名]
# 或者: from ambari_agent.ActionQueue.ActionQueue import put [as 别名]
 def test_parallel_exec(self, CustomServiceOrchestrator_mock,
                        process_command_mock, gpeo_mock):
   CustomServiceOrchestrator_mock.return_value = None
   dummy_controller = MagicMock()
   config = MagicMock()
   gpeo_mock.return_value = 1
   config.get_parallel_exec_option = gpeo_mock
   actionQueue = ActionQueue(config, dummy_controller)
   actionQueue.put([self.datanode_install_command, self.hbase_install_command])
   self.assertEqual(2, actionQueue.commandQueue.qsize())
   actionQueue.start()
   time.sleep(1)
   actionQueue.stop()
   actionQueue.join()
   self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.')
   self.assertEqual(2, process_command_mock.call_count)
   process_command_mock.assert_any_calls([call(self.datanode_install_command), call(self.hbase_install_command)])

# 需要导入模块: from ambari_agent.ActionQueue import ActionQueue [as 别名]
# 或者: from ambari_agent.ActionQueue.ActionQueue import put [as 别名]
 def test_heartbeat_with_status(self, read_stack_version_method):
   actionQueue = ActionQueue(AmbariConfig.AmbariConfig().getConfig())
   read_stack_version_method.return_value="1.3.0"
   heartbeat = Heartbeat(actionQueue)
   statusCommand = {
     "serviceName" : 'HDFS',
     "commandType" : "STATUS_COMMAND",
     "clusterName" : "",
     "componentName" : "DATANODE",
     'configurations':{'global' : {}}
   }
   actionQueue.put(statusCommand)
   actionQueue.start()
   time.sleep(0.1)
   actionQueue.stop()
   actionQueue.join()
   result = heartbeat.build(101)
   self.assertEquals(len(result['componentStatus']) > 0, True, 'Heartbeat should contain status of HDFS components')

# 需要导入模块: from ambari_agent.ActionQueue import ActionQueue [as 别名]
# 或者: from ambari_agent.ActionQueue.ActionQueue import put [as 别名]
 def test_reset_queue(self, CustomServiceOrchestrator_mock,
                               get_mock, process_command_mock, gpeo_mock):
   CustomServiceOrchestrator_mock.return_value = None
   dummy_controller = MagicMock()
   dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp())
   config = MagicMock()
   gpeo_mock.return_value = 0
   config.get_parallel_exec_option = gpeo_mock
   actionQueue = ActionQueue(config, dummy_controller)
   actionQueue.start()
   actionQueue.put([self.datanode_install_command, self.hbase_install_command])
   self.assertEqual(2, actionQueue.commandQueue.qsize())
   self.assertTrue(actionQueue.tasks_in_progress_or_pending())
   actionQueue.reset()
   self.assertTrue(actionQueue.commandQueue.empty())
   self.assertFalse(actionQueue.tasks_in_progress_or_pending())
   time.sleep(0.1)
   actionQueue.stop()
   actionQueue.join()
   self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.')

# 需要导入模块: from ambari_agent.ActionQueue import ActionQueue [as 别名]
# 或者: from ambari_agent.ActionQueue.ActionQueue import put [as 别名]
  def test_heartbeat_no_host_check_cmd_in_queue(self, register_mock):
    config = AmbariConfig.AmbariConfig().getConfig()
    config.set('agent', 'prefix', 'tmp')
    config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
    config.set('agent', 'tolerate_download_failures', "true")
    dummy_controller = MagicMock()
    actionQueue = ActionQueue(config, dummy_controller)
    statusCommand = {
      "serviceName" : 'HDFS',
      "commandType" : "STATUS_COMMAND",
      "clusterName" : "c1",
      "componentName" : "DATANODE",
      'configurations':{'global' : {}}
    }
    actionQueue.put([statusCommand])

    heartbeat = Heartbeat(actionQueue)
    heartbeat.build(12, 6)
    self.assertTrue(register_mock.called)
    args, kwargs = register_mock.call_args_list[0]
    self.assertTrue(args[2])
    self.assertFalse(args[1])

# 需要导入模块: from ambari_agent.ActionQueue import ActionQueue [as 别名]
# 或者: from ambari_agent.ActionQueue.ActionQueue import put [as 别名]
  def test_execute_background_command(self, CustomServiceOrchestrator_mock,
                                  runCommand_mock, read_stack_version_mock
                                  ):
    CustomServiceOrchestrator_mock.return_value = None
    CustomServiceOrchestrator.runCommand.return_value = {'exitcode' : 0,
                                                         'stdout': 'out-11',
                                                         'stderr' : 'err-13'}
    
    dummy_controller = MagicMock()
    actionQueue = ActionQueue(AmbariConfig(), dummy_controller)

    execute_command = copy.deepcopy(self.background_command)
    actionQueue.put([execute_command])
    actionQueue.processBackgroundQueueSafeEmpty();
    actionQueue.processStatusCommandQueueSafeEmpty();
    
    #assert that python execturor start
    self.assertTrue(runCommand_mock.called)
    runningCommand = actionQueue.commandStatuses.current_state.get(execute_command['taskId'])
    self.assertTrue(runningCommand is not None)
    self.assertEqual(runningCommand[1]['status'], ActionQueue.IN_PROGRESS_STATUS)
    
    report = actionQueue.result()
    self.assertEqual(len(report['reports']),1)

# 需要导入模块: from ambari_agent.ActionQueue import ActionQueue [as 别名]
# 或者: from ambari_agent.ActionQueue.ActionQueue import put [as 别名]
 def test_upgradeCommand_dispatching(self, stopped_method, executeCommand_method):
   queue = ActionQueue(config = MagicMock())
   command = {
     'commandId': 17,
     'role' : "role",
     'taskId' : "taskId",
     'clusterName' : "clusterName",
     'serviceName' : "serviceName",
     'roleCommand' : 'UPGRADE',
     'hostname' : "localhost.localdomain",
     'hostLevelParams': "hostLevelParams",
     'clusterHostInfo': "clusterHostInfo",
     'configurations': "configurations",
     'commandType': "EXECUTION_COMMAND",
     'configurations':{'global' : {}},
     'roleParams': {},
     'commandParams' :	{
       'source_stack_version' : 'HDP-1.2.1',
       'target_stack_version' : 'HDP-1.3.0'
     }
   }
   result = [{
     'exitcode' : 0,
     'stdout'   : 'abc',
     'stderr'   : 'def'
   }]
   executeCommand_method.return_value = result
   stopped_method.side_effect = [False, False, True, True, True]
   queue.stopped = stopped_method
   queue.IDLE_SLEEP_TIME = 0.001
   queue.put(command)
   queue.run()
   self.assertTrue(executeCommand_method.called)
   self.assertEquals(queue.resultQueue.qsize(), 1)
   returned_result = queue.resultQueue.get()
   self.assertTrue(returned_result[1] is result[0])