Python CustomServiceOrchestrator.runCommand方法代码示例

# 需要导入模块: from CustomServiceOrchestrator import CustomServiceOrchestrator [as 别名]
# 或者: from CustomServiceOrchestrator.CustomServiceOrchestrator import runCommand [as 别名]
 def test_runCommand_custom_action(self, get_custom_actions_base_dir_mock,
                                   FileCache_mock,
                                   run_file_mock, dump_command_to_json_mock):
   FileCache_mock.return_value = None
   get_custom_actions_base_dir_mock.return_value = "some path"
   _, script = tempfile.mkstemp()
   command = {
     'role' : 'any',
     'commandParams': {
       'script_type': 'PYTHON',
       'script': 'some_custom_action.py',
       'command_timeout': '600',
       'jdk_location' : 'some_location'
     },
     'taskId' : '3',
     'roleCommand': 'ACTIONEXECUTE'
   }
   dummy_controller = MagicMock()
   orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
   # normal run case
   run_file_mock.return_value = {
     'stdout' : 'sss',
     'stderr' : 'eee',
     'exitcode': 0,
     }
   ret = orchestrator.runCommand(command, "out.txt", "err.txt")
   self.assertEqual(ret['exitcode'], 0)
   self.assertTrue(run_file_mock.called)
   # Hoooks are not supported for custom actions,
   # that's why run_file() should be called only once
   self.assertEqual(run_file_mock.call_count, 1)

# 需要导入模块: from CustomServiceOrchestrator import CustomServiceOrchestrator [as 别名]
# 或者: from CustomServiceOrchestrator.CustomServiceOrchestrator import runCommand [as 别名]
  def test_runCommand_background_action(self, get_custom_actions_base_dir_mock,
                                    FileCache_mock,
                                    dump_command_to_json_mock):
    FileCache_mock.return_value = None
    get_custom_actions_base_dir_mock.return_value = "some path"
    _, script = tempfile.mkstemp()
    command = {
      'role' : 'any',
      'commandParams': {
        'script_type': 'PYTHON',
        'script': 'some_custom_action.py',
        'command_timeout': '600',
        'jdk_location' : 'some_location'
      },
      'taskId' : '13',
      'roleCommand': 'ACTIONEXECUTE',
      'commandType': 'BACKGROUND_EXECUTION_COMMAND',
      '__handle': BackgroundCommandExecutionHandle({'taskId': '13'}, 13,
                                                   MagicMock(), MagicMock())
    }
    dummy_controller = MagicMock()
    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)

    import TestActionQueue
    TestActionQueue.patch_output_file(orchestrator.python_executor)
    orchestrator.python_executor.condenseOutput = MagicMock()
    orchestrator.dump_command_to_json = MagicMock()

    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
    self.assertEqual(ret['exitcode'], 777)

# 需要导入模块: from CustomServiceOrchestrator import CustomServiceOrchestrator [as 别名]
# 或者: from CustomServiceOrchestrator.CustomServiceOrchestrator import runCommand [as 别名]
  def test_runCommand_get_port(self,
                               run_file_mock,
                               resolve_script_path_mock,
                               allocate_ports_mock):
    command = {
      'role': 'HBASE_REGIONSERVER',
      'hostLevelParams': {
        'stack_name': 'HDP',
        'stack_version': '2.0.7',
        'jdk_location': 'some_location'
      },
      'commandParams': {
        'script_type': 'PYTHON',
        'script': 'scripts/hbase_regionserver.py',
        'command_timeout': '600',
        'service_package_folder': 'HBASE'
      },
      'taskId': '3',
      'roleCommand': 'INSTALL',
      'commandType': 'EXECUTE',
      'componentName': 'HBASE_REGIONSERVER',
      'configurations': {'a': {'a.port': '${HBASE_REGIONSERVER.ALLOCATED_PORT}'}}
    }

    tempdir = tempfile.gettempdir()
    config = MagicMock()
    config.get.return_value = "something"
    config.getResolvedPath.return_value = tempdir
    config.getWorkRootPath.return_value = tempdir
    config.getLogPath.return_value = tempdir

    allocate_ports_mock.return_value = str(10233)

    resolve_script_path_mock.return_value = "/basedir/scriptpath"
    dummy_controller = MagicMock()
    orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
    # normal run case
    run_file_mock.return_value = {
      'stdout': 'sss',
      'stderr': 'eee',
      'exitcode': 0
    }
    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
    self.assertEqual(ret['exitcode'], 0)
    self.assertEqual(ret['allocated_ports'], {'a.a.port': '10233'})
    self.assertTrue(run_file_mock.called)
    self.assertEqual(run_file_mock.call_count, 1)
    self.assertEqual(orchestrator.allocated_ports, {'a.a.port': '10233'})
    self.assertEqual(orchestrator.stored_command, {})

# 需要导入模块: from CustomServiceOrchestrator import CustomServiceOrchestrator [as 别名]
# 或者: from CustomServiceOrchestrator.CustomServiceOrchestrator import runCommand [as 别名]
  def test_runCommand(self, FileCache_mock,
                      run_file_mock, dump_command_to_json_mock,
                      get_hook_base_dir_mock, get_service_base_dir_mock,
                      resolve_hook_script_path_mock, resolve_script_path_mock):
    FileCache_mock.return_value = None
    command = {
      'role' : 'REGION_SERVER',
      'hostLevelParams' : {
        'stack_name' : 'HDP',
        'stack_version' : '2.0.7',
        'jdk_location' : 'some_location'
      },
      'commandParams': {
        'script_type': 'PYTHON',
        'script': 'scripts/hbase_regionserver.py',
        'command_timeout': '600',
        'service_package_folder' : 'HBASE'
      },
      'taskId' : '3',
      'roleCommand': 'INSTALL'
    }
    get_service_base_dir_mock.return_value = "/basedir/"
    resolve_script_path_mock.return_value = "/basedir/scriptpath"
    resolve_hook_script_path_mock.return_value = \
      ('/hooks_dir/prefix-command/scripts/hook.py',
       '/hooks_dir/prefix-command')
    dummy_controller = MagicMock()
    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
    get_hook_base_dir_mock.return_value = "/hooks/"
    # normal run case
    run_file_mock.return_value = {
        'stdout' : 'sss',
        'stderr' : 'eee',
        'exitcode': 0,
      }
    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
    self.assertEqual(ret['exitcode'], 0)
    self.assertTrue(run_file_mock.called)
    self.assertEqual(run_file_mock.call_count, 3)

    run_file_mock.reset_mock()

    # Case when we force another command
    run_file_mock.return_value = {
        'stdout' : 'sss',
        'stderr' : 'eee',
        'exitcode': 0,
      }
    ret = orchestrator.runCommand(command, "out.txt", "err.txt",
              forced_command_name=CustomServiceOrchestrator.COMMAND_NAME_STATUS)
    ## Check that override_output_files was true only during first call
    self.assertEquals(run_file_mock.call_args_list[0][0][7], True)
    self.assertEquals(run_file_mock.call_args_list[1][0][7], False)
    self.assertEquals(run_file_mock.call_args_list[2][0][7], False)
    ## Check that forced_command_name was taken into account
    self.assertEqual(run_file_mock.call_args_list[0][0][1][0],
                                  CustomServiceOrchestrator.COMMAND_NAME_STATUS)

    run_file_mock.reset_mock()

    # unknown script type case
    command['commandParams']['script_type'] = "SOME_TYPE"
    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
    self.assertEqual(ret['exitcode'], 1)
    self.assertFalse(run_file_mock.called)
    self.assertTrue("Unknown script type" in ret['stdout'])

    #By default returns empty dictionary
    self.assertEqual(ret['structuredOut'], '{}')

    pass

# 需要导入模块: from CustomServiceOrchestrator import CustomServiceOrchestrator [as 别名]
# 或者: from CustomServiceOrchestrator.CustomServiceOrchestrator import runCommand [as 别名]
  def test_runCommand_with_shell_config(self,
                                  run_file_mock,
                                  unlink_mock,
                                  isfile_mock,
                                  hostname_mock):
    hostname_mock.return_value = "test.hst"
    isfile_mock.return_value = True
    command = {
      'role': 'MEMCACHED',
      'componentName': 'MEMCACHED',
      'hostLevelParams': {
        'jdk_location': 'some_location'
      },
      'commandParams': {
        'script_type': 'SHELL',
        'command_timeout': '600'
      },
      'configurations': {
        "memcached-site": {
          "memcached.log": "${AGENT_LOG_ROOT}",
          "memcached.number": "10485760"},
        "memcached-log4j": {"a": "b"}
      },
      'taskId': '3',
      'roleCommand': 'INSTALL',
      'commandType': 'EXECUTION_COMMAND',
      'commandId': '1-1'
    }

    command_get = {
      'roleCommand': 'GET_CONFIG',
      'commandType': 'STATUS_COMMAND'
    }

    command_get_specific = {
      'roleCommand': 'GET_CONFIG',
      'commandType': 'STATUS_COMMAND',
      'commandParams': {
        'config_type': 'memcached-site'
      }
    }

    tempdir = tempfile.gettempdir()
    config = MagicMock()
    config.get.return_value = "something"
    config.getResolvedPath.return_value = tempdir
    config.getWorkRootPath.return_value = tempdir
    config.getLogPath.return_value = tempdir

    dummy_controller = MagicMock()
    orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
    # normal run case
    run_file_mock.return_value = {
      'stdout': 'sss',
      'stderr': 'eee',
      'exitcode': 0,
      }

    expected = {
      'memcached-site': {
        'memcached.log': tempdir, 'memcached.number': '10485760'},
      'memcached-log4j': {'a': 'b'}}

    expected_specific = {
      'memcached-site': {
        'memcached.log': tempdir, 'memcached.number': '10485760'},
      }

    ret = orchestrator.runCommand(command, "out.txt", "err.txt", True, True)
    self.assertEqual.__self__.maxDiff = None
    self.assertEqual(ret['exitcode'], 0)
    self.assertTrue(run_file_mock.called)

    ret = orchestrator.requestComponentStatus(command_get)
    self.assertEqual(ret['configurations'], expected)

    ret = orchestrator.requestComponentStatus(command_get_specific)
    self.assertEqual(ret['configurations'], expected_specific)

    script_path = os.path.realpath(posixpath.join(tempdir,
                                                  "infra", "agent", "slider-agent", "scripts",
                                                  "shell_cmd", "basic_installer.py"))
    run_file_mock.assert_has_calls([call(
      script_path,
      ['INSTALL', os.path.realpath(posixpath.join(tempdir, 'command-3.json')),
       os.path.realpath(posixpath.join(tempdir, 'package'))],
      'out.txt', 'err.txt', 600,
      os.path.realpath(posixpath.join(tempdir, 'structured-out-3.json')),
      'INFO', True,
      [('PYTHONPATH', ":".join([os.path.realpath(posixpath.join(tempdir, 'infra', 'agent', 'slider-agent', 'jinja2')),
                                os.path.realpath(posixpath.join(tempdir, 'infra', 'agent', 'slider-agent'))]))])])
    pass

# 需要导入模块: from CustomServiceOrchestrator import CustomServiceOrchestrator [as 别名]
# 或者: from CustomServiceOrchestrator.CustomServiceOrchestrator import runCommand [as 别名]
  def test_runCommand_with_config(self,
                                  run_file_mock,
                                  resolve_script_path_mock, unlink_mock,
                                  isfile_mock, hostname_mock):
    hostname_mock.return_value = "test.hst"
    isfile_mock.return_value = True
    command = {
      'role': 'REGION_SERVER',
      'componentName': 'REGION_SERVER',
      'hostLevelParams': {
        'stack_name': 'HDP',
        'stack_version': '2.0.7',
        'jdk_location': 'some_location'
      },
      'commandParams': {
        'script_type': 'PYTHON',
        'script': 'scripts/hbase_regionserver.py',
        'command_timeout': '600',
        'service_package_folder': 'HBASE'
      },
      'configurations': {
        "hbase-site": {
          "hbase.log": "${AGENT_LOG_ROOT}",
          "hbase.number": "10485760"},
        "hbase-log4j": {"a": "b"}
      },
      'taskId': '3',
      'roleCommand': 'INSTALL',
      'commandType': 'EXECUTION_COMMAND',
      'commandId': '1-1'
    }

    command_get = {
      'roleCommand': 'GET_CONFIG',
      'commandType': 'STATUS_COMMAND'
    }

    command_get_specific = {
      'roleCommand': 'GET_CONFIG',
      'commandType': 'STATUS_COMMAND',
      'commandParams': {
        'config_type': 'hbase-site'
      }
    }

    tempdir = tempfile.gettempdir()
    config = MagicMock()
    config.get.return_value = "something"
    config.getResolvedPath.return_value = tempdir
    config.getWorkRootPath.return_value = tempdir
    config.getLogPath.return_value = tempdir

    resolve_script_path_mock.return_value = "/basedir/scriptpath"
    dummy_controller = MagicMock()
    orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
    # normal run case
    run_file_mock.return_value = {
      'stdout': 'sss',
      'stderr': 'eee',
      'exitcode': 0,
    }

    expected = {
      'hbase-site': {
        'hbase.log': tempdir, 'hbase.number': '10485760'},
      'hbase-log4j': {'a': 'b'}}

    expected_specific = {
      'hbase-site': {
        'hbase.log': tempdir, 'hbase.number': '10485760'},
    }

    ret = orchestrator.runCommand(command, "out.txt", "err.txt", True, True)
    self.assertEqual.__self__.maxDiff = None
    self.assertEqual(ret['exitcode'], 0)
    self.assertTrue(run_file_mock.called)

    ret = orchestrator.requestComponentStatus(command_get)
    self.assertEqual(ret['configurations'], expected)

    ret = orchestrator.requestComponentStatus(command_get_specific)
    self.assertEqual(ret['configurations'], expected_specific)
    pass

# 需要导入模块: from CustomServiceOrchestrator import CustomServiceOrchestrator [as 别名]
# 或者: from CustomServiceOrchestrator.CustomServiceOrchestrator import runCommand [as 别名]
  def test_runCommand(self,
                      run_file_mock, dump_command_to_json_mock,
                      resolve_script_path_mock):
    command = {
      'role': 'REGION_SERVER',
      'hostLevelParams': {
        'stack_name': 'HDP',
        'stack_version': '2.0.7',
        'jdk_location': 'some_location'
      },
      'commandParams': {
        'script_type': 'PYTHON',
        'script': 'scripts/hbase_regionserver.py',
        'command_timeout': '600',
        'service_package_folder': 'HBASE'
      },
      'taskId': '3',
      'roleCommand': 'INSTALL'
    }

    tempdir = tempfile.gettempdir()
    config = MagicMock()
    config.get.return_value = "something"
    config.getResolvedPath.return_value = tempdir
    config.getWorkRootPath.return_value = tempdir
    config.getLogPath.return_value = tempdir

    resolve_script_path_mock.return_value = "/basedir/scriptpath"
    dummy_controller = MagicMock()
    orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
    # normal run case
    run_file_mock.return_value = {
      'stdout': 'sss',
      'stderr': 'eee',
      'exitcode': 0
    }
    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
    self.assertEqual(ret['exitcode'], 0)
    self.assertTrue(run_file_mock.called)
    self.assertEqual(run_file_mock.call_count, 1)

    run_file_mock.reset_mock()

    # Case when we force another command
    run_file_mock.return_value = {
      'stdout': 'sss',
      'stderr': 'eee',
      'exitcode': 0,
    }
    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
    ## Check that override_output_files was true only during first call
    self.assertEquals(run_file_mock.call_args_list[0][0][7], True)

    run_file_mock.reset_mock()
    # Case when we force another command
    run_file_mock.return_value = {
      'stdout': 'sss',
      'stderr': 'eee',
      'exitcode': 1
    }
    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
    self.assertFalse('allocated_ports' in ret)

    run_file_mock.reset_mock()

    # unknown script type case
    command['commandParams']['script_type'] = "PUPPET"
    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
    self.assertEqual(ret['exitcode'], 1)
    self.assertFalse(run_file_mock.called)
    self.assertTrue("Unknown script type" in ret['stdout'])

    #By default returns empty dictionary
    self.assertEqual(ret['structuredOut'], '{}')
    pass

# 需要导入模块: from CustomServiceOrchestrator import CustomServiceOrchestrator [as 别名]
# 或者: from CustomServiceOrchestrator.CustomServiceOrchestrator import runCommand [as 别名]

#.........这里部分代码省略.........
      in_progress_status.update({
        'tmpout': self.tmpdir + os.sep + 'auto_output-' + str(taskId) + '.txt',
        'tmperr': self.tmpdir + os.sep + 'auto_errors-' + str(taskId) + '.txt',
        'structuredOut' : self.tmpdir + os.sep + 'auto_structured-out-' + str(taskId) + '.json',
        'status': self.IN_PROGRESS_STATUS
      })

    self.commandStatuses.put_command_status(command, in_progress_status)

    numAttempts = 0
    retryDuration = 0  # even with 0 allow one attempt
    retryAble = False
    delay = 1
    log_command_output = True
    if 'commandParams' in command and 'log_output' in command['commandParams'] and "false" == command['commandParams']['log_output']:
      log_command_output = False

    if 'commandParams' in command:
      if 'max_duration_for_retries' in command['commandParams']:
        retryDuration = int(command['commandParams']['max_duration_for_retries'])
      if 'command_retry_enabled' in command['commandParams']:
        retryAble = command['commandParams']['command_retry_enabled'] == "true"
    if isAutoExecuteCommand:
      retryAble = False

    logger.info("Command execution metadata - taskId = {taskId}, retry enabled = {retryAble}, max retry duration (sec) = {retryDuration}, log_output = {log_command_output}".
                 format(taskId=taskId, retryAble=retryAble, retryDuration=retryDuration, log_command_output=log_command_output))
    while retryDuration >= 0:
      numAttempts += 1
      start = 0
      if retryAble:
        start = int(time.time())
      # running command
      commandresult = self.customServiceOrchestrator.runCommand(command,
                                                                in_progress_status['tmpout'],
                                                                in_progress_status['tmperr'],
                                                                override_output_files=numAttempts == 1,
                                                                retry=numAttempts > 1)
      end = 1
      if retryAble:
        end = int(time.time())
      retryDuration -= (end - start)

      # dumping results
      if isCommandBackground:
        logger.info("Command is background command, quit retrying. Exit code: {exitCode}, retryAble: {retryAble}, retryDuration (sec): {retryDuration}, last delay (sec): {delay}"
                    .format(cid=taskId, exitCode=commandresult['exitcode'], retryAble=retryAble, retryDuration=retryDuration, delay=delay))
        return
      else:
        if commandresult['exitcode'] == 0:
          status = self.COMPLETED_STATUS
        else:
          status = self.FAILED_STATUS
          if (commandresult['exitcode'] == -signal.SIGTERM) or (commandresult['exitcode'] == -signal.SIGKILL):
            logger.info('Command {cid} was canceled!'.format(cid=taskId))
            break

      if status != self.COMPLETED_STATUS and retryAble and retryDuration > 0:
        delay = self.get_retry_delay(delay)
        if delay > retryDuration:
          delay = retryDuration
        retryDuration -= delay  # allow one last attempt
        commandresult['stderr'] += "\n\nCommand failed. Retrying command execution ...\n\n"
        logger.info("Retrying command id {cid} after a wait of {delay}".format(cid=taskId, delay=delay))
        time.sleep(delay)
        continue