Python TaskPath.given方法代码示例

# 需要导入模块: from apache.thermos.common.path import TaskPath [as 别名]
# 或者: from apache.thermos.common.path.TaskPath import given [as 别名]
def test_garbage_collector(safe_rmtree, safe_delete):
  with temporary_dir() as sandbox, temporary_dir() as checkpoint_root, temporary_dir() as log_dir:

    path = TaskPath(root=checkpoint_root, task_id='test', log_dir=log_dir)

    touch(os.path.join(sandbox, 'test_file1'))
    touch(os.path.join(sandbox, 'test_file2'))
    safe_mkdir(os.path.dirname(path.given(state='finished').getpath('task_path')))
    safe_mkdir(os.path.dirname(path.getpath('runner_checkpoint')))
    touch(path.given(state='finished').getpath('task_path'))

    header = RunnerHeader(task_id='test', sandbox=sandbox, log_dir=log_dir)
    ckpt = TaskRunnerHelper.open_checkpoint(path.getpath('runner_checkpoint'))
    ckpt.write(RunnerCkpt(runner_header=header))
    ckpt.close()

    gc = TaskGarbageCollector(checkpoint_root, task_id='test')
    assert gc._state.header.log_dir == log_dir
    assert gc._state.header.sandbox == sandbox

    # erase metadata
    gc.erase_metadata()
    safe_delete.assert_has_calls([
        call(path.given(state='finished').getpath('task_path')),
        call(path.getpath('runner_checkpoint'))], any_order=True)
    safe_rmtree.assert_has_calls([call(path.getpath('checkpoint_path'))])

    safe_delete.reset_mock()
    safe_rmtree.reset_mock()

    # erase logs
    gc.erase_logs()
    safe_rmtree.assert_has_calls([call(log_dir)])

    safe_delete.reset_mock()
    safe_rmtree.reset_mock()

    # erase sandbox
    gc.erase_data()

    safe_delete.assert_has_calls([
        call(os.path.join(sandbox, 'test_file1')),
        call(os.path.join(sandbox, 'test_file2'))], any_order=True)
    safe_rmtree.assert_has_calls([call(sandbox)])

# 需要导入模块: from apache.thermos.common.path import TaskPath [as 别名]
# 或者: from apache.thermos.common.path.TaskPath import given [as 别名]
def test_empty_garbage_collector():
  with temporary_dir() as checkpoint_root:
    path = TaskPath(root=checkpoint_root, task_id='does_not_exist')
    gc = TaskGarbageCollector(checkpoint_root, 'does_not_exist')

    assert gc.get_age() == 0

    # assume runner, finished task
    assert set(gc.get_metadata(with_size=False)) == set([
        path.getpath('runner_checkpoint'),
        path.given(state='finished').getpath('task_path'),
    ])

    assert list(gc.get_logs()) == []
    assert list(gc.get_data()) == []

# 需要导入模块: from apache.thermos.common.path import TaskPath [as 别名]
# 或者: from apache.thermos.common.path.TaskPath import given [as 别名]
    def __init__(self, root, task_id):
        """Construct a TaskMonitor.

    :param root: The checkpoint root of the task.
    :param task_id: The task id of the task.
    """
        pathspec = TaskPath(root=root, task_id=task_id)
        self._dispatcher = CheckpointDispatcher()
        self._runnerstate = RunnerState(processes={})
        self._runner_ckpt = pathspec.getpath("runner_checkpoint")
        self._active_file, self._finished_file = (
            pathspec.given(state=state).getpath("task_path") for state in ("active", "finished")
        )
        self._ckpt_head = 0
        self._apply_states()
        self._lock = threading.Lock()

# 需要导入模块: from apache.thermos.common.path import TaskPath [as 别名]
# 或者: from apache.thermos.common.path.TaskPath import given [as 别名]
class CheckpointInspector(object):
  def __init__(self, checkpoint_root):
    self._path = TaskPath(root=checkpoint_root)

  @staticmethod
  def get_timestamp(process_record):
    if process_record :
      for timestamp in ('fork_time', 'start_time', 'stop_time'):
        stamp = getattr(process_record, timestamp, None)
        if stamp:
          return stamp
    return 0

  def inspect(self, task_id):
    """
      Reconstructs the checkpoint stream and returns a CheckpointInspection.
    """
    dispatcher = CheckpointDispatcher()
    state = RunnerState(processes = {})
    muxer = ProcessMuxer(self._path.given(task_id=task_id))

    runner_processes = []
    coordinator_processes = set()
    processes = set()

    def consume_process_record(record):
      if not record.process_status:
        return
      try:
        user_uid = pwd.getpwnam(state.header.user).pw_uid
      except KeyError:
        log.error('Could not find user: %s' % state.header.user)
        return
      if record.process_status.state == ProcessState.FORKED:
        coordinator_processes.add((record.process_status.coordinator_pid, user_uid,
                                   record.process_status.fork_time))
      elif record.process_status.state == ProcessState.RUNNING:
        processes.add((record.process_status.pid, user_uid,
                       record.process_status.start_time))

    # replay runner checkpoint
    runner_pid = None
    runner_latest_update = 0
    try:
      with open(self._path.given(task_id=task_id).getpath('runner_checkpoint')) as fp:
        with closing(ThriftRecordReader(fp, RunnerCkpt)) as ckpt:
          for record in ckpt:
            dispatcher.dispatch(state, record)
            runner_latest_update = max(runner_latest_update,
                self.get_timestamp(record.process_status))
            # collect all bound runners
            if record.task_status:
              if record.task_status.runner_pid != runner_pid:
                runner_processes.append((record.task_status.runner_pid,
                                         record.task_status.runner_uid or 0,
                                         record.task_status.timestamp_ms))
                runner_pid = record.task_status.runner_pid
            elif record.process_status:
              consume_process_record(record)
    except (IOError, OSError, RecordIO.Error) as err:
      log.debug('Error inspecting task runner checkpoint: %s' % err)
      return

    # register existing processes in muxer
    for process_name in state.processes:
      muxer.register(process_name)

    # read process checkpoints
    process_latest_update = runner_latest_update
    for record in muxer.select():
      process_latest_update = max(process_latest_update, self.get_timestamp(record.process_status))
      consume_process_record(record)

    return CheckpointInspection(
      runner_latest_update=runner_latest_update,
      process_latest_update=process_latest_update,
      runner_processes=runner_processes,
      coordinator_processes=coordinator_processes,
      processes=processes)

# 需要导入模块: from apache.thermos.common.path import TaskPath [as 别名]
# 或者: from apache.thermos.common.path.TaskPath import given [as 别名]

#.........这里部分代码省略.........
          self.remove_finished_task(unknown)

  @Lockable.sync
  def process_from_name(self, task_id, process_id):
    if task_id in self.all_tasks:
      task = self.all_tasks[task_id].task
      if task:
        for process in task.processes():
          if process.name().get() == process_id:
            return process

  @Lockable.sync
  def task_count(self):
    """
      Return the count of tasks that could be ready properly from disk.
      This may be <= self.task_id_count()
    """
    return dict(
      active=len(self.active_tasks),
      finished=len(self.finished_tasks),
      all=len(self.all_tasks),
    )

  @Lockable.sync
  def task_id_count(self):
    """
      Return the raw count of active and finished task_ids from the TaskDetector.
    """
    num_active = len(list(self._detector.get_task_ids(state='active')))
    num_finished = len(list(self._detector.get_task_ids(state='finished')))
    return dict(active=num_active, finished=num_finished, all=num_active + num_finished)

  def _get_tasks_of_type(self, type):
    """Convenience function to return all tasks of a given type"""
    tasks = {
      'active': self.active_tasks,
      'finished': self.finished_tasks,
      'all': self.all_tasks,
    }.get(type, None)

    if tasks is None:
      log.error('Unknown task type %s' % type)
      return {}

    return tasks

  @Lockable.sync
  def state(self, task_id):
    """Return a dict containing mapped information about a task's state"""
    real_state = self.raw_state(task_id)
    if real_state is None or real_state.header is None:
      return {}
    else:
      return dict(
        task_id=real_state.header.task_id,
        launch_time=real_state.header.launch_time_ms / 1000.0,
        sandbox=real_state.header.sandbox,
        hostname=real_state.header.hostname,
        user=real_state.header.user
      )

  @Lockable.sync
  def raw_state(self, task_id):
    """
      Return the current runner state (thrift blob: gen.apache.thermos.ttypes.RunnerState)
      of a given task id

# 需要导入模块: from apache.thermos.common.path import TaskPath [as 别名]
# 或者: from apache.thermos.common.path.TaskPath import given [as 别名]
class TaskDetector(object):
  """
    Helper class in front of TaskPath to detect active/finished/running tasks. Performs no
    introspection on the state of a task; merely detects based on file paths on disk.
  """
  class MatchingError(Exception): pass

  def __init__(self, root):
    self._root_dir = root
    self._pathspec = TaskPath()

  def get_task_ids(self, state=None):
    paths = glob.glob(self._pathspec.given(root=self._root_dir,
                                           task_id="*",
                                           state=state or '*')
                                    .getpath('task_path'))
    path_re = re.compile(self._pathspec.given(root=re.escape(self._root_dir),
                                              task_id="(\S+)",
                                              state='(\S+)')
                                       .getpath('task_path'))
    for path in paths:
      try:
        task_state, task_id = path_re.match(path).groups()
      except:
        continue
      if state is None or task_state == state:
        yield (task_state, task_id)

  def get_process_runs(self, task_id, log_dir):
    paths = glob.glob(self._pathspec.given(root=self._root_dir,
                                           task_id=task_id,
                                           log_dir=log_dir,
                                           process='*',
                                           run='*')
                                    .getpath('process_logdir'))
    path_re = re.compile(self._pathspec.given(root=re.escape(self._root_dir),
                                              task_id=re.escape(task_id),
                                              log_dir=log_dir,
                                              process='(\S+)',
                                              run='(\d+)')
                                       .getpath('process_logdir'))
    for path in paths:
      try:
        process, run = path_re.match(path).groups()
      except:
        continue
      yield process, int(run)

  def get_process_logs(self, task_id, log_dir):
    for process, run in self.get_process_runs(task_id, log_dir):
      for logtype in ('stdout', 'stderr'):
        path = (self._pathspec.with_filename(logtype).given(root=self._root_dir,
                                                           task_id=task_id,
                                                           log_dir=log_dir,
                                                           process=process,
                                                           run=run)
                                                     .getpath('process_logdir'))
        if os.path.exists(path):
          yield path

  def get_checkpoint(self, task_id):
    return self._pathspec.given(root=self._root_dir, task_id=task_id).getpath('runner_checkpoint')

  def get_process_checkpoints(self, task_id):
    matching_paths = glob.glob(self._pathspec.given(root=self._root_dir,
                                                    task_id=task_id,
                                                    process='*')
                                             .getpath('process_checkpoint'))
    path_re = re.compile(self._pathspec.given(root=re.escape(self._root_dir),
                                              task_id=re.escape(task_id),
                                              process='(\S+)')
                                       .getpath('process_checkpoint'))
    for path in matching_paths:
      try:
        process, = path_re.match(path).groups()
      except:
        continue
      yield path

# 需要导入模块: from apache.thermos.common.path import TaskPath [as 别名]
# 或者: from apache.thermos.common.path.TaskPath import given [as 别名]

#.........这里部分代码省略.........

    # Otherwise, use the finalization wait provided in the configuration.
    finalization_allocation = self.task.finalization_wait().get()
    if self._finalization_start is None:
      return sys.float_info.max
    else:
      waited = max(0, self._clock.time() - self._finalization_start)
      return max(0, finalization_allocation - waited)

  def _set_process_status(self, process_name, process_state, **kw):
    if 'sequence_number' in kw:
      sequence_number = kw.pop('sequence_number')
      log.debug('_set_process_status(%s <= %s, seq=%s[force])', process_name,
        ProcessState._VALUES_TO_NAMES.get(process_state), sequence_number)
    else:
      current_run = self._current_process_run(process_name)
      if not current_run:
        assert process_state == ProcessState.WAITING
        sequence_number = 0
      else:
        sequence_number = current_run.seq + 1
      log.debug('_set_process_status(%s <= %s, seq=%s[auto])', process_name,
        ProcessState._VALUES_TO_NAMES.get(process_state), sequence_number)
    runner_ckpt = RunnerCkpt(process_status=ProcessStatus(
        process=process_name, state=process_state, seq=sequence_number, **kw))
    self._dispatcher.dispatch(self._state, runner_ckpt, self._recovery)

  def _task_process_from_process_name(self, process_name, sequence_number):
    """
      Construct a Process() object from a process_name, populated with its
      correct run number and fully interpolated commandline.
    """
    run_number = len(self.state.processes[process_name]) - 1
    pathspec = self._pathspec.given(process=process_name, run=run_number)
    process = self._process_map.get(process_name)
    if process is None:
      raise self.InternalError('FATAL: Could not find process: %s' % process_name)
    def close_ckpt_and_fork():
      pid = os.fork()
      if pid == 0 and self._ckpt is not None:
        self._ckpt.close()
      return pid

    (logger_destination,
     logger_mode,
     rotate_log_size,
     rotate_log_backups) = self._build_process_logger_args(process)

    return Process(
      process.name().get(),
      process.cmdline().get(),
      sequence_number,
      pathspec,
      self._sandbox,
      self._user,
      chroot=self._chroot,
      fork=close_ckpt_and_fork,
      logger_destination=logger_destination,
      logger_mode=logger_mode,
      rotate_log_size=rotate_log_size,
      rotate_log_backups=rotate_log_backups,
      preserve_env=self._preserve_env,
      mesos_containerizer_path=self._mesos_containerizer_path,
      container_sandbox=self._container_sandbox)

  _DEFAULT_LOGGER = Logger()

# 需要导入模块: from apache.thermos.common.path import TaskPath [as 别名]
# 或者: from apache.thermos.common.path.TaskPath import given [as 别名]
class ObservedTask(AbstractClass):
  """ Represents a Task being observed """

  @classmethod
  def safe_mtime(cls, path):
    try:
      return os.path.getmtime(path)
    except OSError:
      return None

  def __init__(self, root, task_id):
    self._root = root
    self._task_id = task_id
    self._pathspec = TaskPath(root=self._root, task_id=self._task_id)
    self._mtime = self._get_mtime()

  @property
  def root(self):
    return self._root

  @abstractproperty
  def type(self):
    """Indicates the type of task (active or finished)"""

  def _read_task(self, memoized={}):
    """Read the corresponding task from disk and return a ThermosTask.  Memoizes already-read tasks.
    """
    if self._task_id not in memoized:
      path = self._pathspec.given(state=self.type).getpath('task_path')
      if os.path.exists(path):
        task = ThermosTaskWrapper.from_file(path)
        if task is None:
          log.error('Error reading ThermosTask from %s in observer.' % path)
        else:
          context = self.context(self._task_id)
          if not context:
            log.warning('Task not yet available: %s' % self._task_id)
          task = task.task() % Environment(thermos=context)
          memoized[self._task_id] = task

    return memoized.get(self._task_id, None)

  def _get_mtime(self):
    """Retrieve the mtime of the task's state directory"""
    get_path = lambda state: self._pathspec.given(state=state).getpath('task_path')
    mtime = self.safe_mtime(get_path('active'))
    if mtime is None:
      mtime = self.safe_mtime(get_path('finished'))
    if mtime is None:
      log.error("Couldn't get mtime for task %s!" % self._task_id)
    return mtime

  def context(self, task_id):
    state = self.state
    if state.header is None:
      return None
    return ThermosContext(
      ports=state.header.ports if state.header.ports else {},
      task_id=state.header.task_id,
      user=state.header.user,
    )

  @property
  def task(self):
    """Return a ThermosTask representing this task"""
    return self._read_task()

  @property
  def task_id(self):
    """Return the task's task_id"""
    return self._task_id

  @property
  def mtime(self):
    """Return mtime of task file"""
    return self._mtime

  @abstractproperty
  def state(self):
    """Return state of task (gen.apache.thermos.ttypes.RunnerState)"""

# 需要导入模块: from apache.thermos.common.path import TaskPath [as 别名]
# 或者: from apache.thermos.common.path.TaskPath import given [as 别名]
def test_task_detector():
  with temporary_dir() as root:
    active_log_dir = os.path.join(root, 'active_log')
    finished_log_dir = os.path.join(root, 'finished_log')

    path = TaskPath(root=root)
    detector = TaskDetector(root)

    # test empty paths

    assert list(detector.get_task_ids(state='active')) == []
    assert list(detector.get_task_ids(state='finished')) == []
    assert set(detector.get_task_ids()) == set()

    assert detector.get_checkpoint(task_id='active_task') == path.given(
        task_id='active_task').getpath('runner_checkpoint')

    assert detector.get_checkpoint(task_id='finished_task') == path.given(
        task_id='finished_task').getpath('runner_checkpoint')

    assert set(detector.get_process_checkpoints('active_task')) == set()
    assert set(detector.get_process_checkpoints('finished_task')) == set()
    assert set(detector.get_process_runs('active_task', active_log_dir)) == set()
    assert set(detector.get_process_runs('finished_task', finished_log_dir)) == set()
    assert set(detector.get_process_logs('active_task', active_log_dir)) == set()
    assert set(detector.get_process_logs('finished_task', finished_log_dir)) == set()

    # create paths

    paths = [
        path.given(state='active', task_id='active_task').getpath('task_path'),
        path.given(state='finished', task_id='finished_task').getpath('task_path'),
        path.given(task_id='active_task').getpath('runner_checkpoint'),
        path.given(task_id='finished_task').getpath('runner_checkpoint'),
        path.given(
            task_id='active_task',
            process='hello_world',
            run='0',
            log_dir=active_log_dir
        ).with_filename('stdout').getpath('process_logdir'),
        path.given(
            task_id='finished_task',
            process='goodbye_world',
            run='1',
            log_dir=finished_log_dir
        ).with_filename('stderr').getpath('process_logdir'),
        path.given(task_id='active_task', process='hello_world').getpath('process_checkpoint'),
        path.given(task_id='finished_task', process='goodbye_world').getpath('process_checkpoint'),
    ]

    for p in paths:
      touch(p)

    detector = TaskDetector(root)

    assert list(detector.get_task_ids(state='active')) == list([('active', 'active_task')])
    assert list(detector.get_task_ids(state='finished')) == list([('finished', 'finished_task')])
    assert set(detector.get_task_ids()) == set(
        [('active', 'active_task'), ('finished', 'finished_task')])

    assert list(detector.get_process_checkpoints('active_task')) == [
        path.given(task_id='active_task', process='hello_world').getpath('process_checkpoint')]

    assert list(detector.get_process_checkpoints('finished_task')) == [
        path.given(task_id='finished_task', process='goodbye_world').getpath('process_checkpoint')]

    assert list(detector.get_process_runs('active_task', active_log_dir)) == [
        ('hello_world', 0)]
    assert list(detector.get_process_runs('finished_task', finished_log_dir)) == [
        ('goodbye_world', 1)]

    assert list(detector.get_process_logs('active_task', active_log_dir)) == [
        path.given(
            task_id='active_task',
            process='hello_world',
            run='0',
            log_dir=active_log_dir
        ).with_filename('stdout').getpath('process_logdir')]

    assert list(detector.get_process_logs('finished_task', finished_log_dir)) == [
        path.given(
            task_id='finished_task',
            process='goodbye_world',
            run='1',
            log_dir=finished_log_dir
        ).with_filename('stderr').getpath('process_logdir')]