Python ThriftRecordWriter.close方法代码示例

# 需要导入模块: from twitter.common.recordio import ThriftRecordWriter [as 别名]
# 或者: from twitter.common.recordio.ThriftRecordWriter import close [as 别名]
def test_basic_thriftrecordwriter_write():
  test_string = StringType("hello world")

  with EphemeralFile('w') as fp:
    fn = fp.name

    rw = ThriftRecordWriter(fp)
    rw.write(test_string)
    rw.close()

    with open(fn) as fpr:
      rr = ThriftRecordReader(fpr, StringType)
      assert rr.read() == test_string

# 需要导入模块: from twitter.common.recordio import ThriftRecordWriter [as 别名]
# 或者: from twitter.common.recordio.ThriftRecordWriter import close [as 别名]
def test_thrift_recordwriter_type_mismatch():
  test_string = StringType("hello world")
  with EphemeralFile('w') as fp:
    fn = fp.name

    rw = ThriftRecordWriter(fp)
    rw.write(test_string)
    rw.close()

    with open(fn) as fpr:
      rr = ThriftRecordReader(fpr, IntType)
      # This is a peculiar behavior of Thrift in that it just returns
      # ThriftType() with no serialization applied
      assert rr.read() == IntType()

# 需要导入模块: from twitter.common.recordio import ThriftRecordWriter [as 别名]
# 或者: from twitter.common.recordio.ThriftRecordWriter import close [as 别名]
def test_thriftrecordreader_iteration():
  test_string_1 = StringType("hello world")
  test_string_2 = StringType("ahoy ahoy, bonjour")

  with EphemeralFile('w') as fp:
    fn = fp.name

    rw = ThriftRecordWriter(fp)
    rw.write(test_string_1)
    rw.write(test_string_2)
    rw.close()

    with open(fn) as fpr:
      rr = ThriftRecordReader(fpr, StringType)
      records = []
      for record in rr:
        records.append(record)
      assert records == [test_string_1, test_string_2]

# 需要导入模块: from twitter.common.recordio import ThriftRecordWriter [as 别名]
# 或者: from twitter.common.recordio.ThriftRecordWriter import close [as 别名]
def test_thriftrecordwriter_framing():
  test_string_1 = StringType("hello world")
  test_string_2 = StringType("ahoy ahoy, bonjour")

  with EphemeralFile('w') as fp:
    fn = fp.name

    rw = ThriftRecordWriter(fp)
    rw.write(test_string_1)
    rw.close()

    with open(fn, 'a') as fpa:
      rw = ThriftRecordWriter(fpa)
      rw.write(test_string_2)

    with open(fn) as fpr:
      rr = ThriftRecordReader(fpr, StringType)
      assert rr.read() == test_string_1
      assert rr.read() == test_string_2

# 需要导入模块: from twitter.common.recordio import ThriftRecordWriter [as 别名]
# 或者: from twitter.common.recordio.ThriftRecordWriter import close [as 别名]

#.........这里部分代码省略.........
  def process_logdir(self):
    return self._pathspec.getpath('process_logdir')

  def _setup_ckpt(self):
    """Set up the checkpoint: must be run on the parent."""
    self._log('initializing checkpoint file: %s' % self.ckpt_file())
    ckpt_fp = lock_file(self.ckpt_file(), "a+")
    if ckpt_fp in (None, False):
      raise self.CheckpointError('Could not acquire checkpoint permission or lock for %s!' %
        self.ckpt_file())
    self._ckpt_head = os.path.getsize(self.ckpt_file())
    ckpt_fp.seek(self._ckpt_head)
    self._ckpt = ThriftRecordWriter(ckpt_fp)
    self._ckpt.set_sync(True)

  def _init_ckpt_if_necessary(self):
    if self._ckpt is None:
      self._setup_ckpt()

  def _wait_for_control(self):
    """Wait for control of the checkpoint stream: must be run in the child."""
    total_wait_time = Amount(0, Time.SECONDS)

    with open(self.ckpt_file(), 'r') as fp:
      fp.seek(self._ckpt_head)
      rr = ThriftRecordReader(fp, RunnerCkpt)
      while total_wait_time < self.MAXIMUM_CONTROL_WAIT:
        ckpt_tail = os.path.getsize(self.ckpt_file())
        if ckpt_tail == self._ckpt_head:
          self._platform.clock().sleep(self.CONTROL_WAIT_CHECK_INTERVAL.as_(Time.SECONDS))
          total_wait_time += self.CONTROL_WAIT_CHECK_INTERVAL
          continue
        checkpoint = rr.try_read()
        if checkpoint:
          if not checkpoint.process_status:
            raise self.CheckpointError('No process status in checkpoint!')
          if (checkpoint.process_status.process != self.name() or
              checkpoint.process_status.state != ProcessState.FORKED or
              checkpoint.process_status.fork_time != self._fork_time or
              checkpoint.process_status.coordinator_pid != self._pid):
            self._log('Losing control of the checkpoint stream:')
            self._log('   fork_time [%s] vs self._fork_time [%s]' % (
                checkpoint.process_status.fork_time, self._fork_time))
            self._log('   coordinator_pid [%s] vs self._pid [%s]' % (
                checkpoint.process_status.coordinator_pid, self._pid))
            raise self.CheckpointError('Lost control of the checkpoint stream!')
          self._log('Taking control of the checkpoint stream at record: %s' %
            checkpoint.process_status)
          self._seq = checkpoint.process_status.seq + 1
          return True
    raise self.CheckpointError('Timed out waiting for checkpoint stream!')

  def _prepare_fork(self):
    user, current_user = self._getpwuid()
    if self._user:
      if user != current_user and os.geteuid() != 0:
        raise self.PermissionError('Must be root to run processes as other users!')
    self._fork_time = self._platform.clock().time()
    self._setup_ckpt()
    # Since the forked process is responsible for creating log files, it needs to own the log dir.
    safe_mkdir(self.process_logdir())
    os.chown(self.process_logdir(), user.pw_uid, user.pw_gid)

  def _finalize_fork(self):
    self._write_initial_update()
    self._ckpt.close()
    self._ckpt = None

  def start(self):
    """
      This is the main call point from the runner, and forks a co-ordinator process to run the
      target process (i.e. self.cmdline())

      The parent returns immediately and populates information about the pid of the co-ordinator.
      The child (co-ordinator) will launch the target process in a subprocess.
    """
    self._prepare_fork()  # calls _setup_ckpt which can raise CheckpointError
                          # calls _getpwuid which can raise:
                          #    UnknownUserError
                          #    PermissionError
    self._pid = self._platform.fork()
    if self._pid == 0:
      self._pid = self._platform.getpid()
      self._wait_for_control()  # can raise CheckpointError
      try:
        self.execute()
      except Exception as e:
        self._log('Error trying to execute %s: %s' % (self._name, e))
        raise e
      finally:
        self._ckpt.close()
        self.finish()
    else:
      self._finalize_fork()  # can raise CheckpointError

  def execute(self):
    raise NotImplementedError

  def finish(self):
    pass