Python six.ensure_binary方法代码示例

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def printable_text(text):
    """Returns text encoded in a way suitable for print or `tf.logging`."""

    # These functions want `str` for both Python2 and Python3, but in one case
    # it's a Unicode string and in the other it's a byte string.
    if six.PY3:
        if isinstance(text, str):
            return text
        elif isinstance(text, bytes):
            return six.ensure_text(text, "utf-8", "ignore")
        else:
            raise ValueError("Unsupported string type: %s" % (type(text)))
    elif six.PY2:
        if isinstance(text, str):
            return text
        elif isinstance(text, six.text_type):
            return six.ensure_binary(text, "utf-8")
        else:
            raise ValueError("Unsupported string type: %s" % (type(text)))
    else:
        raise ValueError("Not running on Python2 or Python 3?") 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def RestoreGlobalStepIfNeeded(self, sess):
    """If global step is not initialized, load it from the checkpoint.

    Args:
      sess: tf.Session.
    """
    assert not self._save_only
    uninitialized_vars = self._GetUninitializedVarNames(sess)
    if six.ensure_binary('global_step') not in uninitialized_vars:
      return

    with sess.graph.as_default():
      gstep = py_utils.GetGlobalStep()

    path = tf.train.latest_checkpoint(self._train_dir)
    if path:
      reader = tf.train.NewCheckpointReader(path)
      value = reader.get_tensor('global_step')
      tf.logging.info('Restoring global step: %s', value)
      sess.run(gstep.assign(value))
    else:
      tf.logging.info('Initializing global step')
      sess.run(gstep.initializer) 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def _echo_via_pager(pager, output):
    env = dict(os.environ)
    # When the LESS environment variable is unset, Git sets it to FRX (if
    # LESS environment variable is set, Git does not change it at all).
    if 'LESS' not in env:
        env['LESS'] = 'FRX'

    pager = subprocess.Popen(pager.split(), stdin=subprocess.PIPE, env=env)

    output = six.ensure_binary(output)

    try:
        pager.communicate(input=output)
    except (IOError, KeyboardInterrupt):
        pass
    else:
        pager.stdin.close()

    while True:
        try:
            pager.wait()
        except KeyboardInterrupt:
            pass
        else:
            break 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def printable_text(text):
  """Returns text encoded in a way suitable for print or `tf.logging`."""

  # These functions want `str` for both Python2 and Python3, but in one case
  # it's a Unicode string and in the other it's a byte string.
  if six.PY3:
    if isinstance(text, str):
      return text
    elif isinstance(text, bytes):
      return six.ensure_text(text, "utf-8", "ignore")
    else:
      raise ValueError("Unsupported string type: %s" % (type(text)))
  elif six.PY2:
    if isinstance(text, str):
      return text
    elif isinstance(text, six.text_type):
      return six.ensure_binary(text, "utf-8")
    else:
      raise ValueError("Unsupported string type: %s" % (type(text)))
  else:
    raise ValueError("Not running on Python2 or Python 3?") 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def test_full_tokenizer(self):
    vocab_tokens = [
        "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn",
        "##ing", ","
    ]
    with tempfile.NamedTemporaryFile(delete=False) as vocab_writer:
      if six.PY2:
        vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
      else:
        contents = "".join([six.ensure_str(x) + "\n" for x in vocab_tokens])
        vocab_writer.write(six.ensure_binary(contents, "utf-8"))

      vocab_file = vocab_writer.name

    tokenizer = tokenization.FullTokenizer(vocab_file)
    os.unlink(vocab_file)

    tokens = tokenizer.tokenize(u"UNwant\u00E9d,running")
    self.assertAllEqual(tokens, ["un", "##want", "##ed", ",", "runn", "##ing"])

    self.assertAllEqual(
        tokenizer.convert_tokens_to_ids(tokens), [7, 4, 5, 10, 8, 9]) 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def ensure_binary(s, encoding='utf-8', errors='strict'):
        """Coerce **s** to six.binary_type.

        For Python 2:
          - `unicode` -> encoded to `str`
          - `str` -> `str`

        For Python 3:
          - `str` -> encoded to `bytes`
          - `bytes` -> `bytes`
        """
        if isinstance(s, text_type):
            return s.encode(encoding, errors)
        elif isinstance(s, binary_type):
            return s
        else:
            raise TypeError("not expecting type '%s'" % type(s)) 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def md5files(files):
    """Computes the combined MD5 hash of multiple files, represented as a list
    of (key, path).
    """
    m = hashlib.md5()
    for key, path in sorted(files, key=lambda x: x[0]):
        m.update(six.ensure_binary(key))
        if os.path.isdir(path):
            sub_md5 = md5files([
                (os.path.join(key, filename), os.path.join(path, filename))
                for filename in os.listdir(path)
                if not filename.startswith('.')])
            m.update(six.ensure_binary(sub_md5))
        else:
            with open(path, 'rb') as f:
                m.update(f.read())
    return m.hexdigest() 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def json_encode(obj, output=None, binary=False):
    """Encodes an object into a JSON string.

    @param obj: The object to serialize
    @param output: If not None, a file-like object to which the serialization should be written.
    @param binary: If True return binary string, otherwise text string.
    @type obj: dict|list|six.text_type
    @type binary: bool
    """
    # 2->TODO encode json according to 'binary' flag.
    if binary:

        result = six.ensure_binary(_json_encode(obj, None))
        if output:
            output.write(result)
        else:
            return result
    else:
        return six.ensure_text(_json_encode(obj, output)) 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def test_compute_log_manager_from_config(s3_bucket):
    s3_prefix = 'foobar'

    dagster_yaml = '''
compute_logs:
  module: dagster_aws.s3.compute_log_manager
  class: S3ComputeLogManager
  config:
    bucket: "{s3_bucket}"
    local_dir: "/tmp/cool"
    prefix: "{s3_prefix}"
'''.format(
        s3_bucket=s3_bucket, s3_prefix=s3_prefix
    )

    with seven.TemporaryDirectory() as tempdir:
        with open(os.path.join(tempdir, 'dagster.yaml'), 'wb') as f:
            f.write(six.ensure_binary(dagster_yaml))

        instance = DagsterInstance.from_config(tempdir)
    assert instance.compute_log_manager._s3_bucket == s3_bucket  # pylint: disable=protected-access
    assert instance.compute_log_manager._s3_prefix == s3_prefix  # pylint: disable=protected-access 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def test_parse_report_valid_not_human_readable(self):
    dsrf_xsd_file = path.join(
        path.dirname(__file__), '../testdata/sales-reporting-flat.xsd')
    avs_xsd_file = path.join(
        path.dirname(__file__), '../testdata/avs.xsd')
    files_list = [path.join(
        path.dirname(__file__), '../testdata/DSR_PADPIDA2014999999Z_'
        'PADPIDA2014111801Y_AdSupport_2015-02_AU_1of1_20150723T092522.tsv')]
    self.report_manager.parse_report(
        files_list, dsrf_xsd_file, avs_xsd_file,
        human_readable=False, write_head=False)
    serialized_block_str = open('/tmp/queue.txt', 'r').read().split(
        six.ensure_str('\n' + six.ensure_str(constants.QUEUE_DELIMITER)))[0]
    deserialized_block_str = six.ensure_binary(
        six.text_type(block_pb2.Block.FromString(serialized_block_str)),
        'utf-8')
    self.assertMultiLineEqual(BODY_BLOCK, deserialized_block_str) 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def write_to_queue(self, block, logger, human_readable=False):
    """Writes the block object to the output queue.

    Override this if you want to change the queue form.

    Args:
      block: A block_pb2.Block object to write.
      logger: Logger object.
      human_readable: If True, write to the queue the block in a human readable
                      form. Otherwise, Write the block as a raw bytes.
    """
    output = None
    if human_readable:
      output = six.ensure_binary(six.text_type(block), 'utf8')
    else:
      output = block.SerializeToString()
    try:
      os.write(sys.stdout.fileno(), output)
      os.write(sys.stdout.fileno(),
               bytes('\n' + constants.QUEUE_DELIMITER + '\n'))
    except OSError as e:
      logger.exception('Could not write to queue: %s', e)
      sys.stderr.write(
          'WARNING: Parser interrupted. Some blocks were not parsed.\n')
      sys.exit(-1) 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def __init__(self,
                 metadata_dict,
                 fishing_ranges_map):
        self.metadata_by_split = metadata_dict
        self.metadata_by_id = {}
        self.fishing_ranges_map = fishing_ranges_map
        self.id_map_int2bytes = {}
        for split, vessels in metadata_dict.items():
            for id_, data in vessels.items():
                id_ = six.ensure_binary(id_)
                self.metadata_by_id[id_] = data
                idhash = stable_hash(id_)
                self.id_map_int2bytes[idhash] = id_

        intersection_ids = set(self.metadata_by_id.keys()).intersection(
            set(fishing_ranges_map.keys()))
        logging.info("Metadata for %d ids.", len(self.metadata_by_id))
        logging.info("Fishing ranges for %d ids.", len(fishing_ranges_map))
        logging.info("Vessels with both types of data: %d",
                     len(intersection_ids)) 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def read_fishing_ranges(fishing_range_file):
    """ Read vessel fishing ranges, return a dict of id to classified fishing
        or non-fishing ranges for that vessel.
    """
    fishing_range_dict = defaultdict(lambda: [])
    with open(fishing_range_file, 'r') as f:
        for l in f.readlines()[1:]:
            els = l.split(',')
            id_ = six.ensure_binary(els[0].strip())
            start_time = parse_date(els[1]).replace(tzinfo=pytz.utc)
            end_time = parse_date(els[2]).replace(tzinfo=pytz.utc)
            is_fishing = float(els[3])
            fishing_range_dict[id_].append(
                FishingRange(start_time, end_time, is_fishing))

    return dict(fishing_range_dict) 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def __init__(self, user=None, keydata=None,
               token_uri=TOKEN_URI, destination_url=DESTINATION_URL):
    self.user = user
    self.keydata = keydata
    self.token_uri = token_uri
    self.destination_url = destination_url

    if user and keydata:
      self.credentials = oauth2client.client.SignedJwtAssertionCredentials(
          service_account_name=self.user,
          private_key=six.ensure_binary(self.keydata),
          scope=self.SCOPE_CODE_URI,
          user_agent='OpenHTF Guzzle Upload Client',
          token_uri=self.token_uri)
      self.credentials.set_store(_MemStorage())
    else:
      self.credentials = None

    self.upload_result = None

    self._cached_proto = None 

# 需要导入模块: import six [as 别名]
# 或者: from six import ensure_binary [as 别名]
def verify(self, totp, code, user):
        code = code.replace(' ', '')  # Google authenticator puts a space in their codes
        code = six.ensure_binary(code)  # can't be text

        self.enforce_rate_limit(user)

        # TODO prohibit re-use of a successful code, although it seems unlikely with a 30s window
        # see https://tools.ietf.org/html/rfc6238#section-5.2 paragraph 5

        # try the 1 previous time-window and current
        # per https://tools.ietf.org/html/rfc6238#section-5.2 paragraph 1
        windows = asint(config.get('auth.multifactor.totp.windows', 2))
        for time_window in range(windows):
            try:
                return totp.verify(code, time() - time_window*30)
            except InvalidToken:
                last_window = (time_window == windows - 1)
                if last_window:
                    raise