Python compat.unicode方法代码示例

# 需要导入模块: from numpy import compat [as 别名]
# 或者: from numpy.compat import unicode [as 别名]
def _newnames(datatype, order):
    """
    Given a datatype and an order object, return a new names tuple, with the
    order indicated
    """
    oldnames = datatype.names
    nameslist = list(oldnames)
    if isinstance(order, (str, unicode)):
        order = [order]
    seen = set()
    if isinstance(order, (list, tuple)):
        for name in order:
            try:
                nameslist.remove(name)
            except ValueError:
                if name in seen:
                    raise ValueError("duplicate field name: %s" % (name,))
                else:
                    raise ValueError("unknown field name: %s" % (name,))
            seen.add(name)
        return tuple(list(order) + nameslist)
    raise ValueError("unsupported order value: %s" % (order,)) 

# 需要导入模块: from numpy import compat [as 别名]
# 或者: from numpy.compat import unicode [as 别名]
def content_check(self, ua, ua_scalar, nbytes):

        # Check the length of the unicode base type
        assert_(int(ua.dtype.str[2:]) == self.ulen)
        # Check the length of the data buffer
        assert_(buffer_length(ua) == nbytes)
        # Small check that data in array element is ok
        assert_(ua_scalar == self.ucs_value*self.ulen)
        # Encode to UTF-8 and double check
        assert_(ua_scalar.encode('utf-8') ==
                        (self.ucs_value*self.ulen).encode('utf-8'))
        # Check buffer lengths for scalars
        if ucs4:
            assert_(buffer_length(ua_scalar) == 4*self.ulen)
        else:
            if self.ucs_value == ucs4_value:
                # In UCS2, the \U0010FFFF will be represented using a
                # surrogate *pair*
                assert_(buffer_length(ua_scalar) == 2*2*self.ulen)
            else:
                # In UCS2, the \uFFFF will be represented using a
                # regular 2-byte word
                assert_(buffer_length(ua_scalar) == 2*self.ulen) 

# 需要导入模块: from numpy import compat [as 别名]
# 或者: from numpy.compat import unicode [as 别名]
def content_check(self, ua, ua_scalar, nbytes):

        # Check the length of the unicode base type
        self.assertTrue(int(ua.dtype.str[2:]) == self.ulen)
        # Check the length of the data buffer
        self.assertTrue(buffer_length(ua) == nbytes)
        # Small check that data in array element is ok
        self.assertTrue(ua_scalar == self.ucs_value*self.ulen)
        # Encode to UTF-8 and double check
        self.assertTrue(ua_scalar.encode('utf-8') ==
                        (self.ucs_value*self.ulen).encode('utf-8'))
        # Check buffer lengths for scalars
        if ucs4:
            self.assertTrue(buffer_length(ua_scalar) == 4*self.ulen)
        else:
            if self.ucs_value == ucs4_value:
                # In UCS2, the \U0010FFFF will be represented using a
                # surrogate *pair*
                self.assertTrue(buffer_length(ua_scalar) == 2*2*self.ulen)
            else:
                # In UCS2, the \uFFFF will be represented using a
                # regular 2-byte word
                self.assertTrue(buffer_length(ua_scalar) == 2*self.ulen) 

# 需要导入模块: from numpy import compat [as 别名]
# 或者: from numpy.compat import unicode [as 别名]
def test_unicode_roundtrip(self):
        utf8 = b'\xcf\x96'.decode('UTF-8')
        a = np.array([utf8], dtype=np.unicode)
        # our gz wrapper support encoding
        suffixes = ['', '.gz']
        # stdlib 2 versions do not support encoding
        if MAJVER > 2:
            if HAS_BZ2:
                suffixes.append('.bz2')
            if HAS_LZMA:
                suffixes.extend(['.xz', '.lzma'])
        with tempdir() as tmpdir:
            for suffix in suffixes:
                np.savetxt(os.path.join(tmpdir, 'test.csv' + suffix), a,
                           fmt=['%s'], encoding='UTF-16-LE')
                b = np.loadtxt(os.path.join(tmpdir, 'test.csv' + suffix),
                               encoding='UTF-16-LE', dtype=np.unicode)
                assert_array_equal(a, b) 

# 需要导入模块: from numpy import compat [as 别名]
# 或者: from numpy.compat import unicode [as 别名]
def test_utf8_file(self):
        utf8 = b"\xcf\x96"
        latin1 = b"\xf6\xfc\xf6"
        with temppath() as path:
            with open(path, "wb") as f:
                f.write((b"test1,testNonethe" + utf8 + b",test3\n") * 2)
            test = np.genfromtxt(path, dtype=None, comments=None,
                                 delimiter=',', encoding="UTF-8")
            ctl = np.array([
                     ["test1", "testNonethe" + utf8.decode("UTF-8"), "test3"],
                     ["test1", "testNonethe" + utf8.decode("UTF-8"), "test3"]],
                     dtype=np.unicode)
            assert_array_equal(test, ctl)

            # test a mixed dtype
            with open(path, "wb") as f:
                f.write(b"0,testNonethe" + utf8)
            test = np.genfromtxt(path, dtype=None, comments=None,
                                 delimiter=',', encoding="UTF-8")
            assert_equal(test['f0'], 0)
            assert_equal(test['f1'], "testNonethe" + utf8.decode("UTF-8")) 

# 需要导入模块: from numpy import compat [as 别名]
# 或者: from numpy.compat import unicode [as 别名]
def buffer_length(arr):
        if isinstance(arr, unicode):
            arr = str(arr)
            if not arr:
                charmax = 0
            else:
                charmax = max([ord(c) for c in arr])
            if charmax < 256:
                size = 1
            elif charmax < 65536:
                size = 2
            else:
                size = 4
            return size * len(arr)
        v = memoryview(arr)
        if v.shape is None:
            return len(v) * v.itemsize
        else:
            return np.prod(v.shape) * v.itemsize 

# 需要导入模块: from numpy import compat [as 别名]
# 或者: from numpy.compat import unicode [as 别名]
def test_upgrade(self):
        "Tests the upgrade method."

        converter = StringConverter()
        assert_equal(converter._status, 0)

        # test int
        assert_equal(converter.upgrade('0'), 0)
        assert_equal(converter._status, 1)

        # On systems where long defaults to 32-bit, the statuses will be
        # offset by one, so we check for this here.
        import numpy.core.numeric as nx
        status_offset = int(nx.dtype(nx.int_).itemsize < nx.dtype(nx.int64).itemsize)

        # test int > 2**32
        assert_equal(converter.upgrade('17179869184'), 17179869184)
        assert_equal(converter._status, 1 + status_offset)

        # test float
        assert_allclose(converter.upgrade('0.'), 0.0)
        assert_equal(converter._status, 2 + status_offset)

        # test complex
        assert_equal(converter.upgrade('0j'), complex('0j'))
        assert_equal(converter._status, 3 + status_offset)

        # test str
        # note that the longdouble type has been skipped, so the
        # _status increases by 2. Everything should succeed with
        # unicode conversion (5).
        for s in ['a', u'a', b'a']:
            res = converter.upgrade(s)
            assert_(type(res) is unicode)
            assert_equal(res, u'a')
            assert_equal(converter._status, 5 + status_offset) 

# 需要导入模块: from numpy import compat [as 别名]
# 或者: from numpy.compat import unicode [as 别名]
def test_include_dirs(self):
        # As a sanity check, just test that get_include
        # includes something reasonable.  Somewhat
        # related to ticket #1405.
        include_dirs = [np.get_include()]
        for path in include_dirs:
            assert_(isinstance(path, (str, unicode)))
            assert_(path != '') 

# 需要导入模块: from numpy import compat [as 别名]
# 或者: from numpy.compat import unicode [as 别名]
def _setfieldnames(self, names, titles):
        """convert input field names into a list and assign to the _names
        attribute """

        if (names):
            if (type(names) in [list, tuple]):
                pass
            elif isinstance(names, (str, unicode)):
                names = names.split(',')
            else:
                raise NameError("illegal input names %s" % repr(names))

            self._names = [n.strip() for n in names[:self._nfields]]
        else:
            self._names = []

        # if the names are not specified, they will be assigned as
        #  "f0, f1, f2,..."
        # if not enough names are specified, they will be assigned as "f[n],
        # f[n+1],..." etc. where n is the number of specified names..."
        self._names += ['f%d' % i for i in range(len(self._names),
                                                 self._nfields)]
        # check for redundant names
        _dup = find_duplicate(self._names)
        if _dup:
            raise ValueError("Duplicate field names: %s" % _dup)

        if (titles):
            self._titles = [n.strip() for n in titles[:self._nfields]]
        else:
            self._titles = []
            titles = []

        if (self._nfields > len(titles)):
            self._titles += [None] * (self._nfields - len(titles)) 

# 需要导入模块: from numpy import compat [as 别名]
# 或者: from numpy.compat import unicode [as 别名]
def buffer_length(arr):
        if isinstance(arr, np.ndarray):
            return len(arr.data)
        return len(buffer(arr))

# In both cases below we need to make sure that the byte swapped value (as
# UCS4) is still a valid unicode:
# Value that can be represented in UCS2 interpreters 

# 需要导入模块: from numpy import compat [as 别名]
# 或者: from numpy.compat import unicode [as 别名]
def test_comments_unicode(self):
        c = TextIO()
        c.write('# comment\n1,2,3,5\n')
        c.seek(0)
        x = np.loadtxt(c, dtype=int, delimiter=',',
                       comments=unicode('#'))
        a = np.array([1, 2, 3, 5], int)
        assert_array_equal(x, a)