Python itertools.islice方法代码示例

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def assert_equal(t1, t2, show_num_wrong=3, names=None, msg=''):
    if t1.shape != t2.shape:
        raise AssertionError('Different shapes! {} != {}'.format(t1.shape, t2.shape))
    wrong = t1 != t2
    if not wrong.any():
        return
    if names is None:
        names = ('t1', 't2')
    wrong_idxs = wrong.nonzero()
    num_wrong = len(wrong_idxs)
    show_num_wrong = min(show_num_wrong, num_wrong)
    wrong_idxs = itertools.islice((tuple(i.tolist()) for i in wrong_idxs),
                                  show_num_wrong)
    err_msg = ' // '.join('{}: {}!={}'.format(idx, t1[idx], t2[idx])
                          for idx in wrong_idxs)
    raise AssertionError(('{} != {}: {}, and {}/{} other(s) '.format(
            names[0], names[1], err_msg, num_wrong - show_num_wrong, np.prod(t1.shape)) + msg).strip()) 

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def nlargest(n, iterable):
    """Find the n largest elements in a dataset.

    Equivalent to:  sorted(iterable, reverse=True)[:n]
    """
    if n < 0:
        return []
    it = iter(iterable)
    result = list(islice(it, n))
    if not result:
        return result
    heapify(result)
    _heappushpop = heappushpop
    for elem in it:
        _heappushpop(result, elem)
    result.sort(reverse=True)
    return result 

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def nsmallest(n, iterable):
    """Find the n smallest elements in a dataset.

    Equivalent to:  sorted(iterable)[:n]
    """
    if n < 0:
        return []
    it = iter(iterable)
    result = list(islice(it, n))
    if not result:
        return result
    _heapify_max(result)
    _heappushpop = _heappushpop_max
    for elem in it:
        _heappushpop(result, elem)
    result.sort()
    return result

# 'heap' is a heap at all indices >= startpos, except possibly for pos.  pos
# is the index of a leaf with a possibly out-of-order value.  Restore the
# heap invariant. 

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def demo():
    from en.parser.nltk_lite.corpora import conll2000
    from itertools import islice

    print "CONLL Chunked data\n"
    
    print "Raw text:"
    for sent in islice(conll2000.raw(), 0, 5):
        print sent
    print

    print "Tagged text:"
    for sent in islice(conll2000.tagged(), 0, 5):
        print sent
    print

    print "Chunked text:"
    for tree in islice(conll2000.chunked(chunk_types=('NP', 'PP', 'VP')), 0, 5):
        print tree.pp()
    print 

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def demo():
    from en.parser.nltk_lite.corpora import toolbox
    from itertools import islice
    from pprint import pprint

    print 'Raw:'
    pprint(list(islice(toolbox.raw(), 3)))

    print 'Dictionary:'
    pprint(list(islice(toolbox.dictionary(), 3)))

    print 'Dictionary-List:'
    pprint(list(islice(toolbox.dict_list(), 3)))

    print 'Complex test cases, no header'
    pprint(list(toolbox.raw("test.dic")))

    print 'Complex test cases, no header, dictionary'
    pprint(list(toolbox.dictionary("test.dic")))

    print 'Complex test cases, no header, dictionary list'
    pprint(list(toolbox.dict_list("test.dic")))

    print 'Complex test cases, with header'
    pprint(list(toolbox.raw("test.dic", include_header=True))) 

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def native_concat(nodes):
    """Return a native Python type from the list of compiled nodes. If the
    result is a single node, its value is returned. Otherwise, the nodes are
    concatenated as strings. If the result can be parsed with
    :func:`ast.literal_eval`, the parsed value is returned. Otherwise, the
    string is returned.
    """
    head = list(islice(nodes, 2))

    if not head:
        return None

    if len(head) == 1:
        out = head[0]
    else:
        out = u''.join([text_type(v) for v in chain(head, nodes)])

    try:
        return literal_eval(out)
    except (ValueError, SyntaxError, MemoryError):
        return out 

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def chunkize_serial(iterable, chunksize, as_numpy=False):
    """
    Return elements from the iterable in `chunksize`-ed lists. The last returned
    element may be smaller (if length of collection is not divisible by `chunksize`).

    >>> print(list(grouper(range(10), 3)))
    [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]

    """
    import numpy
    it = iter(iterable)
    while True:
        if as_numpy:
            # convert each document to a 2d numpy array (~6x faster when transmitting
            # chunk data over the wire, in Pyro)
            wrapped_chunk = [[numpy.array(doc) for doc in itertools.islice(it, int(chunksize))]]
        else:
            wrapped_chunk = [list(itertools.islice(it, int(chunksize)))]
        if not wrapped_chunk[0]:
            break
        # memory opt: wrap the chunk and then pop(), to avoid leaving behind a dangling reference
        yield wrapped_chunk.pop() 

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def test_iterview_infinite():
    loader = LiveLoader('.cache', SourceTest())
    iterview = IteratorView(loader, 'train', infinite=True)
    assert list(map(lambda tp: tp[0], itertools.islice(iterview, 150))) \
        == list(range(100)) + list(range(50)) 

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def test_iterview_random():
    loader = LiveLoader('.cache', SourceTest())
    iterview = IteratorView(loader, 'train', randomize=True, fetch_size=1)
    assert list(map(lambda tp: tp[0], itertools.islice(iterview, 10))) \
        == [92, 1, 43, 61, 35, 73, 48, 18, 98, 36] 

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def test_iterview_random_fetch_size():
    loader = LiveLoader('.cache', SourceTest())
    iterview = IteratorView(loader, 'train', randomize=True, fetch_size=10)
    assert list(map(lambda tp: tp[0], itertools.islice(iterview, 10))) \
        == list(range(70, 80)) 

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def test_iterview_random2():
    loader = LiveLoader('.cache', SourceTest())
    iterview = IteratorView(loader, 'train', randomize=True, fetch_size=1)
    iterview2 = IteratorView(loader, 'train', randomize=True, random_seed=2601, fetch_size=1)
    assert list(map(lambda tp: tp[0], itertools.islice(iterview2, 10))) \
        != list(map(lambda tp: tp[0], itertools.islice(iterview, 10))) 

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def chunks(data, SIZE=500):
    it = iter(data)
    for i in range(0, len(data), SIZE):
        if isinstance(data, dict):
            yield {k: data[k] for k in islice(it, SIZE)}
        else:
            yield data[i : i + SIZE] 

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def chunk_map(data, SIZE=50):
    """Yield successive n-sized chunks from l."""
    it = iter(data)
    for i in xrange(0, len(data), SIZE):
        yield {k: data[k] for k in islice(it, SIZE)} 

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def strip_signature(txn):
    unsigned_parts = itertools.islice(txn, len(UNSIGNED_TRANSACTION_FIELDS))
    return list(unsigned_parts) 

# 需要导入模块: import itertools [as 别名]
# 或者: from itertools import islice [as 别名]
def stitch(parts):
    side = int(math.sqrt(len(parts)))
    if side * side != len(parts):
        raise ValueError(f'Invalid number of parts {len(parts)}')

    rows = []

    # Sort by original position in image
    crops_idx_mapping = _get_crop_idx_mapping(side)
    parts_sorted = (
        part for _, part in sorted(
        enumerate(parts), key=lambda ip: crops_idx_mapping[ip[0]]))

    parts_itr = iter(parts_sorted)  # Turn into iterator so we can easily grab elements
    for _ in range(side):
        parts_row = itertools.islice(parts_itr, side)  # Get `side` number of parts
        row = torch.cat(list(parts_row), dim=3)  # cat on W dimension
        rows.append(row)

    assert next(parts_itr, None) is None, f'Iterator should be empty, got {len(rows)} rows'
    img = torch.cat(rows, dim=2)  # cat on H dimension

    # Validate.
    B, C, H_part, W_part = parts[0].shape
    expected_shape = (B, C, H_part * side, W_part * side)
    assert img.shape == expected_shape, f'{img.shape} != {expected_shape}'

    return img