Python tabulator.topen函数代码示例

def test_native_iterator():

    # Get table
    def generator():
        yield ['id', 'name']
        yield ['1', 'english']
        yield ['2', '中国人']
    with pytest.raises(exceptions.ParsingError) as excinfo:
        iterator = generator()
        topen(iterator)
    assert 'callable' in str(excinfo.value)

def test_storage():

    # Get resources
    articles_schema = json.load(io.open('data/articles.json', encoding='utf-8'))
    comments_schema = json.load(io.open('data/comments.json', encoding='utf-8'))
    articles_data = topen('data/articles.csv', with_headers=True).read()
    comments_data = topen('data/comments.csv', with_headers=True).read()

    # Engine
    engine = create_engine(os.environ['DATABASE_URL'])

    # Storage
    storage = Storage(engine=engine, prefix='prefix_')

    # Delete tables
    for table in reversed(storage.tables):
        storage.delete(table)

    # Create tables
    storage.create(['articles', 'comments'], [articles_schema, comments_schema])

    # Write data to tables
    storage.write('articles', articles_data)
    storage.write('comments', comments_data)

    # Create new storage to use reflection only
    storage = Storage(engine=engine, prefix='prefix_')

    # Create existent table
    with pytest.raises(RuntimeError):
        storage.create('articles', articles_schema)

    # Get table representation
    assert repr(storage).startswith('Storage')

    # Get tables list
    assert storage.tables == ['articles', 'comments']

    # Get table schemas
    assert storage.describe('articles') == convert_schema(articles_schema)
    assert storage.describe('comments') == convert_schema(comments_schema)

    # Get table data
    assert list(storage.read('articles')) == convert_data(articles_schema, articles_data)
    assert list(storage.read('comments')) == convert_data(comments_schema, comments_data)

    # Delete tables
    for table in reversed(storage.tables):
        storage.delete(table)

    # Delete non existent table
    with pytest.raises(RuntimeError):
        storage.delete('articles')

def test_native_iterator():

    # Get table
    def generator():
        yield ["id", "name"]
        yield ["1", "english"]
        yield ["2", "中国人"]

    with pytest.raises(exceptions.SourceError) as excinfo:
        iterator = generator()
        topen(iterator)
    assert "callable" in str(excinfo.value)

def test_save_csv(tmpdir):

    # Save table
    path = str(tmpdir.join("table.csv"))
    table = topen("data/table.csv", headers=1)
    table.save(path)

    # Open saved table
    table = topen(path, headers=1)

    # Make assertions
    assert table.headers == ["id", "name"]
    assert table.read(extended=True) == [(2, ["id", "name"], ["1", "english"]), (3, ["id", "name"], ["2", "中国人"])]

def test_file_csv_with_bom():

    # Get table
    table = topen('data/special/bom.csv', encoding='utf-8')

    # Make assertions
    assert table.headers is None
    assert table.read() == [['id', 'name'], ['1', 'english'], ['2', '中国人']]

    # Get table
    table = topen('data/special/bom.csv')

    # Make assertions
    assert table.headers is None
    assert table.read() == [['id', 'name'], ['1', 'english'], ['2', '中国人']]

def test_save_csv(tmpdir):

    # Save table
    path = str(tmpdir.join('table.csv'))
    table = topen('data/table.csv', headers=1)
    table.save(path)

    # Open saved table
    table = topen(path, headers=1)

    # Make assertions
    assert table.headers == ['id', 'name']
    assert table.read(extended=True) == [
        (2, ['id', 'name'], ['1', 'english']),
        (3, ['id', 'name'], ['2', '中国人'])]

def test_processors_chain():

    # Processors
    def skip_commented_rows(extended_rows):
        for number, headers, row in extended_rows:
            if row and hasattr(row[0], "startswith") and row[0].startswith("#"):
                continue
            yield (number, headers, row)

    def skip_blank_rows(extended_rows):
        for number, headers, row in extended_rows:
            if not row:
                continue
            yield (number, headers, row)

    def cast_rows(extended_rows):
        for number, headers, row in extended_rows:
            crow = []
            for value in row:
                try:
                    if isinstance(value, six.string_types):
                        value = ast.literal_eval(value)
                except Exception:
                    pass
                crow.append(value)
            yield (number, headers, crow)

    # Get table
    source = [["id", "name"], ["#1", "english"], [], ["2", "中国人"]]
    table = topen(source, headers="row1", post_parse=[skip_commented_rows, skip_blank_rows, cast_rows])

    # Make assertions
    assert table.headers == ["id", "name"]

def test_reset_and_sample_size():

    # Get table
    table = topen("data/special/long.csv", headers=1, sample_size=3)

    # Make assertions
    assert table.read(extended=True) == [
        (2, ["id", "name"], ["1", "a"]),
        (3, ["id", "name"], ["2", "b"]),
        (4, ["id", "name"], ["3", "c"]),
        (5, ["id", "name"], ["4", "d"]),
        (6, ["id", "name"], ["5", "e"]),
        (7, ["id", "name"], ["6", "f"]),
    ]
    assert table.sample == [["1", "a"], ["2", "b"]]
    assert table.read() == []

    # Reset table
    table.reset()

    # Make assertions
    assert table.read(extended=True, limit=3) == [
        (2, ["id", "name"], ["1", "a"]),
        (3, ["id", "name"], ["2", "b"]),
        (4, ["id", "name"], ["3", "c"]),
    ]
    assert table.sample == [["1", "a"], ["2", "b"]]
    assert table.read(extended=True) == [
        (5, ["id", "name"], ["4", "d"]),
        (6, ["id", "name"], ["5", "e"]),
        (7, ["id", "name"], ["6", "f"]),
    ]

def test_html_content():

    # Check raises
    source = "https://github.com/frictionlessdata/tabulator-py/blob/master/data/table.csv"
    with pytest.raises(exceptions.FormatError) as excinfo:
        table = topen(source, headers="row1")
    assert "HTML" in str(excinfo.value)

def test_web_csv_non_ascii_url():

    # Get table
    table = topen("http://data.defra.gov.uk/ops/government_procurement_card/over_£500_GPC_apr_2013.csv")

    # Make assertions
    assert table.sample[0] == ["Entity", "Transaction Posting Date", "Merchant Name", "Amount", "Description"]

def test_reset_and_sample_size():

    # Get table
    table = topen('data/special/long.csv', headers=1, sample_size=3)

    # Make assertions
    assert table.read(extended=True) == [
        (2, ['id', 'name'], ['1', 'a']),
        (3, ['id', 'name'], ['2', 'b']),
        (4, ['id', 'name'], ['3', 'c']),
        (5, ['id', 'name'], ['4', 'd']),
        (6, ['id', 'name'], ['5', 'e']),
        (7, ['id', 'name'], ['6', 'f'])]
    assert table.sample == [['1', 'a'], ['2', 'b']]
    assert table.read() == []

    # Reset table
    table.reset()

    # Make assertions
    assert table.read(extended=True, limit=3) == [
        (2, ['id', 'name'], ['1', 'a']),
        (3, ['id', 'name'], ['2', 'b']),
        (4, ['id', 'name'], ['3', 'c'])]
    assert table.sample == [['1', 'a'], ['2', 'b']]
    assert table.read(extended=True) == [
        (5, ['id', 'name'], ['4', 'd']),
        (6, ['id', 'name'], ['5', 'e']),
        (7, ['id', 'name'], ['6', 'f'])]

def import_resource(storage, table, schema, data):
    """Import JSONTableSchema resource to storage's table.

    Parameters
    ----------
    storage: object
        Storage object.
    table: str
        Table name.
    schema: str
        Path to schema file.
    data: str
        Path to data file.

    """

    # Create table
    model = SchemaModel(schema)
    schema = model.as_python
    if storage.check(table):
        storage.delete(table)
    storage.create(table, schema)

    # Write data
    with topen(data, with_headers=True) as data:
        storage.write(table, data)

def test_html_content():

    # Check raises
    source = 'https://github.com/frictionlessdata/tabulator-py/blob/master/data/table.csv'
    with pytest.raises(exceptions.TabulatorException) as excinfo:
        table = topen(source, headers='row1')
    assert 'HTML' in str(excinfo.value)

def test_processors_chain():

    # Processors
    def skip_commented_rows(extended_rows):
        for number, headers, row in extended_rows:
            if (row and hasattr(row[0], 'startswith') and
                    row[0].startswith('#')):
                continue
            yield (number, headers, row)
    def skip_blank_rows(extended_rows):
        for number, headers, row in extended_rows:
            if not row:
                continue
            yield (number, headers, row)
    def cast_rows(extended_rows):
        for number, headers, row in extended_rows:
            crow = []
            for value in row:
                try:
                    if isinstance(value, six.string_types):
                        value = ast.literal_eval(value)
                except Exception:
                    pass
                crow.append(value)
            yield (number, headers, crow)

    # Get table
    source = [['id', 'name'], ['#1', 'english'], [], ['2', '中国人']]
    table = topen(source, headers='row1', post_parse=[
        skip_commented_rows,
        skip_blank_rows,
        cast_rows])

    # Make assertions
    assert table.headers == ['id', 'name']

def test_headers_with_headers_argument():

    # Get table
    table = topen("data/table.csv", with_headers=True)

    # Make assertions
    assert table.headers == ["id", "name"]
    assert list(table.iter(keyed=True)) == [{"id": "1", "name": "english"}, {"id": "2", "name": "中国人"}]