Python data.CSV类代码示例

    def test_as_py(self):
        dd = CSV(self.csv_file, schema=self.schema)

        self.assertEqual(dd.as_py(), [
            {u'f0': u'k1', u'f1': u'v1', u'f2': 1, u'f3': False},
            {u'f0': u'k2', u'f1': u'v2', u'f2': 2, u'f3': True},
            {u'f0': u'k3', u'f1': u'v3', u'f2': 3, u'f3': False}])

def csv():
    csv = CSV('test.csv', schema=schema, mode='w')
    csv.extend(data)
    yield csv
    try:
        os.remove(csv.path)
    except OSError:
        pass

def test_table_resource():
    with tmpfile('csv') as filename:
        csv = CSV(filename, 'w', schema='{x: int, y: int}')
        csv.extend([[1, 2], [10, 20]])

        t = Data(filename)
        assert isinstance(t.data, CSV)
        assert list(compute(t)) == list(csv)

def date_data():
    data = [('Alice', 100.0, datetime(2014, 9, 11, 0, 0, 0, 0)),
            ('Alice', -200.0, datetime(2014, 9, 10, 0, 0, 0, 0)),
            ('Bob', 300.0, None)]
    schema = dshape('{name: string, amount: float32, date: ?datetime}')
    with tmpfile('.csv') as f:
        csv = CSV(f, schema=schema, mode='w')
        csv.extend(data)
        yield CSV(f, schema=schema, mode='r')

def test_into_DataFrame_concat():
    csv = CSV(os.path.join(os.path.dirname(__file__),
                           'accounts.csv'))
    df = into(pd.DataFrame, Concat([csv, csv]))
    csv_df = csv.pandas_read_csv()
    assert df.index.tolist() == list(range(len(df)))
    assert df.values.tolist() == (csv_df.values.tolist() +
                                  csv_df.values.tolist())
    assert df.columns.tolist() == csv_df.columns.tolist()

 def test_extend_structured_many_newlines(self):
     inan = np.array([np.nan]).astype('int32').item()
     with filetext('1,1.0\n2,2.0\n\n\n\n') as fn:
         csv = CSV(fn, 'r+', schema='{x: int32, y: float32}', delimiter=',')
         csv.extend([(3, 3)])
         result = tuplify(tuple(csv))
         expected = ((1, 1.0), (2, 2.0), (inan, np.nan), (inan, np.nan),
                     (inan, np.nan), (3, 3.0))
         assert np.isclose(result, expected, equal_nan=True).all()

    def test_a_mode(self):
        text = ("id, name, balance\n1, Alice, 100\n2, Bob, 200\n"
                "3, Charlie, 300\n4, Denis, 400\n5, Edith, 500")
        with filetext(text) as fn:
            csv = CSV(fn, 'a')
            csv.extend([(6, 'Frank', 600),
                        (7, 'Georgina', 700)])

            assert 'Georgina' in set(csv.py[:, 'name'])

    def test_chunks(self):
        dd = CSV(self.csv_file, schema=self.schema)

        vals = []
        for el in dd.chunks(blen=2):
            self.assertTrue(isinstance(el, nd.array))
            vals.extend(nd.as_py(el))
        self.assertEqual(vals, [
            {u'f0': u'k1', u'f1': u'v1', u'f2': 1, u'f3': False},
            {u'f0': u'k2', u'f1': u'v2', u'f2': 2, u'f3': True},
            {u'f0': u'k3', u'f1': u'v3', u'f2': 3, u'f3': False}])

def test_datetime_csv_reader_same_as_into_types():
    csv = CSV(os.path.join(os.path.dirname(__file__),
                           'accounts.csv'))
    rhs = csv.pandas_read_csv().dtypes
    df = into(pd.DataFrame, csv)
    dtypes = df.dtypes
    expected = pd.Series([np.dtype(x) for x in
                          ['i8', 'i8', 'O', 'datetime64[ns]']],
                         index=csv.columns)
    assert dtypes.index.tolist() == expected.index.tolist()
    assert dtypes.tolist() == expected.tolist()

def test_datetime_csv_reader_same_as_into():
    csv = CSV(os.path.join(os.path.dirname(__file__),
                           'accounts.csv'))
    rhs = csv.pandas_read_csv().dtypes
    df = into(pd.DataFrame, csv)
    dtypes = df.dtypes
    expected = pd.Series([np.dtype(x) for x in
                          ['i8', 'i8', 'O', 'datetime64[ns]']],
                         index=csv.columns)
    # make sure reader with no args does the same thing as into()
    # Values the same
    assert dtypes.index.tolist() == rhs.index.tolist()
    assert dtypes.tolist() == rhs.tolist()

    def test_extend(self):
        dd = CSV(self.filename, 'w', schema=self.schema, delimiter=' ')
        dd.extend(self.data)
        with open(self.filename) as f:
            lines = f.readlines()
            self.assertEqual(lines[0].strip(), 'Alice 100')
            self.assertEqual(lines[1].strip(), 'Bob 200')
            self.assertEqual(lines[2].strip(), 'Alice 50')

        expected_dshape = datashape.DataShape(datashape.Var(), self.schema)
        # TODO: datashape comparison is broken
        self.assertEqual(str(dd.dshape).replace(' ', ''),
                         str(expected_dshape).replace(' ', ''))

def test_extend(tmpcsv, schema):
    dd = CSV(tmpcsv, 'w', schema=schema, delimiter=' ')
    dd.extend(data)
    with open(tmpcsv) as f:
        lines = f.readlines()
    expected_lines = 'Alice 100', 'Bob 200', 'Alice 50'
    for i, eline in enumerate(expected_lines):
        assert lines[i].strip() == eline

    expected_dshape = datashape.DataShape(datashape.Var(),
                                          datashape.dshape(schema))

    assert str(dd.dshape) == str(expected_dshape)

class Test_Dialect(unittest.TestCase):

    buf = sanitize(
    u"""Name Amount
        Alice 100
        Bob 200
        Alice 50
    """)

    schema = "{ f0: string, f1: int }"

    def setUp(self):
        self.csv_file = tempfile.mktemp(".csv")
        with open(self.csv_file, "w") as f:
            f.write(self.buf)
        self.dd = CSV(self.csv_file, dialect='excel', schema=self.schema,
                            delimiter=' ', mode='r+')

    def tearDown(self):
        os.remove(self.csv_file)

    def test_has_header(self):
        assert has_header(self.buf)

    def test_overwrite_delimiter(self):
        self.assertEquals(self.dd.dialect['delimiter'], ' ')

    def test_content(self):
        s = str(list(self.dd))
        assert 'Alice' in s and 'Bob' in s

    def test_append(self):
        self.dd.extend([('Alice', 100)])
        with open(self.csv_file) as f:
            self.assertEqual(f.readlines()[-1].strip(), 'Alice 100')

    def test_append_dict(self):
        self.dd.extend([{'f0': 'Alice', 'f1': 100}])
        with open(self.csv_file) as f:
            self.assertEqual(f.readlines()[-1].strip(), 'Alice 100')

    def test_extend_structured(self):
        with filetext('1,1.0\n2,2.0\n') as fn:
            csv = CSV(fn, 'r+', schema='{x: int32, y: float32}',
                            delimiter=',')
            csv.extend([(3, 3)])
            assert (list(csv) == [[1, 1.0], [2, 2.0], [3, 3.0]]
                 or list(csv) == [{'x': 1, 'y': 1.0},
                                  {'x': 2, 'y': 2.0},
                                  {'x': 3, 'y': 3.0}])

    def test_json_csv_structured(self):
        data = [{'x': 1, 'y': 1}, {'x': 2, 'y': 2}]
        text = '\n'.join(map(json.dumps, data))
        schema = '{x: int, y: int}'

        with filetext(text) as json_fn:
            with filetext('') as csv_fn:
                js = JSON_Streaming(json_fn, schema=schema)
                csv = CSV(csv_fn, mode='r+', schema=schema)

                csv.extend(js)

                self.assertEquals(tuple(map(tuple, (csv))),
                                  ((1, 1), (2, 2)))

 def test_append(self):
     # Get a private file so as to not mess the original one
     csv_file = tempfile.mktemp(".csv")
     with open(csv_file, "w") as f:
         f.write(self.buf)
     dd = CSV(csv_file, schema=self.schema, mode='r+')
     dd.extend([["k4", "v4", 4, True]])
     vals = [nd.as_py(v) for v in dd.chunks(blen=2)]
     self.assertEqual(vals, [
         [{u'f0': u'k1', u'f1': u'v1', u'f2': 1, u'f3': False},
          {u'f0': u'k2', u'f1': u'v2', u'f2': 2, u'f3': True}],
         [{u'f0': u'k3', u'f1': u'v3', u'f2': 3, u'f3': False},
          {u'f0': u'k4', u'f1': u'v4', u'f2': 4, u'f3': True}]])
     self.assertRaises(ValueError, lambda: dd.extend([3.3]))
     os.remove(csv_file)