Python csvvalidator.CSVValidator类代码示例

def test_unique_checks():
    """Test the uniqueness checks."""

    field_names = ('foo', 'bar')
    validator = CSVValidator(field_names)
    validator.add_unique_check('foo')

    data = (
            ('foo', 'bar'),
            ('1', 'A'),
            ('2', 'B'),
            ('1', 'C')
            )

    problems = validator.validate(data)
    n = len(problems)
    assert n == 1, n

    p = problems[0]
    assert p['code'] == UNIQUE_CHECK_FAILED
    assert p['message'] == MESSAGES[UNIQUE_CHECK_FAILED]
    assert p['row'] == 4
    assert p['key'] == 'foo'
    assert p['value'] == '1'
    assert p['record'] == ('1', 'C')

def test_compound_unique_checks_with_variable_record_lengths():
    """Test the uniqueness checks on compound keys when record lengths vary."""

    field_names = ('something', 'foo', 'bar')
    validator = CSVValidator(field_names)
    validator.add_unique_check(('foo', 'bar'), 'X5', 'custom message')

    data = (
            ('something', 'foo', 'bar'),
            ('Z', '1', 'A'),
            ('Z', '2', 'B'),
            ('Z'),
            ('Z', '2', 'A'),
            ('Z', '1', 'A')
            )

    problems = validator.validate(data)
    print problems
    n = len(problems)
    assert n == 1, n

    p = problems[0]
    assert p['code'] == 'X5'
    assert p['message'] == 'custom message'
    assert p['row'] == 6
    assert p['key'] == ('foo', 'bar')
    assert p['value'] == ('1', 'A')
    assert p['record'] == ('Z', '1', 'A')

def test_unique_checks_with_variable_record_lengths():
    """Test the uniqueness checks still work when record lengths vary."""

    field_names = ('foo', 'bar')
    validator = CSVValidator(field_names)
    validator.add_unique_check('bar')

    data = (
            ('foo', 'bar'),
            ('1', 'A'),
            ('2'),
            ('3', 'A')
            )

    problems = validator.validate(data)
    n = len(problems)
    assert n == 1, n

    p = problems[0]
    assert p['code'] == UNIQUE_CHECK_FAILED
    assert p['message'] == MESSAGES[UNIQUE_CHECK_FAILED]
    assert p['row'] == 4
    assert p['key'] == 'bar'
    assert p['value'] == 'A'
    assert p['record'] == ('3', 'A')

def test_compound_unique_checks():
    """Test the uniqueness checks on compound keys."""

    field_names = ('foo', 'bar')
    validator = CSVValidator(field_names)
    validator.add_unique_check(('foo', 'bar'), 'X5', 'custom message')

    data = (
            ('foo', 'bar'),
            ('1', 'A'),
            ('2', 'B'),
            ('1', 'B'),
            ('2', 'A'),
            ('1', 'A')
            )

    problems = validator.validate(data)
    n = len(problems)
    assert n == 1, n

    p = problems[0]
    assert p['code'] == 'X5'
    assert p['message'] == 'custom message'
    assert p['row'] == 6
    assert p['key'] == ('foo', 'bar')
    assert p['value'] == ('1', 'A')
    assert p['record'] == ('1', 'A')

    def test_response_contents(self, register_fields, endpoint):
        response = requests.get(urljoin(endpoint, 'blobs.csv'))

        validator = CSVValidator(['_id'] + register_fields)
        validator.add_header_check()

        problems = validator.validate(csv.reader(response.text.split('\r\n')))

        assert problems == [], '/blobs CSV fields do not match the register definition'

def test_guard_conditions():
    """Test some guard conditions."""

    field_names = ('foo', 'bar')
    validator = CSVValidator(field_names)
    try:
        validator.add_value_check('foo', 'i am not callable')
    except AssertionError:
        pass # expected
    else:
        assert False, 'expected exception'

def test_ignore_lines():
    """Test instructions to ignore lines works."""

    field_names = ('foo', 'bar')
    validator = CSVValidator(field_names)
    validator.add_header_check()
    validator.add_value_check('foo', int)
    validator.add_value_check('bar', float)

    data = (
            ('ignore', 'me', 'please'),
            ('ignore', 'me', 'too', 'please'),
            ('foo', 'baz'),
            ('1.2', 'abc')
            )

    problems = validator.validate(data, ignore_lines=2)
    assert len(problems) == 3

    header_problems = [p for p in problems if p['code'] == HEADER_CHECK_FAILED]
    assert len(header_problems) == 1
    assert header_problems[0]['row'] == 3

    value_problems = [p for p in problems if p['code'] == VALUE_CHECK_FAILED]
    assert len(value_problems) == 2
    for p in value_problems:
        assert p['row'] == 4

def test_value_check_numeric_ranges():
    """Test value checks with numerical range functions."""

    field_names = ('foo', 'bar', 'baz', 'quux')
    validator = CSVValidator(field_names)
    validator.add_value_check('foo', number_range_inclusive(2, 6, int))
    validator.add_value_check('bar', number_range_exclusive(2, 6, int))
    validator.add_value_check('baz', number_range_inclusive(2.0, 6.3, float))
    validator.add_value_check('quux', number_range_exclusive(2.0, 6.3, float))

    data = (
            ('foo', 'bar', 'baz', 'quux'),
            ('2', '3', '2.0', '2.1'), # valid
            ('1', '3', '2.0', '2.1'), # foo invalid
            ('2', '2', '2.0', '2.1'), # bar invalid
            ('2', '3', '1.9', '2.1'), # baz invalid
            ('2', '3', '2.0', '2.0') # quux invalid
            )

    problems = validator.validate(data)
    assert len(problems) == 4, len(problems)
    for p in problems:
        assert p['code'] == VALUE_CHECK_FAILED

    assert problems[0]['row'] == 3 and problems[0]['field'] == 'foo'
    assert problems[1]['row'] == 4 and problems[1]['field'] == 'bar'
    assert problems[2]['row'] == 5 and problems[2]['field'] == 'baz'
    assert problems[3]['row'] == 6 and problems[3]['field'] == 'quux'

def test_value_check_search_pattern():
    """Test value checks with the search_pattern() function."""

    field_names = ('foo', 'bar')
    validator = CSVValidator(field_names)
    validator.add_value_check('bar', search_pattern('\d{4}-\d{2}-\d{2}'))

    data = (
            ('foo', 'bar'),
            ('1', '1999-01-01'),
            ('2', 'abcd-ef-gh'),
            ('3', 'a1999-01-01'), # this is valid - pattern attempts to match anywhere in line
            ('4', '1999-01-01a') # this is valid - pattern attempts to match anywhere in line
            )

    problems = validator.validate(data)
    assert len(problems) == 1, len(problems)
    assert problems[0]['code'] == VALUE_CHECK_FAILED
    assert problems[0]['row'] == 3

def test_value_checks_with_missing_values():
    """
    Establish expected behaviour for value checks where there are missing values
    in the records.

    """

    field_names = ('foo', 'bar')
    validator = CSVValidator(field_names)
    validator.add_value_check('bar', float)

    data = (
            ('foo', 'bar'),
            ('12',) # this is missing value for bar, what happens to value check?
            )

    problems = validator.validate(data)

    # missing values are ignored - use record length checks to find these
    assert len(problems) == 0

def test_header_check():
    """Test the header checks work."""

    field_names = ('foo', 'bar')
    validator = CSVValidator(field_names)
    validator.add_header_check() # use default code and message
    validator.add_header_check(code='X1', message='custom message') # provide custom code and message

    data = (
            ('foo', 'baz'),
            ('123', '456')
            )

    problems = validator.validate(data)
    assert len(problems) == 2

    p0 = problems[0]
    assert p0['code'] == HEADER_CHECK_FAILED
    assert p0['message'] == MESSAGES[HEADER_CHECK_FAILED]
    assert p0['record'] == ('foo', 'baz')
    assert p0['missing'] == set(['bar'])
    assert p0['unexpected'] == set(['baz'])
    assert p0['row'] == 1

    p1 = problems[1]
    assert p1['code'] == 'X1'
    assert p1['message'] == 'custom message'
    assert p1['missing'] == set(['bar'])
    assert p1['unexpected'] == set(['baz'])
    assert p1['record'] == ('foo', 'baz')
    assert p1['row'] == 1

def test_value_checks_datetime_range():
    """Test value checks with datetime ranges."""

    field_names = ('foo', 'bar')
    validator = CSVValidator(field_names)
    validator.add_value_check('bar', datetime_range_inclusive('1999-09-09',
                                                              '2009-09-09',
                                                              '%Y-%m-%d'))
    validator.add_value_check('bar', datetime_range_exclusive('1999-09-09',
                                                              '2009-09-09',
                                                              '%Y-%m-%d'))

    data = (
            ('foo', 'bar'),
            ('A', '1999-09-10'), # valid
            ('B', '1999-09-09'), # invalid (exclusive)
            ('C', '2009-09-09'), # invalid (exclusive)
            ('D', '1999-09-08'), # invalid (both)
            ('E', '2009-09-10') # invalid (both)
            )

    problems = validator.validate(data)

    assert len(problems) == 6, len(problems)
    assert len([p for p in problems if p['row'] == 3]) == 1
    assert len([p for p in problems if p['row'] == 4]) == 1
    assert len([p for p in problems if p['row'] == 5]) == 2
    assert len([p for p in problems if p['row'] == 6]) == 2

def test_value_check_match_pattern():
    """Test value checks with the match_pattern() function."""

    field_names = ('foo', 'bar')
    validator = CSVValidator(field_names)
    validator.add_value_check('bar', match_pattern('\d{4}-\d{2}-\d{2}'))

    data = (
            ('foo', 'bar'),
            ('1', '1999-01-01'),
            ('2', 'abcd-ef-gh'),
            ('3', 'a1999-01-01'),
            ('4', '1999-01-01a') # this is valid - pattern attempts to match at beginning of line
            )

    problems = validator.validate(data)
    assert len(problems) == 2, len(problems)
    for p in problems:
        assert p['code'] == VALUE_CHECK_FAILED

    assert problems[0]['row'] == 3
    assert problems[1]['row'] == 4

def test_context():
    """Test passing in of context information."""

    field_names = ('foo', 'bar')
    validator = CSVValidator(field_names)

    def foo_gt_bar(r):
        return int(r['foo']) > int(r['bar'])
    validator.add_record_predicate(foo_gt_bar)

    data = (
            ('foo', 'bar'),
            ('7', '3'), # valid
            ('1', '3') # invalid
            )

    context = {'info': 'file X'}
    problems = validator.validate(data, context=context)
    n = len(problems)
    assert n == 1, n

    p = problems[0]
    assert p['context'] == context

def test_value_checks_datetime():
    """Test value checks with datetimes."""

    field_names = ('foo', 'bar')
    validator = CSVValidator(field_names)
    validator.add_value_check('bar', datetime_string('%Y-%m-%d'))

    data = (
            ('foo', 'bar'),
            ('A', '1999-09-09'), # valid
            ('B', '1999-13-09'), # invalid month
            ('C', '1999-09-32'), # invalid day
            ('D', '1999-09-09ss') # invalid string
            )

    problems = validator.validate(data)
    assert len(problems) == 3, problems
    for p in problems:
        assert p['code'] == VALUE_CHECK_FAILED

    assert problems[0]['row'] == 3 and problems[0]['field'] == 'bar'
    assert problems[1]['row'] == 4 and problems[1]['field'] == 'bar'
    assert problems[2]['row'] == 5 and problems[2]['field'] == 'bar'