Python tableschema.Schema方法代码示例

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def schema_validator(resource, iterator,
                     field_names=None, on_error=None):
    if on_error is None:
        on_error = raise_exception
    on_error = wrap_handler(on_error)

    if isinstance(resource, Resource):
        schema: Schema = resource.schema
        assert schema is not None
        resource = resource.descriptor
    else:
        schema: Schema = Schema(resource.get('schema', {}))
    if field_names is None:
        field_names = [f.name for f in schema.fields]
    schema_fields = [f for f in schema.fields if f.name in field_names]
    for i, row in enumerate(iterator):
        field = None
        try:
            for field in schema_fields:
                row[field.name] = field.cast_value(row.get(field.name))
        except CastError as e:
            if not on_error(resource['name'], row, i, e, field):
                continue

        yield row 

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def foreign_keys(self):
        """Schema's foreign keys

        # Returns
            dict[]: foreign keys

        """
        foreign_keys = self.__current_descriptor.get('foreignKeys', [])
        for key in foreign_keys:
            key.setdefault('fields', [])
            key.setdefault('reference', {})
            key['reference'].setdefault('resource', '')
            key['reference'].setdefault('fields', [])
            if not isinstance(key['fields'], list):
                key['fields'] = [key['fields']]
            if not isinstance(key['reference']['fields'], list):
                key['reference']['fields'] = [key['reference']['fields']]
        return foreign_keys 

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def test_cast_row_too_short_handled():
    schema = Schema(DESCRIPTOR_MAX)
    source = ['string', '10.0', '1', 'string']
    # Missing values get substituted by None
    target = ['string', Decimal(10.0), 1, 'string', None]
    errors = []
    def handler(exc, row_number, row_data, error_data):
        errors.append((exc, row_number, row_data, error_data))
    assert schema.cast_row(source, exc_handler=handler) == target
    assert len(errors) == 1
    expect_row_data = OrderedDict(
        [('id', 'string'), ('height', '10.0'), ('age', '1'),
         ('name', 'string'), ('occupation', None)])
    _check_error(
        errors[0], expect_exc_class=exceptions.CastError,
        expect_exc_str='Row length', expect_row_number=None,
        expect_row_data=expect_row_data, expect_error_data=expect_row_data) 

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def test_cast_row_wrong_type_handled():
    schema = Schema(DESCRIPTOR_MAX)
    source = ['string', 'notdecimal', '1', 'string', 'string']
    target = ['string', 'notdecimal', 1, 'string', 'string']
    errors = []
    def handler(exc, row_number, row_data, error_data):
        errors.append((exc, row_number, row_data, error_data))
    actual = schema.cast_row(source, exc_handler=handler)
    assert actual == target
    assert isinstance(actual[1], FailedCast)
    assert len(errors) == 1
    expect_row_data = OrderedDict(
        [('id', 'string'), ('height', 'notdecimal'), ('age', '1'),
         ('name', 'string'), ('occupation', 'string')])
    expect_error_data = OrderedDict([('height', 'notdecimal')])
    _check_error(
        errors[0], expect_exc_class=exceptions.CastError,
        expect_exc_str='There are 1 cast errors', expect_row_number=None,
        expect_row_data=expect_row_data, expect_error_data=expect_error_data)
    exc = errors[0][0]
    assert len(exc.errors) == 1 

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def test_cast_row_wrong_type_multiple_errors_handled():
    schema = Schema(DESCRIPTOR_MAX)
    source = ['string', 'notdecimal', '10.6', 'string', 'string']
    target = ['string', 'notdecimal', '10.6', 'string', 'string']
    errors = []
    def handler(exc, row_number, row_data, error_data):
        errors.append((exc, row_number, row_data, error_data))
    actual = schema.cast_row(source, exc_handler=handler)
    assert actual == target
    assert isinstance(actual[1], FailedCast)
    assert isinstance(actual[2], FailedCast)
    assert len(errors) == 1
    expect_row_data = OrderedDict(
        [('id', 'string'), ('height', 'notdecimal'), ('age', '10.6'),
         ('name', 'string'), ('occupation', 'string')])
    expect_error_data = OrderedDict(
        [('height', 'notdecimal'),('age', '10.6')])
    _check_error(
        errors[0], expect_exc_class=exceptions.CastError,
        expect_exc_str='There are 2 cast errors', expect_row_number=None,
        expect_row_data=expect_row_data, expect_error_data=expect_error_data)
    exc = errors[0][0]
    assert len(exc.errors) == 2 

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def iter(self, bucket):

        # Get table and fallbacks
        table = self.__get_table(bucket)
        schema = tableschema.Schema(self.describe(bucket))
        autoincrement = self.__get_autoincrement_for_bucket(bucket)

        # Open and close transaction
        with self.__connection.begin():
            # Streaming could be not working for some backends:
            # http://docs.sqlalchemy.org/en/latest/core/connections.html
            select = table.select().execution_options(stream_results=True)
            result = select.execute()
            for row in result:
                row = self.__mapper.restore_row(
                    row, schema=schema, autoincrement=autoincrement)
                yield row 

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def check_headers(self, cells, sample):
        errors = []

        for cell in copy(cells):

            # Skip if cell has field
            if 'field' in cell and cell['field'] is not None:
                continue

            # Infer field
            if self.__infer_fields:
                column_sample = []
                for row in sample:
                    value = None
                    if len(row) >= cell['column-number']:
                        value = row[cell['column-number'] - 1]
                    column_sample.append([value])
                schema = Schema()
                schema.infer(column_sample, headers=[cell.get('header')])
                cell['field'] = schema.fields[0]

            # Add error/remove column
            else:
                message_substitutions = {
                    'header': '"{}"'.format(cell.get('header')),
                }
                error = Error(
                    'extra-header',
                    cell,
                    message_substitutions=message_substitutions
                )
                errors.append(error)
                cells.remove(cell)

        return errors 

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def table(source, schema=None, **options):
    warnings = []
    tables = []

    # Ensure not a datapackage
    if isinstance(source, six.string_types):
        if source.endswith('datapackage.json'):
            warnings.append('Use "datapackage" preset for Data Packages')

    # Prepare schema
    if not warnings:
        if schema is not None:
            try:
                schema = Schema(schema)
            except exceptions.TableSchemaException as error:
                warnings.append(
                    'Table Schema "%s" has a loading error "%s"' %
                    (schema, error))

    # Add table
    if not warnings:
        options.setdefault('headers', 1)
        tables.append({
            'source': str(source) if isinstance(source, six.string_types) else 'inline',
            'stream': Stream(source, **options),
            'schema': schema,
            'extra': {},
        })

    return warnings, tables 

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def __infer(self, sample, headers=None):
        if headers is None:
            headers = [None] * len(sample[0]) if sample else []
        schema = tableschema.Schema()
        schema.infer(sample, headers=headers)
        return schema 

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def descriptor(self):
        """Schema's descriptor

        # Returns
            dict: descriptor

        """
        # Never use this.descriptor inside this class (!!!)
        return self.__next_descriptor 

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def missing_values(self):
        """Schema's missing values

        # Returns
            str[]: missing values

        """
        return self.__current_descriptor.get('missingValues', []) 

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def primary_key(self):
        """Schema's primary keys

        # Returns
            str[]: primary keys

        """
        primary_key = self.__current_descriptor.get('primaryKey', [])
        if not isinstance(primary_key, list):
            primary_key = [primary_key]
        return primary_key 

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def fields(self):
        """Schema's fields

        # Returns
            Field[]: an array of field instances

        """
        return self.__fields 

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def test_infer_schema():
    runner = CliRunner()
    result = runner.invoke(infer, ['data/data_infer.csv'])
    # output is a string, evaluate to a dict
    schema = ast.literal_eval(result.output)
    schema_model = Schema(schema)
    assert schema_model.get_field('id').type == 'integer'
    assert schema_model.get_field('age').type == 'integer'
    assert schema_model.get_field('name').type == 'string' 

# 需要导入模块: import tableschema [as 别名]
# 或者: from tableschema import Schema [as 别名]
def test_infer_schema_utf8():
    """UTF8 encoded data containing non-ascii characters."""
    runner = CliRunner()
    result = runner.invoke(infer, ['data/data_infer_utf8.csv'])
    # output is a string, evaluate to a dict
    schema = ast.literal_eval(result.output)
    schema_model = Schema(schema)
    assert schema_model.get_field('id').type == 'integer'
    assert schema_model.get_field('age').type == 'integer'
    assert schema_model.get_field('name').type == 'string'