Python datapackage.Package方法代码示例

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def test_descriptor_dereferencing_uri_remote():
    # Mocks
    httpretty.register_uri(httpretty.GET,
        'http://example.com/schema', body='{"fields": [{"name": "name"}]}')
    httpretty.register_uri(httpretty.GET,
        'https://example.com/dialect', body='{"delimiter": ","}')
    # Tests
    descriptor = {
        'resources': [
            {'name': 'name1', 'data': 'data', 'schema': 'http://example.com/schema'},
            {'name': 'name2', 'data': 'data', 'dialect': 'https://example.com/dialect'},
         ],
    }
    package = Package(descriptor)
    assert package.descriptor['resources'] == list(map(helpers.expand_resource_descriptor, [
        {'name': 'name1', 'data': 'data', 'schema': {'fields': [{'name': 'name'}]}},
        {'name': 'name2', 'data': 'data', 'dialect': {'delimiter': ','}},
    ])) 

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def test_descriptor_apply_defaults_resource_tabular_schema():
    descriptor = {
        'resources': [{
            'name': 'name',
            'data': 'data',
            'profile': 'tabular-data-resource',
            'schema': {
                'fields': [{'name': 'name'}],
            }
        }],
    }
    package = Package(descriptor)
    assert package.descriptor == {
        'profile': 'data-package',
        'resources': [{
            'name': 'name',
            'data': 'data',
            'profile': 'tabular-data-resource',
            'schema': {
                'fields': [{'name': 'name', 'type': 'string', 'format': 'default'}],
                'missingValues': [''],
            }
        }],
    } 

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def test_adds_resources_inside_data_subfolder(tmpfile):
    descriptor = {
        'name': 'proverbs',
        'resources': [
            {'name': 'name', 'path': 'unicode.txt'}
        ]
    }
    schema = {}
    package = Package(descriptor, schema, default_base_path='data')
    package.save(tmpfile)
    with zipfile.ZipFile(tmpfile, 'r') as z:
        filename = [name for name in z.namelist()
                    if name.startswith('data/')]
        assert len(filename) == 1
        resource_data = z.read(filename[0]).decode('utf-8')
    assert resource_data == '万事开头难\n' 

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def test_cli_init():
    resource_path = 'data/valid.csv'

    result = CliRunner().invoke(init, [resource_path])

    assert result.exit_code == 0

    dp = datapackage.Package(json.loads(result.output), strict=True)
    resource = dp.resources[0]
    assert resource.descriptor['path'] == resource_path
    assert 'schema' in resource.descriptor 

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def datapackage(source, **options):
    warnings = []
    tables = []

    # Load datapackage
    try:
        package = Package(source, **options)
    except exceptions.DataPackageException as error:
        warnings.append(
            'Data Package "%s" has a loading error "%s"' %
            (source, error))

    # Extract datapackage tables
    if not warnings:
        for resource in package.resources:
            if resource.tabular:
                hash = resource.descriptor.get('hash')
                if hash and not hash.startswith('sha256:'):
                    message = 'Resource "%s" does not use the SHA256 hash. The check will be skipped'
                    warnings.append(message % resource.name)
                tables.append({
                    'source': resource.source if not resource.inline else 'inline',
                    'stream': Stream(partial(_iter_resource_rows, resource), headers=1),
                    'schema': resource.schema,
                    'extra': {
                        'datapackage': json.dumps(source) if isinstance(source, dict) else source,
                        'resource-name': resource.name,
                    },
                })

    # Extrace datapackage errors
    if not warnings:
        for error in package.errors:
            warnings.append(
                'Data Package "%s" has a validation error "%s"' %
                (source, error))

    return warnings, tables


# Internal 

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def init_datapackage(resource_paths):
    # "Create tabular data package with resources.
    dp = Package({
        'name': 'change-me',
        'schema': 'tabular-data-package',
    })

    for path in resource_paths:
        dp.infer(path)

    return dp 

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def prepare(self, stream, schema, extra):

        # Prepare package
        if 'datapackage' not in extra or 'resource-name' not in extra:
            return False
        descriptor = extra['datapackage']
        if descriptor.strip().startswith('{'):
            descriptor = json.loads(descriptor)
        self.__package = datapackage.Package(descriptor)

        # Prepare schema
        if not schema:
            return False
        if not schema.foreign_keys:
            return False
        self.__schema = schema

        # Prepare foreign keys values
        try:
            self.__relations = _get_relations(
                self.__package, self.__schema,
                current_resource_name=extra['resource-name'])
            self.__foreign_keys_values = _get_foreign_keys_values(
                self.__schema, self.__relations)
            self.__relations_exception = None
        except _ReferenceTableError as exception:
            self.__relations_exception = exception

        return True 

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def _get_relations(package, schema, current_resource_name=None):
    # It's based on the following code:
    # https://github.com/frictionlessdata/datapackage-py/blob/master/datapackage/resource.py#L393

    # Prepare relations
    relations = {}
    for fk in schema.foreign_keys:
        resource_name = fk['reference'].get('resource')
        package_name = fk['reference'].get('package')
        resource = None

        # Self-referenced resource
        if not resource_name:
            for item in package.resources:
                if item.name == current_resource_name:
                    resource = item

        # Internal resource
        elif not package_name:
            resource = package.get_resource(resource_name)

        # External resource (experimental)
        # For now, we rely on uniqueness of resource names and support relative paths
        else:
            descriptor = package_name
            if not descriptor.startswith('http'):
                descriptor = '/'.join([package.base_path, package_name])
            package = datapackage.Package(descriptor)
            resource = package.get_resource(resource_name)

        # Add to relations (can be None)
        relations[resource_name] = resource
        if resource and resource.tabular:
            try:
                relations[resource_name] = resource.read(keyed=True)
            # TODO: datapackage should raise `IntegrityError` here
            except datapackage.exceptions.CastError as exception:
                raise _ReferenceTableError('[%s] %s' % (resource_name, str(exception)))

    return relations 

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def test_init_accepts_dicts():
    descriptor = {
        'profile': 'data-package',
    }
    package = Package(descriptor)
    assert package.descriptor == descriptor 

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def test_init_accepts_filelike_object():
    descriptor = {
        'profile': 'data-package',
    }
    filelike_descriptor = six.StringIO(json.dumps(descriptor))
    package = Package(filelike_descriptor)
    assert package.descriptor == descriptor 

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def test_init_accepts_file_paths():
    pakcage = Package('data/empty_datapackage.json')
    assert pakcage.descriptor == {
        'profile': 'data-package',
    } 

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def test_init_raises_if_file_path_doesnt_exist():
    path = 'this-file-doesnt-exist.json'
    with pytest.raises(exceptions.DataPackageException):
        Package(path) 

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def test_init_raises_if_path_isnt_a_json():
    with pytest.raises(exceptions.DataPackageException):
        Package('data/not_a_json') 

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def test_init_raises_if_path_json_isnt_a_dict():
    with pytest.raises(exceptions.DataPackageException):
        Package('data/empty_array.json') 

# 需要导入模块: import datapackage [as 别名]
# 或者: from datapackage import Package [as 别名]
def test_init_raises_if_filelike_object_isnt_a_json():
    invalid_json = six.StringIO('{"foo"}')
    with pytest.raises(exceptions.DataPackageException):
        Package(invalid_json)