Python models.FileInstance类代码示例

def test_fileinstance_copy_contents(app, db, dummy_location):
    """Test copy contents."""
    counter = dict(called=False)

    def callback(total, size):
        counter['called'] = True

    # Create source and set data.
    data = b('this is some data')
    src = FileInstance.create()
    src.set_contents(BytesIO(data), location=dummy_location)
    db.session.commit()

    # Create destination - and use it to copy_contents from another object.
    dst = FileInstance.create()
    assert dst.size == 0
    assert dst.uri is None
    db.session.commit()

    # Copy contents
    dst.copy_contents(src, progress_callback=callback, location=dummy_location)
    db.session.commit()
    assert dst.size == src.size
    assert dst.checksum == src.checksum
    assert dst.uri != src.uri
    assert counter['called']

    # Read data
    fp = dst.storage().open()
    assert data == fp.read()
    fp.close()

def test_fileinstance_get(app, db, dummy_location):
    """Test fileinstance get."""
    f = FileInstance.create()
    db.session.commit()
    # Get existing file.
    assert FileInstance.get(f.id) is not None
    # Non-existing files returns none
    assert FileInstance.get(uuid.uuid4()) is None

def test_fileinstance_get_by_uri(app, db, dummy_location):
    """Test file get by uri."""
    f = FileInstance.create()
    f.uri = "LICENSE"
    db.session.commit()

    assert FileInstance.get_by_uri("LICENSE") is not None
    FileInstance.get_by_uri("NOTVALID") is None
    pytest.raises(AssertionError, FileInstance.get_by_uri, None)

def test_b2share_storage_with_pid(base_app, app, tmp_location, login_user, test_users):
    """Check that the storage class will redirect pid files."""
    pid = 'http://hdl.handle.net/11304/74c66f0b-f814-4202-9dcb-4889ba9b1047'
    with app.app_context():
        # Disable access control for this test
        tmp_location = Location.query.first()
        with db.session.begin_nested():
            bucket = Bucket.create(tmp_location, storage_class='B')
            pid_file = FileInstance.create()
            pid_file.set_uri(pid, 1, 0, storage_class='B')
            ObjectVersion.create(bucket, 'test.txt', pid_file.id)

        db.session.commit()
        url = url_for('invenio_files_rest.object_api',
                        bucket_id=bucket.id,
                        key='test.txt')
    try:
        with app.app_context():
            permission = current_files_rest.permission_factory
            current_files_rest.permission_factory = allow_all
        # Check that accessing the file redirects to the PID
        with app.test_client() as client:
            resp = client.get(url)
            assert resp.headers['Location'] == pid
            assert resp.status_code == 302
    finally:
        with app.app_context():
            current_files_rest.permission_factory = permission

def test_fileinstance_set_contents(app, db, dummy_location):
    """Test file instance create."""
    counter = dict(called=False)

    def callback(total, size):
        counter['called'] = True

    f = FileInstance.create()
    db.session.commit()
    assert f.readable is False
    assert f.writable is True
    data = BytesIO(b("test file instance set contents"))
    f.set_contents(
        data, default_location=dummy_location.uri, progress_callback=callback)
    db.session.commit()
    assert f.readable is True
    assert f.writable is False
    assert counter['called']

    pytest.raises(
        ValueError,
        f.set_contents,
        BytesIO(b("different content")),
        location=dummy_location,
    )

def test_pyfilesystemstorage_make_path():
    """Test path for files."""
    fi = FileInstance.create()
    fi.id = uuid.uuid5(uuid.NAMESPACE_DNS, 'Testing-')
    fs = PyFilesystemStorage(fi, base_uri='Base')
    assert 'Base/45/629316-6e69-5006-82ba-1ee2f18df5b2' == fs.make_path()
    assert 'Base/4/5629316-6e69-5006-82ba-1ee2f18df5b2' == fs.make_path(1, 1)
    assert 'Base/4/5/6/29316-6e69-5006-82ba-1ee2f18df5b2' == fs.make_path(3, 1)
    assert 'Base/456/29316-6e69-5006-82ba-1ee2f18df5b2' == fs.make_path(1, 3)

    # If length 0, it should take the default value.
    assert 'Base/45/629316-6e69-5006-82ba-1ee2f18df5b2' == fs.make_path(1, 0)

    # If dimensions are 0, it should take the default value.
    assert 'Base/4/5629316-6e69-5006-82ba-1ee2f18df5b2' == fs.make_path(0, 1)

    # Length of each partition is too long.
    with pytest.raises(AssertionError):
        fs.make_path(1, 50)

    # Number of partitions is too high.
    with pytest.raises(AssertionError):
        fs.make_path(50, 1)

    # Both values produce the exception.
    with pytest.raises(AssertionError):
        fs.make_path(50, 50)

def handle_record_files(data, bucket, files, skip_files):
    """Handles record files."""
    for file in files:
        if skip_files:
            break
        assert 'uri' in file
        assert 'size' in file
        assert 'checksum' in file

        try:
            f = FileInstance.create()
            filename = file.get("uri").split('/')[-1:][0]
            f.set_uri(file.get("uri"), file.get(
                "size"), file.get("checksum"))
            obj = ObjectVersion.create(
                bucket,
                filename,
                _file_id=f.id
            )

            file.update({
                'bucket': str(obj.bucket_id),
                'checksum': obj.file.checksum,
                'key': obj.key,
                'version_id': str(obj.version_id),
            })

        except Exception as e:
            click.echo(
                'Recid {0} file {1} could not be loaded due '
                'to {2}.'.format(data.get('recid'), filename,
                                 str(e)))
            continue

def test_pyfilesystemstorage(app, db, dummy_location):
    """Test pyfs storage."""
    # Create bucket and object
    with db.session.begin_nested():
        b1 = Bucket.create()
        obj = ObjectVersion.create(b1, "LICENSE")
        obj.file = FileInstance.create()

    storage = PyFilesystemStorage(obj.file, base_uri=obj.bucket.location.uri)
    counter = dict(size=0)

    def callback(total, size):
        counter['size'] = size

    data = b("this is some content")
    stream = BytesIO(data)
    loc, size, checksum = storage.save(stream, progress_callback=callback)

    # Verify checksum, size and location.
    m = hashlib.md5()
    m.update(data)
    assert "md5:{0}".format(m.hexdigest()) == checksum

    assert size == len(data)
    assert loc == join(
        dummy_location.uri,
        str(obj.file.id),
        "data")

def test_object_relink_all(app, db, dummy_location):
    """Test relinking files."""
    b1 = Bucket.create()
    obj1 = ObjectVersion.create(
        b1, "relink-test", stream=BytesIO(b('relinkthis')))
    ObjectVersion.create(b1, "do-not-touch", stream=BytesIO(b('na')))
    b1.snapshot()
    db.session.commit()

    assert ObjectVersion.query.count() == 4
    assert FileInstance.query.count() == 2

    fnew = FileInstance.create()
    fnew.copy_contents(obj1.file, location=b1.location)
    db.session.commit()

    fold = obj1.file

    assert ObjectVersion.query.filter_by(file_id=fold.id).count() == 2
    assert ObjectVersion.query.filter_by(file_id=fnew.id).count() == 0

    ObjectVersion.relink_all(obj1.file, fnew)
    db.session.commit()

    assert ObjectVersion.query.filter_by(file_id=fold.id).count() == 0
    assert ObjectVersion.query.filter_by(file_id=fnew.id).count() == 2

def test_fileinstance_create(app, db, dummy_location):
    """Test file instance create."""
    f = FileInstance.create()
    assert f.id
    assert f.readable is False
    assert f.writable is True
    assert f.uri is None
    assert f.size == 0
    assert f.checksum is None
    assert f.last_check_at is None
    assert f.last_check is None
    db.session.commit()

    # Check unique constraint on URI with none values.
    f = FileInstance.create()
    f = FileInstance.create()
    db.session.commit()

def data_policies(skip_files):
    """Load demo Data Policy records."""
    from invenio_db import db
    from invenio_indexer.api import RecordIndexer
    from cernopendata.modules.records.minters.recid import \
        cernopendata_recid_minter

    from invenio_files_rest.models import \
        Bucket, FileInstance, ObjectVersion
    from invenio_records_files.models import RecordsBuckets
    from invenio_records_files.api import Record

    from invenio_records.models import RecordMetadata

    indexer = RecordIndexer()
    schema = current_app.extensions['invenio-jsonschemas'].path_to_url(
        'records/data-policies-v1.0.0.json'
    )
    data = pkg_resources.resource_filename('cernopendata',
                                           'modules/fixtures/data')
    data_policies_json = glob.glob(os.path.join(data, '*.json'))

    for filename in data_policies_json:

        click.echo('Loading data-policies from {0} ...'.format(filename))

        with open(filename, 'rb') as source:
            for data in json.load(source):
                files = data.pop('files', [])

                id = uuid.uuid4()
                cernopendata_recid_minter(id, data)
                data['$schema'] = schema
                record = Record.create(data, id_=id)

                bucket = Bucket.create()
                RecordsBuckets.create(
                    record=record.model, bucket=bucket)

                for file in files:
                    if skip_files:
                        break
                    assert 'uri' in file
                    assert 'size' in file
                    assert 'checksum' in file

                    f = FileInstance.create()
                    filename = file.get("uri").split('/')[-1:][0]
                    f.set_uri(file.get("uri"), file.get(
                        "size"), file.get("checksum"))
                    ObjectVersion.create(
                        bucket,
                        filename,
                        _file_id=f.id
                    )
                db.session.commit()
                indexer.index(record)
                db.session.expunge_all()

def create_b2safe_file(external_pids, bucket):
    """Create a FileInstance which contains a PID in its uri."""
    validate_schema(external_pids, {
        'type': 'array',
        'items': {
            'type': 'object',
            'properties': {
                'ePIC_PID': {'type': 'string'},
                'key': {'type': 'string'}
            },
            'additionalProperties': False,
            'required': ['ePIC_PID', 'key']
        }
    })

    keys_list = [e['key'] for e in external_pids]
    keys_set = set(keys_list)
    if len(keys_list) != len(keys_set):
        raise InvalidDepositError([FieldError('external_pids',
            'Field external_pids contains duplicate keys.')])
    for external_pid in external_pids:
        if not external_pid['ePIC_PID'].startswith('http://hdl.handle.net/'):
            external_pid['ePIC_PID'] = 'http://hdl.handle.net/' + \
                external_pid['ePIC_PID']
        if external_pid['key'].startswith('/'):
            raise InvalidDepositError(
                [FieldError('external_pids',
                            'File key cannot start with a "/".')])
        try:
            # Create the file instance if it does not already exist
            file_instance = FileInstance.get_by_uri(external_pid['ePIC_PID'])
            if file_instance is None:
                file_instance = FileInstance.create()
                file_instance.set_uri(
                    external_pid['ePIC_PID'], 1, 0, storage_class='B')
            assert file_instance.storage_class == 'B'
            # Add the file to the bucket if it is not already in it
            current_version = ObjectVersion.get(bucket, external_pid['key'])
            if not current_version or \
                    current_version.file_id != file_instance.id:
                ObjectVersion.create(bucket, external_pid['key'],
                                     file_instance.id)
        except IntegrityError as e:
            raise InvalidDepositError(
                [FieldError('external_pids', 'File URI already exists.')])

def test_storage_interface():
    """Test storage interface."""
    f = FileInstance.create()
    s = Storage(f)

    pytest.raises(NotImplementedError, s.open)
    pytest.raises(NotImplementedError, s.send_file)
    pytest.raises(NotImplementedError, s.save, None)
    pytest.raises(NotImplementedError, s.compute_checksum, None)

def test_pyfs_send_file_fail(app, db, dummy_location):
    """Test send file."""
    f = FileInstance.create()
    f.set_contents(BytesIO(b("test")), location=dummy_location)

    with patch('invenio_files_rest.storage.send_stream') as send_stream:
        send_stream.side_effect = OSError(errno.EPERM, "Permission problem")
        with app.test_request_context():
            pytest.raises(StorageError, f.send_file)

def test_sip_file_model(db):
    """Test the SIPFile model."""
    sip1 = SIP.create('json', '{}')
    file1 = FileInstance.create()
    sipfile1 = SIPFile(sip_id=sip1.id, filepath="foobar.zip",
                       file_id=file1.id)

    db.session.add(sipfile1)
    db.session.commit()
    assert SIP.query.count() == 1
    assert SIPFile.query.count() == 1