Python RecordIndexer.index方法代码示例

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def records():
    """Load records."""
    import pkg_resources
    import uuid
    from flask_login import login_user, logout_user
    from dojson.contrib.marc21 import marc21
    from dojson.contrib.marc21.utils import create_record, split_blob
    from invenio_accounts.models import User
    from invenio_deposit.api import Deposit

    users = User.query.all()

    # pkg resources the demodata
    data_path = pkg_resources.resource_filename(
        'invenio_records', 'data/marc21/bibliographic.xml'
    )
    with open(data_path) as source:
        with current_app.test_request_context():
            indexer = RecordIndexer()
            with db.session.begin_nested():
                for index, data in enumerate(split_blob(source.read()),
                                             start=1):
                    login_user(users[index % len(users)])
                    # do translate
                    record = marc21.do(create_record(data))
                    # create record
                    indexer.index(Deposit.create(record))
                    logout_user()
            db.session.commit()

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def records():
    """Load records."""
    import pkg_resources
    import uuid
    from dojson.contrib.marc21 import marc21
    from dojson.contrib.marc21.utils import create_record, split_blob
    from invenio_pidstore import current_pidstore
    from invenio_records.api import Record

    # pkg resources the demodata
    data_path = pkg_resources.resource_filename(
        'invenio_records', 'data/marc21/bibliographic.xml'
    )
    with open(data_path) as source:
        indexer = RecordIndexer()
        with db.session.begin_nested():
            for index, data in enumerate(split_blob(source.read()), start=1):
                # create uuid
                rec_uuid = uuid.uuid4()
                # do translate
                record = marc21.do(create_record(data))
                # create PID
                current_pidstore.minters['recid_minter'](
                    rec_uuid, record
                )
                # create record
                indexer.index(Record.create(record, id_=rec_uuid))
        db.session.commit()

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def index_after_commit(sender, changes):
    """Index a record in ES after it was committed to the DB.

    This cannot happen in an ``after_record_commit`` receiver from Invenio-Records
    because, despite the name, at that point we are not yet sure whether the record
    has been really committed to the DB.
    """
    indexer = RecordIndexer()
    for model_instance, change in changes:
        if isinstance(model_instance, RecordMetadata):
            if change in ('insert', 'update') and not model_instance.json.get("deleted"):
                if hasattr(model_instance, '_enhanced_record'):
                    record = model_instance._enhanced_record
                else:
                    record = model_instance.json
                indexer.index(InspireRecord(record, model_instance))
            else:
                try:
                    indexer.delete(InspireRecord(
                        model_instance.json, model_instance))
                except NotFoundError:
                    # Record not found in ES
                    LOGGER.debug('Record %s not found in ES',
                                 model_instance.json.get("id"))
                    pass

            pid_type = get_pid_type_from_schema(model_instance.json['$schema'])
            pid_value = model_instance.json['control_number']
            db_version = model_instance.version_id

            index_modified_citations_from_record.delay(pid_type, pid_value, db_version)

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def load_records(app, filename, schema, tries=5):
    """Try to index records."""
    indexer = RecordIndexer()
    records = []
    with app.app_context():
        with mock.patch('invenio_records.api.Record.validate',
                        return_value=None):
            data_filename = pkg_resources.resource_filename(
                'invenio_records', filename)
            records_data = load(data_filename)
            with db.session.begin_nested():
                for item in records_data:
                    record_id = uuid.uuid4()
                    item_dict = dict(marc21.do(item))
                    item_dict['$schema'] = schema
                    recid_minter(record_id, item_dict)
                    oaiid_minter(record_id, item_dict)
                    record = Record.create(item_dict, id_=record_id)
                    indexer.index(record)
                    records.append(record.id)
            db.session.commit()

        # Wait for indexer to finish
        for i in range(tries):
            response = current_search_client.search()
            if response['hits']['total'] >= len(records):
                break
            current_search.flush_and_refresh('_all')

    return records

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def glossary_terms():
    """Load demo terms records."""
    from invenio_db import db
    from invenio_records import Record
    from invenio_indexer.api import RecordIndexer
    from cernopendata.modules.records.minters.termid import \
        cernopendata_termid_minter

    indexer = RecordIndexer()
    schema = current_app.extensions['invenio-jsonschemas'].path_to_url(
        'records/glossary-term-v1.0.0.json'
    )
    data = pkg_resources.resource_filename('cernopendata',
                                           'modules/fixtures/data')
    glossary_terms_json = glob.glob(os.path.join(data, 'terms', '*.json'))

    for filename in glossary_terms_json:

        click.echo('Loading glossary-terms from {0} ...'.format(filename))

        with open(filename, 'rb') as source:
            for data in json.load(source):
                if "collections" not in data and \
                    not isinstance(
                        data.get("collections", None), basestring):
                    data["collections"] = []
                data["collections"].append({"primary": "Terms"})
                id = uuid.uuid4()
                cernopendata_termid_minter(id, data)
                data['$schema'] = schema
                record = Record.create(data, id_=id)
                db.session.commit()
                indexer.index(record)
                db.session.expunge_all()

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def record_not_yet_deleted(app):
    snippet = (
        '<record>'
        '  <controlfield tag="001">333</controlfield>'
        '  <controlfield tag="005">20160913214552.0</controlfield>'
        '  <datafield tag="980" ind1=" " ind2=" ">'
        '    <subfield code="a">HEP</subfield>'
        '  </datafield>'
        '</record>'
    )

    with app.app_context():
        json_record = hep.do(create_record(snippet))
        json_record['$schema'] = 'http://localhost:5000/schemas/records/hep.json'

        with db.session.begin_nested():
            record = record_upsert(json_record)
            if record:
                ri = RecordIndexer()
                ri.index(record)

        db.session.commit()

    yield

    with app.app_context():
        _delete_record_from_everywhere('literature', 333)

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def test_basic_search(app, db, es):
    """Test basic search functionality."""
    # The index should be empty
    assert len(ItemSearch().execute()) == 0

    # Create item1, search for everything
    item1 = Item.create({})
    item1.commit()

    record_indexer = RecordIndexer()
    record_indexer.index(item1)

    current_search.flush_and_refresh('_all')

    assert len(ItemSearch().execute()) == 1

    # Create item2, search for everything again
    item2 = Item.create({'foo': 'bar'})
    item2.commit()
    record_indexer.index(item2)

    current_search.flush_and_refresh('_all')

    assert len(ItemSearch().execute()) == 2

    # Search for item2
    assert len(ItemSearch().query('match', foo='bar').execute()) == 1

    # Search for nonsense
    assert len(ItemSearch().query('match', foo='banana').execute()) == 0

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def data_policies(skip_files):
    """Load demo Data Policy records."""
    from invenio_db import db
    from invenio_indexer.api import RecordIndexer
    from cernopendata.modules.records.minters.recid import \
        cernopendata_recid_minter

    from invenio_files_rest.models import \
        Bucket, FileInstance, ObjectVersion
    from invenio_records_files.models import RecordsBuckets
    from invenio_records_files.api import Record

    from invenio_records.models import RecordMetadata

    indexer = RecordIndexer()
    schema = current_app.extensions['invenio-jsonschemas'].path_to_url(
        'records/data-policies-v1.0.0.json'
    )
    data = pkg_resources.resource_filename('cernopendata',
                                           'modules/fixtures/data')
    data_policies_json = glob.glob(os.path.join(data, '*.json'))

    for filename in data_policies_json:

        click.echo('Loading data-policies from {0} ...'.format(filename))

        with open(filename, 'rb') as source:
            for data in json.load(source):
                files = data.pop('files', [])

                id = uuid.uuid4()
                cernopendata_recid_minter(id, data)
                data['$schema'] = schema
                record = Record.create(data, id_=id)

                bucket = Bucket.create()
                RecordsBuckets.create(
                    record=record.model, bucket=bucket)

                for file in files:
                    if skip_files:
                        break
                    assert 'uri' in file
                    assert 'size' in file
                    assert 'checksum' in file

                    f = FileInstance.create()
                    filename = file.get("uri").split('/')[-1:][0]
                    f.set_uri(file.get("uri"), file.get(
                        "size"), file.get("checksum"))
                    ObjectVersion.create(
                        bucket,
                        filename,
                        _file_id=f.id
                    )
                db.session.commit()
                indexer.index(record)
                db.session.expunge_all()

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def receive_after_model_commit(sender, changes):
    """Perform actions after models committed to database."""
    indexer = RecordIndexer()
    for model_instance, change in changes:
        if isinstance(model_instance, RecordMetadata):
            if change in ('insert', 'update'):
                indexer.index(InspireRecord(model_instance.json, model_instance))
            else:
                indexer.delete(InspireRecord(model_instance.json, model_instance))

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def closed_access_record(db, es, record_with_files_creation):
    """Creation of a full record with closed access right."""
    _, record, record_url = record_with_files_creation
    record['access_right'] = AccessRight.CLOSED
    record.commit()
    db.session.commit()
    indexer = RecordIndexer()
    indexer.index(record)
    current_search.flush_and_refresh(index='records')
    return record

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def create_record(app, item_dict, mint_oaiid=True):
    """Create test record."""
    indexer = RecordIndexer()
    with app.test_request_context():
        record_id = uuid.uuid4()
        recid_minter(record_id, item_dict)
        if mint_oaiid:
            oaiid_minter(record_id, item_dict)
        record = Record.create(item_dict, id_=record_id)
        indexer.index(record)
        return record

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def indexed_loans(es, test_loans):
    """Get a function to wait for records to be flushed to index."""
    indexer = RecordIndexer()
    for pid, loan in test_loans:
        indexer.index(loan)
    current_search.flush_and_refresh(index="loans")

    yield test_loans

    for pid, loan in test_loans:
        indexer.delete_by_id(loan.id)
    current_search.flush_and_refresh(index="loans")

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def _create_and_index_record(record):
    record = Record.create(record)
    inspire_recid_minter(record.id, record)
    # invenio-collections will populate _collections field in record upon
    # commit
    db.session.commit()

    # Record needs to be indexed since views fetch records from ES
    r = RecordIndexer()
    r.index(record)
    es.indices.refresh('records-hep')

    return record

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def oaiserver(sets, records):
    """Initialize OAI-PMH server."""
    from invenio_db import db
    from invenio_oaiserver.models import OAISet
    from invenio_records.api import Record

    # create a OAI Set
    with db.session.begin_nested():
        for i in range(sets):
            db.session.add(OAISet(
                spec='test{0}'.format(i),
                name='Test{0}'.format(i),
                description='test desc {0}'.format(i),
                search_pattern='title_statement.title:Test{0}'.format(i),
            ))

    # create a record
    schema = {
        'type': 'object',
        'properties': {
            'title_statement': {
                'type': 'object',
                'properties': {
                    'title': {
                        'type': 'string',
                    },
                },
            },
            'field': {'type': 'boolean'},
        },
    }

    search.client.indices.delete_alias('_all', '_all', ignore=[400, 404])
    search.client.indices.delete('*')

    with app.app_context():
        indexer = RecordIndexer()
        with db.session.begin_nested():
            for i in range(records):
                record_id = uuid.uuid4()
                data = {
                    'title_statement': {'title': 'Test{0}'.format(i)},
                    '$schema': schema,
                }
                recid_minter(record_id, data)
                oaiid_minter(record_id, data)
                record = Record.create(data, id_=record_id)
                indexer.index(record)

        db.session.commit()

# 需要导入模块: from invenio_indexer.api import RecordIndexer [as 别名]
# 或者: from invenio_indexer.api.RecordIndexer import index [as 别名]
def test_record_can_be_deleted(app, record_not_yet_deleted):
    with app.test_client() as client:
        assert client.get('/api/literature/333').status_code == 200

    record = get_db_record('literature', 333)
    record['deleted'] = True
    record.commit()
    if record:
        ri = RecordIndexer()
        ri.index(record)
    db.session.commit()

    with app.test_client() as client:
        assert client.get('/api/literature/333').status_code == 410