Python TestRecordMetadata.create_from_file方法代码示例

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
def test_train_and_save_distance_model(isolated_app, tmpdir):
    TestRecordMetadata.create_from_file(__name__, '765515.json')
    TestRecordMetadata.create_from_file(__name__, '765975.json')

    curated_signatures_fd = tmpdir.join('curated_signatures.jsonl')
    sampled_pairs_fd = tmpdir.join('sampled_pairs.jsonl')
    publications_fd = tmpdir.join('publications.jsonl')
    ethnicity_data_fd = tmpdir.join('ethnicity.csv')
    ethnicity_data_fd.write(ETHNICITY_TRAINING_DATA)
    ethnicity_model_fd = tmpdir.join('ethnicity.pkl')
    distance_model_fd = tmpdir.join('distance.pkl')

    config = {
        'DISAMBIGUATION_CURATED_SIGNATURES_PATH': str(curated_signatures_fd),
        'DISAMBIGUATION_SAMPLED_PAIRS_PATH': str(sampled_pairs_fd),
        'DISAMBIGUATION_SAMPLED_PAIRS_SIZE': 12 * 100,
        'DISAMBIGUATION_PUBLICATIONS_PATH': str(publications_fd),
        'DISAMBIGUATION_ETHNICITY_DATA_PATH': str(ethnicity_data_fd),
        'DISAMBIGUATION_ETHNICITY_MODEL_PATH': str(ethnicity_model_fd),
        'DISAMBIGUATION_DISTANCE_MODEL_PATH': str(distance_model_fd)
    }

    with patch.dict(current_app.config, config):
        save_curated_signatures_and_input_clusters()
        save_sampled_pairs()
        save_publications()
        train_and_save_ethnicity_model()
        train_and_save_distance_model()

    ethnicity_estimator = EthnicityEstimator()
    ethnicity_estimator.load_model(str(ethnicity_model_fd))

    distance_estimator = DistanceEstimator(ethnicity_estimator)
    distance_estimator.load_model(str(distance_model_fd))

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
def test_get_all_publications():
    TestRecordMetadata.create_from_file(__name__, '792017.json')

    expected = {
        'abstract': 'This talk describes past progress in probing the structure of matter and the content of the Universe, which has led to the Standard Model of elementary particles, and the prospects for establishing new physics beyond the Standard Model using the LHC particle collider at CERN.',
        'authors': ['Ellis, John R.'],
        'collaborations': [],
        'keywords': [
            '29.20.Db',
            '98.80.-k',
            '12.60.-i',
            'elementary particles',
            'standard model',
            'cosmology',
            'particle accelerators',
            'Elementary particles',
            'Beyond the Standard Model',
            'Colliders',
            'Large Hadron Collider',
            'Standard Model',
        ],
        'publication_id': 792017,
        'title': 'The quest for elementary particles',
        'topics': ['Phenomenology-HEP'],
    }
    result = list(get_all_publications())

    assert expected in result

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
def test_save_publications(isolated_app, tmpdir):
    TestRecordMetadata.create_from_file(__name__, '792017.json')

    publications_fd = tmpdir.join('publications.jsonl')

    config = {'DISAMBIGUATION_PUBLICATIONS_PATH': str(publications_fd)}

    with patch.dict(current_app.config, config):
        save_publications()

    publications = [json.loads(line) for line in publications_fd.readlines()]

    assert {
        'abstract': 'This talk describes past progress in probing the structure of matter and the content of the Universe, which has led to the Standard Model of elementary particles, and the prospects for establishing new physics beyond the Standard Model using the LHC particle collider at CERN.',
        'authors': ['Ellis, John R.'],
        'collaborations': [],
        'keywords': [
            '29.20.Db',
            '98.80.-k',
            '12.60.-i',
            'elementary particles',
            'standard model',
            'cosmology',
            'particle accelerators',
            'Elementary particles',
            'Beyond the Standard Model',
            'Colliders',
            'Large Hadron Collider',
            'Standard Model',
        ],
        'publication_id': 792017,
        'title': 'The quest for elementary particles',
        'topics': ['Phenomenology-HEP'],
    } in publications

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
def test_get_signatures_matching_a_phonetic_encoding(isolated_app):
    TestRecordMetadata.create_from_file(__name__, '8201.json', index_name='records-hep')
    TestRecordMetadata.create_from_file(__name__, '1518353.json', index_name='records-hep')

    expected = [
        {
            'author_affiliation': 'SUNY, Stony Brook',
            'author_id': None,
            'author_name': 'Rho, Mannque',
            'publication_id': 8201,
            'signature_block': 'Rm',
            'signature_uuid': 'fe4220bf-7c57-4191-8d0c-6e791e84c505',
        },
        {
            'author_affiliation': '',
            'author_id': None,
            'author_name': 'Rao, Mayuri Sathyanarayana',
            'publication_id': 1518353,
            'signature_block': 'Rm',
            'signature_uuid': '383d97f4-5213-4735-9c7d-0d1cf992cce7',
        },
    ]
    result = list(get_signatures_matching_a_phonetic_encoding('Rm'))

    assert sorted(expected) == sorted(result)

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
def test_workflows_halts_on_multiple_exact_matches(workflow_app):
    # Record from arxiv with just arxiv ID in DB
    TestRecordMetadata.create_from_file(
        __name__, "multiple_matches_arxiv.json", index_name="records-hep"
    )

    # Record from publisher with just DOI in DB
    TestRecordMetadata.create_from_file(
        __name__, "multiple_matches_publisher.json", index_name="records-hep"
    )

    path = pkg_resources.resource_filename(
        __name__, "fixtures/multiple_matches_arxiv_update.json"
    )
    update_from_arxiv = json.load(open(path))

    # An update from arxiv with the same arxiv and DOI as above records
    workflow_id = build_workflow(update_from_arxiv).id
    start("article", object_id=workflow_id)

    obj = workflow_object_class.get(workflow_id)

    assert len(set(obj.extra_data["matches"]["exact"])) == 2

    assert obj.status == ObjectStatus.HALTED
    assert obj.extra_data["_action"] == "resolve_multiple_exact_matches"

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
def test_save_sampled_signature_pairs(isolated_app, tmpdir):
    TestRecordMetadata.create_from_file(__name__, '765515.json')
    TestRecordMetadata.create_from_file(__name__, '765975.json')

    curated_signatures_fd = tmpdir.join('curated_signatures.jsonl')
    input_clusters_fd = tmpdir.join('input_clusters.jsonl')
    sampled_pairs_fd = tmpdir.join('sampled_pairs.jsonl')

    config = {
        'DISAMBIGUATION_CURATED_SIGNATURES_PATH': str(curated_signatures_fd),
        'DISAMBIGUATION_INPUT_CLUSTERS_PATH': str(input_clusters_fd),
        'DISAMBIGUATION_SAMPLED_PAIRS_PATH': str(sampled_pairs_fd),
        'DISAMBIGUATION_SAMPLED_PAIRS_SIZE': 12 * 100,
    }

    with patch.dict(current_app.config, config):
        save_curated_signatures_and_input_clusters()
        save_sampled_pairs()

    sampled_pairs = [json.loads(line) for line in sampled_pairs_fd.readlines()]
    normalized_sampled_pairs = [
        {
            'same_cluster': sampled_pair['same_cluster'],
            'signature_uuids': sorted(sampled_pair['signature_uuids']),
        } for sampled_pair in sampled_pairs
    ]  # XXX: so that the assertion doesn't depend on signature order.

    assert {
        'same_cluster': True,
        'signature_uuids': [
            'cbf081db-fcb7-4386-baaf-f30636debfa7',
            'd08e1eb7-fa1b-4ea0-8917-5b3de969c582',
        ],
    } in normalized_sampled_pairs

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
def test_merge_with_conflicts_rootful(
        mocked_api_request_magpie,
        mocked_beard_api,
        workflow_app,
        mocked_external_services,
        disable_file_upload,
        enable_merge_on_update,
):
    with patch('inspire_json_merger.config.ArxivOnArxivOperations.conflict_filters', ['acquisition_source.source']):
        TestRecordMetadata.create_from_file(
            __name__, 'merge_record_arxiv.json', index_name='records-hep')

        update_workflow_id = build_workflow(RECORD_WITH_CONFLICTS).id

        # By default the root is {}.

        eng_uuid = start('article', object_id=update_workflow_id)

        eng = WorkflowEngine.from_uuid(eng_uuid)
        obj = eng.objects[0]

        conflicts = obj.extra_data.get('conflicts')
        assert obj.status == ObjectStatus.HALTED
        assert len(conflicts) == 1

        assert obj.extra_data.get('callback_url') is not None
        assert obj.extra_data.get('is-update') is True
        assert obj.extra_data['merger_root'] == RECORD_WITH_CONFLICTS
        assert obj.extra_data['merger_head_revision'] == 0
        assert obj.extra_data['merger_original_root'] == {}

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
def insert_journals_in_db(workflow_app):
    """Temporarily add few journals in the DB"""
    TestRecordMetadata.create_from_file(
        __name__, 'jou_record_refereed.json', pid_type='jou', index_name='records-journals'
    )
    TestRecordMetadata.create_from_file(
        __name__, 'jou_record_refereed_and_proceedings.json', pid_type='jou', index_name='records-journals'
    )

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
def test_get_all_signatures(isolated_app):
    TestRecordMetadata.create_from_file(__name__, '8201.json')

    expected = {
        'author_affiliation': 'SUNY, Stony Brook',
        'author_id': None,
        'author_name': 'Rho, Mannque',
        'publication_id': 8201,
        'signature_block': 'Rm',
        'signature_uuid': 'fe4220bf-7c57-4191-8d0c-6e791e84c505',
    }
    result = list(get_all_signatures())

    assert expected in result

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
def test_get_all_curated_signatures(isolated_app):
    TestRecordMetadata.create_from_file(__name__, '792017.json')

    expected = {
        'author_affiliation': 'CERN',
        'author_id': 1010819,
        'author_name': 'Ellis, John R.',
        'publication_id': 792017,
        'signature_block': 'ELj',
        'signature_uuid': '94f560d2-6791-43ec-a379-d3dc4ad0ceb7',
    }
    result = list(get_all_curated_signatures())

    assert expected in result

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
def test_merge_without_conflicts_handles_update_without_acquisition_source_and_acts_as_rootless(
        mocked_api_request_magpie,
        mocked_beard_api,
        workflow_app,
        mocked_external_services,
        disable_file_upload,
        enable_merge_on_update,
):
    with patch('inspire_json_merger.config.PublisherOnArxivOperations.conflict_filters', ['acquisition_source.source']):
        factory = TestRecordMetadata.create_from_file(
            __name__, 'merge_record_arxiv.json', index_name='records-hep')

        update_workflow_id = build_workflow(RECORD_WITHOUT_ACQUISITION_SOURCE_AND_NO_CONFLICTS).id

        eng_uuid = start('article', object_id=update_workflow_id)

        eng = WorkflowEngine.from_uuid(eng_uuid)
        obj = eng.objects[0]

        conflicts = obj.extra_data.get('conflicts')

        assert obj.status == ObjectStatus.COMPLETED
        assert not conflicts

        assert obj.extra_data.get('callback_url') is None
        assert obj.extra_data.get('is-update') is True
        assert obj.extra_data['merger_head_revision'] == 0
        assert obj.extra_data['merger_original_root'] == {}

        # source us unknown, so no new root is saved.
        roots = read_all_wf_record_sources(factory.record_metadata.id)
        assert not roots

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
def test_merge_with_disabled_merge_on_update_feature_flag(
        mocked_api_request_magpie,
        mocked_beard_api,
        workflow_app,
        mocked_external_services,
        disable_file_upload,
):

    with patch.dict(workflow_app.config, {'FEATURE_FLAG_ENABLE_MERGER': False}):
        factory = TestRecordMetadata.create_from_file(
            __name__, 'merge_record_arxiv.json', index_name='records-hep')

        update_workflow_id = build_workflow(RECORD_WITHOUT_CONFLICTS).id
        eng_uuid = start('article', object_id=update_workflow_id)

        eng = WorkflowEngine.from_uuid(eng_uuid)
        obj = eng.objects[0]

        assert obj.status == ObjectStatus.COMPLETED

        assert obj.extra_data.get('callback_url') is None
        assert obj.extra_data.get('conflicts') is None
        assert obj.extra_data.get('merged') is True
        assert obj.extra_data.get('merger_root') is None
        assert obj.extra_data.get('is-update') is True

        updated_root = read_wf_record_source(factory.record_metadata.id, 'arxiv')
        assert updated_root is None

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
 def setup(self):
     factory = TestRecordMetadata.create_from_file(__name__, 'test_orcid_domain_models_TestOrcidPusherPostNewWork.json')
     self.orcid = '0000-0002-0942-3697'
     self.recid = factory.record_metadata.json['control_number']
     self.inspire_record = factory.inspire_record
     # Disable logging.
     logging.getLogger('inspirehep.modules.orcid.domain_models').disabled = logging.CRITICAL

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
 def setup(self):
     factory = TestRecordMetadata.create_from_file(__name__, 'test_orcid_tasks_orcid_push_TestOrcidPush.json')
     self.orcid = '0000-0002-0942-3697'
     self.recid = factory.record_metadata.json['control_number']
     self.inspire_record = factory.inspire_record
     self.cache = OrcidCache(self.orcid, self.recid)
     self.oauth_token = get_local_access_tokens(self.orcid) or 'mytoken'

# 需要导入模块: from factories.db.invenio_records import TestRecordMetadata [as 别名]
# 或者: from factories.db.invenio_records.TestRecordMetadata import create_from_file [as 别名]
def test_merge_without_conflicts_rootful(
        mocked_api_request_magpie,
        mocked_beard_api,
        workflow_app,
        mocked_external_services,
        disable_file_upload,
        enable_merge_on_update,
):
    with patch('inspire_json_merger.config.ArxivOnArxivOperations.conflict_filters', ['acquisition_source.source']):
        factory = TestRecordMetadata.create_from_file(
            __name__, 'merge_record_arxiv.json', index_name='records-hep')

        update_workflow_id = build_workflow(RECORD_WITH_CONFLICTS).id

        insert_wf_record_source(json=ARXIV_ROOT, record_uuid=factory.record_metadata.id, source='arxiv')

        eng_uuid = start('article', object_id=update_workflow_id)

        eng = WorkflowEngine.from_uuid(eng_uuid)
        obj = eng.objects[0]

        conflicts = obj.extra_data.get('conflicts')

        assert obj.status == ObjectStatus.COMPLETED
        assert not conflicts

        assert obj.extra_data.get('callback_url') is None
        assert obj.extra_data.get('is-update') is True
        assert obj.extra_data['merger_head_revision'] == 0
        assert obj.extra_data['merger_original_root'] == ARXIV_ROOT

        updated_root = read_wf_record_source(factory.record_metadata.id, 'arxiv')
        assert updated_root.json == RECORD_WITH_CONFLICTS