Python Search.index_refresh方法代码示例

# 需要导入模块: from elastic.search import Search [as 别名]
# 或者: from elastic.search.Search import index_refresh [as 别名]
def setUpModule():
    ''' Change ini config (MY_INI_FILE) to use the test suffix when
    creating pipeline indices. '''
    ini_file = os.path.join(os.path.dirname(__file__), 'test_download.ini')
    if os.path.isfile(MY_INI_FILE):
        return

    with open(MY_INI_FILE, 'w') as new_file:
        with open(ini_file) as old_file:
            for line in old_file:
                new_file.write(line.replace('auto_tests', IDX_SUFFIX))

    '''load ensembl GTF  and GENE_HISTORY'''
    INI_CONFIG = IniParser().read_ini(MY_INI_FILE)
    idx = INI_CONFIG['ENSEMBL_GENE_GTF']['index']

    call_command('pipeline', '--steps', 'load', sections='GENE_HISTORY',
                 dir=TEST_DATA_DIR, ini=MY_INI_FILE)
    call_command('pipeline', '--steps', 'stage', 'load', sections='ENSEMBL_GENE_GTF',
                 dir=TEST_DATA_DIR, ini=MY_INI_FILE)
    Search.index_refresh(idx)
    call_command('pipeline', '--steps', 'load', sections='GENE2ENSEMBL',
                 dir=TEST_DATA_DIR, ini=MY_INI_FILE)

    Search.index_refresh(idx)

# 需要导入模块: from elastic.search import Search [as 别名]
# 或者: from elastic.search.Search import index_refresh [as 别名]
    def test_region_idx_loader(self):
        ''' Test loader has created and populated indices.  '''

        key = 'PRIVATE_REGIONS_GFF'

        if key in IDX.keys():
            idx = IDX[key]['indexName']
            Search.index_refresh(idx)
            self.assertTrue(Search.index_exists(idx=idx), 'Index exists: '+idx)
            ndocs = Search(idx=idx).get_count()['count']
            self.assertTrue(ndocs > 0, "Elastic count documents in " + idx + ": " + str(ndocs))

# 需要导入模块: from elastic.search import Search [as 别名]
# 或者: from elastic.search.Search import index_refresh [as 别名]
 def test_pub_ini_file2(self):
     ''' Test publication pipeline with a list of PMIDs. '''
     out = StringIO()
     call_command('publications', '--dir', TEST_DATA_DIR, '--steps', 'load',
                  sections='DISEASE::TEST', ini=MY_PUB_INI_FILE, stdout=out)
     INI_CONFIG = IniParser().read_ini(MY_PUB_INI_FILE)
     idx = INI_CONFIG['DISEASE']['index']
     Search.index_refresh(idx)
     query = ElasticQuery.query_string("test", fields=["tags.disease"])
     elastic = Search(query, idx=idx)
     docs = elastic.search().docs
     self.assertGreater(len(docs), 1)

# 需要导入模块: from elastic.search import Search [as 别名]
# 或者: from elastic.search.Search import index_refresh [as 别名]
    def test_gene_history_loader(self):
        """ Test the gene history loading. """
        call_command("pipeline", "--steps", "load", sections="GENE_HISTORY", dir=TEST_DATA_DIR, ini=MY_INI_FILE)

        INI_CONFIG = IniParser().read_ini(MY_INI_FILE)
        idx = INI_CONFIG["GENE_HISTORY"]["index"]
        idx_type = INI_CONFIG["GENE_HISTORY"]["index_type"]
        elastic = Search(idx=idx, idx_type=idx_type)
        Search.index_refresh(idx)

        self.assertTrue(elastic.get_count()["count"] > 1, "Count documents in the index")
        map1_props = Gene.gene_history_mapping(idx, idx_type, test_mode=True).mapping_properties
        map2_props = elastic.get_mapping()
        if idx not in map2_props:
            logger.error("MAPPING ERROR: " + json.dumps(map2_props))
        self._cmpMappings(map2_props[idx]["mappings"], map1_props, idx_type)

# 需要导入模块: from elastic.search import Search [as 别名]
# 或者: from elastic.search.Search import index_refresh [as 别名]
    def test_marker_pipeline(self):
        """ Test marker pipeline. """
        call_command("pipeline", "--steps", "load", sections="DBSNP", dir=TEST_DATA_DIR, ini=MY_INI_FILE)

        INI_CONFIG = IniParser().read_ini(MY_INI_FILE)
        idx = INI_CONFIG["DBSNP"]["index"]
        idx_type = INI_CONFIG["DBSNP"]["index_type"]
        elastic = Search(idx=idx, idx_type=idx_type)
        Search.index_refresh(idx)
        self.assertGreater(elastic.get_count()["count"], 0)

        call_command("pipeline", "--steps", "load", sections="RSMERGEARCH", dir=TEST_DATA_DIR, ini=MY_INI_FILE)
        idx = INI_CONFIG["RSMERGEARCH"]["index"]
        idx_type = INI_CONFIG["RSMERGEARCH"]["index_type"]
        elastic = Search(idx=idx, idx_type=idx_type)
        Search.index_refresh(idx)
        self.assertGreater(elastic.get_count()["count"], 0)

# 需要导入模块: from elastic.search import Search [as 别名]
# 或者: from elastic.search.Search import index_refresh [as 别名]
 def setupIdx(cls, idx_name_arr):
     ''' Setup indices in the given array of key names (e.g. ['GENE', 'DIISEASE', ...]). '''
     idx_settings = {
         "settings": {
             "analysis": {
                 "analyzer": {
                     "full_name": {"filter": ["standard", "lowercase"], "tokenizer": "keyword"}}
             },
             "number_of_shards": 1
         }
     }
     IDX = PydginTestSettings.IDX
     for name in idx_name_arr:
         requests.put(ElasticSettings.url() + '/' + IDX[name]['indexName'], data=json.dumps(idx_settings))
         call_command('index_search', **IDX[name])
     for name in idx_name_arr:
         # wait for the elastic load to finish
         Search.index_refresh(IDX[name]['indexName'])

# 需要导入模块: from elastic.search import Search [as 别名]
# 或者: from elastic.search.Search import index_refresh [as 别名]
    def test_doc_auth(self):
        idx = PydginTestSettings.IDX["STUDY_HITS"]["indexName"]
        docs = Search(ElasticQuery(Query.match_all(), sources=["chr_band", "marker"]), idx=idx, size=1).search().docs
        self.assertEquals(len(docs), 1, "STUDY_HITS document")
        marker_id = getattr(docs[0], "marker")

        url = reverse("search_page")
        resp = self.client.post(url + "?idx=ALL&query=" + marker_id)
        nhits1 = resp.context["hits_total"]
        self.assertGreater(nhits1, 0, "search hits > 0")
        # update document to be in DIL
        update_field = {"doc": {"group_name": "DIL"}}
        Update.update_doc(docs[0], update_field)
        Search.index_refresh(PydginTestSettings.IDX["STUDY_HITS"]["indexName"])

        url = reverse("search_page")
        resp = self.client.post(url + "?idx=ALL&query=" + marker_id)
        nhits2 = resp.context["hits_total"]
        self.assertEqual(nhits1 - 1, nhits2, "private document hidden")

# 需要导入模块: from elastic.search import Search [as 别名]
# 或者: from elastic.search.Search import index_refresh [as 别名]
    def test_doc_auth(self):
        ''' Test private documents are not returned in the search. '''
        idx = PydginTestSettings.IDX['MARKER']['indexName']
        docs = Search(ElasticQuery(Query.match_all(), sources=['id']), idx=idx, size=1).search().docs
        self.assertEquals(len(docs), 1, "MARKER document")
        marker_id = getattr(docs[0], 'id')

        url = reverse('search_page')
        resp = self.client.post(url+'?idx=ALL&query='+marker_id)
        nhits1 = resp.context['hits_total']
        self.assertGreater(nhits1, 0, 'search hits > 0')
        # update document to be in DIL
        update_field = {"doc": {"group_name": "DIL"}}
        Update.update_doc(docs[0], update_field)
        Search.index_refresh(PydginTestSettings.IDX['MARKER']['indexName'])

        resp = self.client.post(url+'?idx=ALL&query='+marker_id)
        nhits2 = resp.context['hits_total']
        self.assertEqual(nhits1-1, nhits2, 'private document hidden')

# 需要导入模块: from elastic.search import Search [as 别名]
# 或者: from elastic.search.Search import index_refresh [as 别名]
    def test_elastic_group_name(self):
        '''
        Testing the workflow defined in: https://killin.cimr.cam.ac.uk/nextgensite/2015/08/05/region-authorization/
        Testing various elastic queries

        idx doc:
         "_source":{"attr": {"region_id": "803", "group_name": "[\"DIL\"]", "Name": "4q27"},
         "seqid": "chr4", "source": "immunobase", "type": "region",
         "score": ".", "strand": ".", "phase": ".", "start": 122061159, "end": 122684373}
        idx_query:
        Private(in given group) OR Public
        -d '{"query":{"filtered":{"filter":{"bool": {
                                            "should": [
                                                        {"terms": {"group_name":["dil"]}},
                                                        { "missing": { "field": "group_name"   }}
                                                      ]
                                                    }}}}}'
        Private(in given group):
        -d '{"query":{"filtered":{"filter":{"terms":{"group_name":["dil"]}}}}}'
        Public:
        -d {'query': {'filtered': {'filter': {'missing': {'field': 'group_name'}},
-                         'query': {'term': {'match_all': '{}'}}}}}
        '''
        # get the groups for the given user
        response = self.client.post('/accounts/login/', {'username': 'test_user', 'password': 'test_pass'})
        self.assertTrue(response.status_code, "200")

        logged_in_user = User.objects.get(id=self.client.session['_auth_user_id'])
        if logged_in_user and logged_in_user.is_authenticated():
            user_groups = get_user_groups(logged_in_user)
            self.assertTrue('READ' in user_groups, "user present in READ group")
            # make sure the user is not yet in DIL group
            self.assertFalse('DIL' in user_groups, "user not present in DIL group")

        group_names = get_user_groups(logged_in_user)
        if 'READ' in group_names : group_names.remove('READ')  # @IgnorePep8
        group_names = [x.lower() for x in group_names]
        self.assertTrue(len(group_names) == 0, "No group present")

        # Match all query, as there is no group we do a match all
        query = ElasticQuery(Query.match_all())
        expected_query_string = {"query": {"match_all": {}}}
        self.assertJSONEqual(json.dumps(query.query), json.dumps(expected_query_string), "Query string matched")

        Search.index_refresh(self.index_name)
        elastic = Search(query, idx=self.index_name)
        docs = elastic.search().docs
        self.assertTrue(len(docs) == 12, "Elastic string query retrieved all public regions")

        # Filtered query for group names, add the user to DIL group and get the query string
        self.dil_group = Group.objects.create(name='DIL')
        logged_in_user.groups.add(self.dil_group)
        group_names = get_user_groups(logged_in_user)
        if 'READ' in group_names : group_names.remove('READ')  # @IgnorePep8
        group_names = [x.lower() for x in group_names]
        self.assertTrue(len(group_names) > 0, "More than 1 group present")
        self.assertTrue("dil" in group_names, "DIL group present")

        # retrieves all docs with missing field group_name - 11 docs
        terms_filter = TermsFilter.get_missing_terms_filter("field", "attr.group_name")
        query = ElasticQuery.filtered(Query.match_all(), terms_filter)
        elastic = Search(query, idx=self.index_name)
        docs = elastic.search().docs
        self.assertTrue(len(docs) == 11, "Elastic string query retrieved all public regions")

        # build filtered boolean query to bring all public docs + private docs 11+1 = 12 docs
        query_bool = BoolQuery()
        query_bool.should(Query.missing_terms("field", "group_name")) \
                  .should(Query.terms("group_name", group_names).query_wrap())

        query = ElasticQuery.filtered_bool(Query.match_all(), query_bool)
        elastic = Search(query, idx=self.index_name)
        docs = elastic.search().docs
        self.assertTrue(len(docs) == 12, "Elastic string query retrieved both public + private regions")

        terms_filter = TermsFilter.get_terms_filter("attr.group_name", group_names)
        query = ElasticQuery.filtered(Query.match_all(), terms_filter)
        elastic = Search(query, idx=self.index_name)
        docs = elastic.search().docs
        self.assertTrue(len(docs) == 1, "Elastic string query retrieved one private regions")
        self.assertEqual(docs[0].attr['Name'], "4q27", "type matched region")
        self.assertEqual(docs[0].attr['region_id'], "803", "type matched region")
        self.assertEqual(docs[0].attr['group_name'], "[\"DIL\"]", "type matched region")

# 需要导入模块: from elastic.search import Search [as 别名]
# 或者: from elastic.search.Search import index_refresh [as 别名]
        "tier": tier,
        "species": species,
        "tags": {"weight": weight},
        "locus_id": locus_id,
        "seqid": seqid,
        "hits": doc_ids
    }
    #    "suggest": {"input": [disease+" "+regionName, regionName], "weight": weight}
    resp = requests.put(ElasticSettings.url()+'/' + idx+'/disease_locus/'+locus_id, data=json.dumps(data))
    if resp.status_code != 201:
        print(str(resp.content))
        print("Problem loading "+getattr(doc, "disease")+" "+regionName)
    else:
        print("Loaded "+locus_id+" - "+regionName)

Search.index_refresh(idx)
diseases_by_seqid = Agg('diseases_by_seqid', 'terms', {"size": 0, "field": "disease"})
disease_hits = Agg('disease_hits', 'reverse_nested', {}, sub_agg=diseases_by_seqid)
seq_hits = Agg('seq_hits', 'terms', {'field': 'build_info.seqid', 'size': 0}, sub_agg=disease_hits)
build_info = Agg('build_info', 'nested', {"path": 'build_info'}, sub_agg=[seq_hits])

qnested = ElasticQuery(Query.nested('build_info', Query.term("build_info.build", build)))
elastic = Search(qnested, idx=idx, aggs=Aggs(build_info), search_type='count')
resultObj = elastic.search()
resultAggs = resultObj.aggs
seq_hits = getattr(resultAggs['build_info'], 'seq_hits')['buckets']

for chr_bucket in seq_hits:
    seqid = chr_bucket['key'].upper()

    for disease_bucket in chr_bucket['disease_hits']['diseases_by_seqid']['buckets']:

# 需要导入模块: from elastic.search import Search [as 别名]
# 或者: from elastic.search.Search import index_refresh [as 别名]
    def test_gene_pipeline(self):
        """ Test gene pipeline. """

        INI_CONFIG = IniParser().read_ini(MY_INI_FILE)
        idx = INI_CONFIG["ENSEMBL_GENE_GTF"]["index"]
        idx_type = INI_CONFIG["ENSEMBL_GENE_GTF"]["index_type"]

        """ 1. Test ensembl GTF loading. """
        call_command(
            "pipeline", "--steps", "stage", "load", sections="ENSEMBL_GENE_GTF", dir=TEST_DATA_DIR, ini=MY_INI_FILE
        )
        Search.index_refresh(idx)

        elastic = Search(idx=idx, idx_type=idx_type)
        self.assertGreaterEqual(elastic.get_count()["count"], 1, "Count documents in the index")
        map1_props = Gene.gene_mapping(idx, idx_type, test_mode=True).mapping_properties
        map2_props = elastic.get_mapping()
        if idx not in map2_props:
            logger.error("MAPPING ERROR: " + json.dumps(map2_props))
        self._cmpMappings(map2_props[idx]["mappings"], map1_props, idx_type)

        """ 2. Test adding entrez ID to documents """
        call_command("pipeline", "--steps", "load", sections="GENE2ENSEMBL", dir=TEST_DATA_DIR, ini=MY_INI_FILE)
        Search.index_refresh(idx)
        query = ElasticQuery.query_string("PTPN22", fields=["symbol"])
        elastic = Search(query, idx=idx)
        docs = elastic.search().docs
        self.assertEqual(len(docs), 1)
        self.assertTrue("entrez" in getattr(docs[0], "dbxrefs"))
        self.assertEqual(getattr(docs[0], "dbxrefs")["entrez"], "26191")

        """ 3. Add uniprot and fill in missing entrez fields. """
        call_command(
            "pipeline", "--steps", "download", "load", sections="ENSMART_GENE", dir=TEST_DATA_DIR, ini=MY_INI_FILE
        )
        Search.index_refresh(idx)
        query = ElasticQuery.query_string("DNMT3L", fields=["symbol"])
        elastic = Search(query, idx=idx)
        docs = elastic.search().docs
        self.assertTrue("entrez" in getattr(docs[0], "dbxrefs"))
        self.assertTrue("swissprot" in getattr(docs[0], "dbxrefs"))

        """ 4. Add gene synonyms and dbxrefs. """
        call_command("pipeline", "--steps", "load", sections="GENE_INFO", dir=TEST_DATA_DIR, ini=MY_INI_FILE)
        Search.index_refresh(idx)
        query = ElasticQuery.query_string("PTPN22", fields=["symbol"])
        elastic = Search(query, idx=idx)
        docs = elastic.search().docs
        self.assertTrue("PTPN8" in getattr(docs[0], "synonyms"))

        """ 5. Add PMIDs to gene docs. """
        call_command("pipeline", "--steps", "load", sections="GENE_PUBS", dir=TEST_DATA_DIR, ini=MY_INI_FILE)
        Search.index_refresh(idx)
        query = ElasticQuery.query_string("PTPN22", fields=["symbol"])
        elastic = Search(query, idx=idx)
        docs = elastic.search().docs
        self.assertGreater(len(getattr(docs[0], "pmids")), 0)

        """ 6. Add ortholog data. """
        call_command("pipeline", "--steps", "load", sections="ENSMART_HOMOLOG", dir=TEST_DATA_DIR, ini=MY_INI_FILE)
        Search.index_refresh(idx)
        query = ElasticQuery.query_string("PTPN22", fields=["symbol"])
        elastic = Search(query, idx=idx)
        docs = elastic.search().docs
        dbxrefs = getattr(docs[0], "dbxrefs")
        self.assertTrue("orthologs" in dbxrefs, dbxrefs)
        self.assertTrue("mmusculus" in dbxrefs["orthologs"], dbxrefs)
        self.assertEqual("ENSMUSG00000027843", dbxrefs["orthologs"]["mmusculus"]["ensembl"])

        query = ElasticQuery.filtered(
            Query.match_all(),
            TermsFilter.get_terms_filter("dbxrefs.orthologs.mmusculus.ensembl", ["ENSMUSG00000027843"]),
        )
        docs = Search(query, idx=idx, size=1).search().docs
        self.assertEqual(len(docs), 1)

        """ 7. Add mouse ortholog link to MGI """
        call_command("pipeline", "--steps", "load", sections="ENSEMBL2MGI", dir=TEST_DATA_DIR, ini=MY_INI_FILE)
        Search.index_refresh(idx)
        docs = Search(query, idx=idx, size=1).search().docs
        dbxrefs = getattr(docs[0], "dbxrefs")
        self.assertEqual("ENSMUSG00000027843", dbxrefs["orthologs"]["mmusculus"]["ensembl"])
        self.assertEqual("107170", dbxrefs["orthologs"]["mmusculus"]["MGI"])

# 需要导入模块: from elastic.search import Search [as 别名]
# 或者: from elastic.search.Search import index_refresh [as 别名]
def setUpModule():
    ''' Load test indices (marker) '''
    call_command('index_search', **IDX['MARKER'])
    call_command('index_search', **IDX['MARKER_RS_HISTORY'])
    call_command('index_search', **IDX['JSON_MARKER_IC'])
    Search.index_refresh(IDX['MARKER']['indexName'])