Python SolrClient.SolrClient类代码示例

 def test_solr_to_solr_reindex_and_resume_reverse(self):
     """
     Only reindexes half of the collection on the first time. Then goes back and does a resume to make sure it works. 
     """
     self._index_docs(50000, self.colls[0])
     solr = SolrClient(test_config["SOLR_SERVER"][0], auth=test_config["SOLR_CREDENTIALS"])
     reindexer = Reindexer(
         source=solr, source_coll="source_coll", dest=solr, dest_coll="dest_coll", date_field="date"
     )
     # Make sure only source has data
     self.assertEqual(len(solr.query(self.colls[0], {"q": "*:*", "rows": 10000000}).docs), 50000)
     self.assertEqual(len(solr.query(self.colls[1], {"q": "*:*", "rows": 10000000}).docs), 0)
     # This gets somehwat of a mid point date in the range.
     midpoint = datetime.datetime.now() - datetime.timedelta(days=((self._end_date - self._start_date).days / 2))
     # Reindex approximately half of the data by restricting FQ
     reindexer.reindex(fq=["date:[{} TO *]".format(midpoint.isoformat() + "Z")])
     sleep(10)
     # Make sure we have at least 20% of the data.
     dest_count = len(solr.query(self.colls[1], {"q": "*:*", "rows": 10000000}).docs)
     s_count = len(solr.query(self.colls[0], {"q": "*:*", "rows": 10000000}).docs)
     self.assertTrue(s_count > dest_count > s_count * 0.20)
     reindexer.resume()
     sleep(10)
     # Make sure countc match up after reindex
     self.assertEqual(
         len(solr.query(self.colls[0], {"q": "*:*", "rows": 10000000}).docs),
         len(solr.query(self.colls[1], {"q": "*:*", "rows": 10000000}).docs),
     )

def index_data():

    docs = get_data()

    client = SolrClient('http://localhost:8983/solr')

    client.index_json('stocks', json.dumps(docs))

    client.commit('stocks')

def index_json():

    client = SolrClient('http://localhost:8983/solr')

    docs = [
        {'id' : '8', 'field8' : 'value8'},
    ]

    client.index_json('test', json.dumps(docs))
    client.commit('test')

def read_all():

    client = SolrClient('http://localhost:8983/solr')

    res = client.query('test', {
        'q' : '*:*'
    })

    res = json.loads(res.get_json())
    docs = res['response']['docs']

    for doc in docs:
        print (doc)

def process_one(record_list, coll):
    url_solr = 'http://xxx.xxx.xxx.xx:xxxx/solr/%s/'% coll
    solrClient = SolrClient(url_solr)

    for record in record_list:
        htmlEntity = HtmlEntity()
        parser = ParserHtml()
        htmlEntity = parser.parseHtml(record, htmlEntity)
        solrClient.addDoc(htmlEntity)
    solrClient.addDocs()
    last_id = record_list[-1]['_id']
    _LOGGER_.info(last_id + ' is Done!')
    del record_list

 def test_index_bad_data(self):
     index = IndexQ(test_config['indexqbase'], 'testq')
     solr = SolrClient(test_config['SOLR_SERVER'], devel=True, auth=test_config['SOLR_CREDENTIALS'])
     if index._is_locked():
         index._unlock()
     self.assertEqual(index.get_all_as_list(),[])
     solr.delete_doc_by_id(test_config['SOLR_COLLECTION'],'*')
     todo_file = index.add({'date':'asd'}, finalize=True)
     self.assertEqual(index.get_all_as_list()[0],todo_file)
     with self.assertRaises(SolrError):
         index.index(solr,test_config['SOLR_COLLECTION'])
     self.assertEqual(index.get_all_as_list()[0],todo_file)
     self.assertFalse(index._is_locked())

 def test_solr_to_solr_with_date(self):
     self._index_docs(50000, self.colls[0])
     solr = SolrClient(test_config["SOLR_SERVER"][0], devel=True, auth=test_config["SOLR_CREDENTIALS"])
     reindexer = Reindexer(
         source=solr, source_coll="source_coll", dest=solr, dest_coll="dest_coll", date_field="index_date"
     )
     reindexer.reindex()
     try:
         self.assertTrue(solr.transport._action_log[1]["params"]["params"]["sort"] == "index_date asc, id desc")
     except KeyError:
         self.assertTrue(solr.transport._action_log[2]["params"]["params"]["sort"] == "index_date asc, id desc")
     self.assertEqual(
         solr.query(self.colls[0], {"q": "*:*", "rows": 10000000}).docs.sort(key=lambda x: x["id"]),
         solr.query(self.colls[1], {"q": "*:*", "rows": 10000000}).docs.sort(key=lambda x: x["id"]),
     )

 def test_solr_to_solr_reindexer_per_shard(self):
     self._index_docs(50000, self.colls[0])
     solr = SolrClient(test_config["SOLR_SERVER"][0], auth=test_config["SOLR_CREDENTIALS"])
     reindexer = Reindexer(
         source=solr, source_coll="source_coll", dest=solr, dest_coll="dest_coll", per_shard=True, date_field="date"
     )
     # Make sure only source has data
     self.assertEqual(len(solr.query(self.colls[0], {"q": "*:*", "rows": 10000000}).docs), 50000)
     self.assertEqual(len(solr.query(self.colls[1], {"q": "*:*", "rows": 10000000}).docs), 0)
     reindexer.reindex()
     # sloppy check over here, will improve later
     self.assertEqual(
         len(solr.query(self.colls[0], {"q": "*:*", "rows": 10000000}).docs),
         len(solr.query(self.colls[1], {"q": "*:*", "rows": 10000000}).docs),
     )

 def test_solr_to_solr_resume_checkonly(self):
     """
     Checks the date_range_query generation function. Since it's pretty simple, running all the tests as one
     """
     self._index_docs(50000, self.colls[0])
     solr = SolrClient(test_config["SOLR_SERVER"][0], devel=True, auth=test_config["SOLR_CREDENTIALS"])
     reindexer = Reindexer(
         source=solr, source_coll="source_coll", dest=solr, dest_coll="dest_coll", date_field="date"
     )
     # Make sure only source has data
     self.assertEqual(len(solr.query(self.colls[0], {"q": "*:*", "rows": 10000000}).docs), 50000)
     self.assertEqual(len(solr.query(self.colls[1], {"q": "*:*", "rows": 10000000}).docs), 0)
     reindexer.resume(check=True)
     # Makes sure nothing got indexed
     self.assertEqual(len(solr.query(self.colls[0], {"q": "*:*", "rows": 10000000}).docs), 50000)
     self.assertEqual(len(solr.query(self.colls[1], {"q": "*:*", "rows": 10000000}).docs), 0)

 def test_complete_compress_basic_re_indexing(self):
     log = logging.getLogger()
     solr = SolrClient(test_config['SOLR_SERVER'],
                       devel=True,
                       auth=test_config['SOLR_CREDENTIALS'])
     index = IndexQ(test_config['indexqbase'], 'testq', size = 1, log = log,
                    compress=True)
     solr.delete_doc_by_id(test_config['SOLR_COLLECTION'], '*')
     for item in self.docs[1:10]:
         index.add(item, finalize=True)
     index.index(solr, test_config['SOLR_COLLECTION'])
     # At this point items are indexed and are moved into the done directory
     # Lets re-index them to make sure all json got properly encoded
     files = index.get_all_as_list('_done_dir')
     for f in index.get_all_as_list('_done_dir'):
         shutil.move(f, index._todo_dir)
     index.index(solr, test_config['SOLR_COLLECTION'])
     self.assertEqual(files, index.get_all_as_list('_done_dir'))

 def test_solr_to_solr_resume_basic(self):
     """
     Checks the date_range_query generation function. Since it's pretty simple, running all the tests as one
     """
     self._index_docs(50000, self.colls[0])
     solr = SolrClient(test_config["SOLR_SERVER"][0], auth=test_config["SOLR_CREDENTIALS"])
     reindexer = Reindexer(
         source=solr, source_coll="source_coll", dest=solr, dest_coll="dest_coll", date_field="date"
     )
     # Make sure only source has datae
     self.assertEqual(len(solr.query(self.colls[0], {"q": "*:*", "rows": 10000000}).docs), 50000)
     self.assertEqual(len(solr.query(self.colls[1], {"q": "*:*", "rows": 10000000}).docs), 0)
     reindexer.resume()
     sleep(10)
     # Make sure countc match up after reindex
     self.assertEqual(
         len(solr.query(self.colls[0], {"q": "*:*", "rows": 10000000}).docs),
         len(solr.query(self.colls[1], {"q": "*:*", "rows": 10000000}).docs),
     )

def computeScores1(type, query, output_file):
    solr = SolrClient('http://localhost:8983/solr')

    res = solr.query(query['index'], {
        'q': '*:*',
        'wt': 'json',
        'indent': True,
        'rows': 1000,
    })

    docs = res.data['response']['docs']

    with open(output_file, "wb") as outF:
        a = csv.writer(outF, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
        a.writerow(["type", "x-coordinate", "y-coordinate", "Similarity_score"])

        for doc in docs:
            for key in doc:
                if key in ["id", "_version_"]:
                    continue
                try:
                    doc[key] = doc[key][0].encode("ascii", "ignore")
                except:
                    doc[key] = str(doc[key][0]).decode("unicode_escape").encode("ascii", "ignore")

        doc_tuples = itertools.combinations(docs, 2)
        for raw1, raw2 in doc_tuples:

            doc1 = raw1.copy()
            doc2 = raw2.copy()

            if "Name" in doc1:
                row_cosine_distance = [type, doc1["Name"], doc2["Name"]]
            else:
                row_cosine_distance = [type, doc1["name"], doc2["name"]]

            v1 = Vector(row_cosine_distance[0], doc1)
            v2 = Vector(row_cosine_distance[1], doc2)

            row_cosine_distance.append(v1.cosTheta(v2))

            a.writerow(row_cosine_distance)

 def test_index(self):
     index = IndexQ(test_config['indexqbase'], 'testq')
     solr = SolrClient(test_config['SOLR_SERVER'], devel=True, auth=test_config['SOLR_CREDENTIALS'])
     solr.delete_doc_by_id(test_config['SOLR_COLLECTION'],'*')
     buff = []
     files = []
     for doc in self.docs:
         files.append(index.add(doc, finalize=True))
     index.index(solr,test_config['SOLR_COLLECTION'])
     solr.commit(test_config['SOLR_COLLECTION'],openSearcher=True)
     for doc in self.docs:
         res = solr.query(test_config['SOLR_COLLECTION'],{'q':'id:{}'.format(doc['id'])})
         self.assertTrue(res.get_results_count()==1)
     solr.delete_doc_by_id(test_config['SOLR_COLLECTION'],'*')

    def get(self):

        term = self.get_argument('term')

        client = SolrClient('http://localhost:8983/solr')
        res = client.query('stocks', {
            #'q' : 'symbol:%s' % '*'
            'q' : term
        })

        res = json.loads(res.get_json())
        docs = res['response']['docs']

        formatted = []

        for doc in docs:
            formatted.append({
                'name' : doc['name'],
                'symbol' : doc['symbol'],
                'sector' : doc['sector'],
                'open' : doc['open']
            })

        self.write(json.dumps(formatted))

 def setUpClass(self):
     self.solr = SolrClient(test_config['SOLR_SERVER'][0], devel=True, auth=test_config['SOLR_CREDENTIALS'])
     self.rand_docs = RandomTestData()
     self.docs = self.rand_docs.get_docs(50)
     
     for field in test_config['collections']['copy_fields']:
         try:
             self.solr.schema.delete_copy_field(test_config['SOLR_COLLECTION'],field)
         except:
             pass
     for field in test_config['collections']['fields']:
         try:
             self.solr.schema.create_field(test_config['SOLR_COLLECTION'],field)
         except:
             pass