Python IndexQ.add方法代码示例

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
 def test_add_callback_with_size(self):
     docs = self.rand_docs.get_docs(5)
     index = IndexQ(test_config['indexqbase'], 'testq', size=1)
     temp = []
     def cb(path):
         temp.append(path)
     t = index.add(docs[0], callback=cb)
     t = index.add(docs[1], callback=cb, finalize=True)
     self.assertTrue(t in temp)

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
 def test_complete_compress_basic(self):
     log = logging.getLogger()
     index = IndexQ(test_config['indexqbase'], 'testq', size = 1, log = log,
                    compress=True)
     for item in self.docs[1:10]:
         index.add(item, finalize=True)
     files = []
     for item in index.get_all_as_list():
         files.append(index.complete(item))
     [self.assertTrue(os.path.exists(x)) for x in files]

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
 def test_complete_dir_rotate(self):
     log = logging.getLogger()
     rotate_func = lambda: '{}/{}/{}'.format(dt.now().year, dt.now().month, dt.now().day)
     index = IndexQ(test_config['indexqbase'], 'testq', size = 1, log = log,
                    rotate_complete = rotate_func)
     dir_set = rotate_func()
     docs = self.rand_docs.get_docs(69)
     for item in self.docs[1:10]:
         index.add(item, finalize=True)
     files = []
     for item in index.get_all_as_list():
         files.append(index.complete(item))
     [self.assertTrue(os.path.exists(x)) for x in files]

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
 def test_buffer_list_75m_dump_early(self):
     size = 75
     index = IndexQ(test_config['indexqbase'], 'testq', size=size)
     buff = []
     while True:
         doc = index.add(self.docs)
         [buff.append(x) for x in self.docs]
         if doc > 40000000:
             doc = index.add(finalize=True)
         if type(doc) is str:
             break
     self.check_file_contents(doc,buff)
     os.remove(doc)

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
 def test_thread_pool_mid(self):
     '''
     Index data using multiple threads.
     Verity that each thread
     '''
     docs = self.rand_docs.get_docs(5000)
     threads = 5
     index = IndexQ(test_config['indexqbase'],'testq', size = 1)
     with ThreadPool(threads) as p:
         p.map(index.add, docs)
     index.add(finalize=True)
     d = index.get_all_json_from_indexq()
     self.assertEqual(sorted(d, key=lambda x: x['id']), sorted(docs, key=lambda x: x['id']))

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
 def test_add_string(self):
     index = IndexQ(test_config['indexqbase'], 'testq')
     string_test = 'asd'
     doc = index.add(string_test)
     with open(doc) as f:
         doc_data = f.read()
     self.assertEqual(string_test, doc_data)

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
 def test_complete_compress_basic_re_indexing(self):
     log = logging.getLogger()
     solr = SolrClient(test_config['SOLR_SERVER'],
                       devel=True,
                       auth=test_config['SOLR_CREDENTIALS'])
     index = IndexQ(test_config['indexqbase'], 'testq', size = 1, log = log,
                    compress=True)
     solr.delete_doc_by_id(test_config['SOLR_COLLECTION'], '*')
     for item in self.docs[1:10]:
         index.add(item, finalize=True)
     index.index(solr, test_config['SOLR_COLLECTION'])
     # At this point items are indexed and are moved into the done directory
     # Lets re-index them to make sure all json got properly encoded
     files = index.get_all_as_list('_done_dir')
     for f in index.get_all_as_list('_done_dir'):
         shutil.move(f, index._todo_dir)
     index.index(solr, test_config['SOLR_COLLECTION'])
     self.assertEqual(files, index.get_all_as_list('_done_dir'))

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
 def test_thread_pool_low(self):
     '''
     Index data using multiple threads.
     Verity that each thread
     '''
     docs = self.rand_docs.get_docs(5)
     threads = 5
     index = IndexQ(test_config['indexqbase'],'testq', size = 1)
     with ThreadPool(threads) as p:
         p.map(index.add, docs)
     self.check_file_contents(index.add(finalize=True), docs)

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
    def test_thread_pool_high(self):
        '''
        Index data using multiple threads.
        Verity that each thread
        '''
        docs = self.rand_docs.get_docs(25000)
        index = IndexQ(test_config['indexqbase'],
                       'testq',
                       size=.1,
                       devel=True)
        for dir in ['_todo_dir', '_done_dir']:
            [os.remove(x) for x in index.get_all_as_list(dir=dir)]
        threads = 25

        with ThreadPool(threads) as p:
            p.map(index.add, docs)
        index.add(finalize=True)
        d = index.get_all_json_from_indexq()
        self.assertEqual(len(d), len(docs))
        self.assertEqual(sorted(d, key=lambda x: x['id']),
                         sorted(docs, key=lambda x: x['id']))

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
 def test_buffer_list_75m(self):
     size = 75
     index = IndexQ(test_config['indexqbase'], 'testq', size=size)
     buff = []
     while True:
         doc = index.add(self.docs)
         [buff.append(x) for x in self.docs]
         if type(doc) is str:
             break
     self.check_file_contents(doc,buff)
     self.assertLessEqual(os.path.getsize(doc),size*1000000)
     self.assertGreaterEqual(os.path.getsize(doc), size*1000000*.90)
     os.remove(doc)

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
 def test_index_bad_data(self):
     index = IndexQ(test_config['indexqbase'], 'testq')
     solr = SolrClient(test_config['SOLR_SERVER'], devel=True, auth=test_config['SOLR_CREDENTIALS'])
     if index._is_locked():
         index._unlock()
     self.assertEqual(index.get_all_as_list(),[])
     solr.delete_doc_by_id(test_config['SOLR_COLLECTION'],'*')
     todo_file = index.add({'date':'asd'}, finalize=True)
     self.assertEqual(index.get_all_as_list()[0],todo_file)
     with self.assertRaises(SolrError):
         index.index(solr,test_config['SOLR_COLLECTION'])
     self.assertEqual(index.get_all_as_list()[0],todo_file)
     self.assertFalse(index._is_locked())

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
 def test_index_multiproc(self):
     index = IndexQ(test_config['indexqbase'], 'testq')
     solr = SolrClient(test_config['SOLR_SERVER'], devel=True, auth=test_config['SOLR_CREDENTIALS'])
     solr.delete_doc_by_id(test_config['SOLR_COLLECTION'],'*')
     buff = []
     files = []
     for doc in self.docs:
         files.append(index.add(doc, finalize=True))
     index.index(solr,test_config['SOLR_COLLECTION'],threads=10)
     solr.commit(test_config['SOLR_COLLECTION'],openSearcher=True)
     for doc in self.docs:
         res = solr.query(test_config['SOLR_COLLECTION'],{'q':'id:{}'.format(doc['id'])})
         self.assertTrue(res.get_results_count()==1)

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
 def test_by_get_all_default_compression(self):
     size = 1
     files = 2
     index = IndexQ(test_config['indexqbase'], 'testq', size=size)
     buff = []
     docs = []
     for _ in range(files):
         doc = index.add(self.docs,finalize=True)
         docs.append(doc)
         sleep(1)
     index = IndexQ(test_config['indexqbase'], 'testq', mode='out')
     indexdocs = index.get_all_as_list()
     self.assertEqual(docs,indexdocs)
     [os.remove(doc) for doc in docs]

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
 def test_buffer_dict_75m(self):
     size = 75
     index = IndexQ(test_config['indexqbase'], 'testq', size=size)
     buff = []
     while True:
         item = random.choice(self.docs)
         doc = index.add(item)
         buff.append(item)
         if type(doc) is str:
             break
     self.check_file_contents(doc,buff)
     self.assertLessEqual(os.path.getsize(doc), size * 1000000)
     self.assertGreaterEqual(os.path.getsize(doc), size * 1000000 * .90)
     os.remove(doc)

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import add [as 别名]
 def test_dequeue(self):
     size = 1
     files = 2
     index = IndexQ(test_config['indexqbase'], 'testq', size=size)
     buff = []
     docs = []
     for _ in range(files):
         doc = index.add(self.docs,finalize=True)
         docs.append(doc)
     index = IndexQ(test_config['indexqbase'], 'testq')
     indexdocs = []
     for x in index.get_todo_items():
         indexdocs.append(x)
     self.assertEqual(docs,indexdocs)
     [os.remove(doc) for doc in docs]