Python IndexQ.get_all_as_list方法代码示例

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import get_all_as_list [as 别名]
 def test_index_bad_data(self):
     index = IndexQ(test_config['indexqbase'], 'testq')
     solr = SolrClient(test_config['SOLR_SERVER'], devel=True, auth=test_config['SOLR_CREDENTIALS'])
     if index._is_locked():
         index._unlock()
     self.assertEqual(index.get_all_as_list(),[])
     solr.delete_doc_by_id(test_config['SOLR_COLLECTION'],'*')
     todo_file = index.add({'date':'asd'}, finalize=True)
     self.assertEqual(index.get_all_as_list()[0],todo_file)
     with self.assertRaises(SolrError):
         index.index(solr,test_config['SOLR_COLLECTION'])
     self.assertEqual(index.get_all_as_list()[0],todo_file)
     self.assertFalse(index._is_locked())

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import get_all_as_list [as 别名]
 def test_complete_compress_basic_re_indexing(self):
     log = logging.getLogger()
     solr = SolrClient(test_config['SOLR_SERVER'],
                       devel=True,
                       auth=test_config['SOLR_CREDENTIALS'])
     index = IndexQ(test_config['indexqbase'], 'testq', size = 1, log = log,
                    compress=True)
     solr.delete_doc_by_id(test_config['SOLR_COLLECTION'], '*')
     for item in self.docs[1:10]:
         index.add(item, finalize=True)
     index.index(solr, test_config['SOLR_COLLECTION'])
     # At this point items are indexed and are moved into the done directory
     # Lets re-index them to make sure all json got properly encoded
     files = index.get_all_as_list('_done_dir')
     for f in index.get_all_as_list('_done_dir'):
         shutil.move(f, index._todo_dir)
     index.index(solr, test_config['SOLR_COLLECTION'])
     self.assertEqual(files, index.get_all_as_list('_done_dir'))

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import get_all_as_list [as 别名]
 def test_complete_compress_basic(self):
     log = logging.getLogger()
     index = IndexQ(test_config['indexqbase'], 'testq', size = 1, log = log,
                    compress=True)
     for item in self.docs[1:10]:
         index.add(item, finalize=True)
     files = []
     for item in index.get_all_as_list():
         files.append(index.complete(item))
     [self.assertTrue(os.path.exists(x)) for x in files]

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import get_all_as_list [as 别名]
 def test_ignore_fields(self):
     """
     Will export documents from Solr and put them into an IndexQ. 
     """
     index = IndexQ(test_config["indexqbase"], "test_reindexer", size=0)
     for dir in ["_todo_dir", "_done_dir"]:
         [os.remove(x) for x in index.get_all_as_list(dir=dir)]
     reindexer = Reindexer(source=self.solr, source_coll="source_coll", dest=index)
     for field in ["_version_", "product_name_exact"]:
         self.assertTrue(field in reindexer._ignore_fields)

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import get_all_as_list [as 别名]
 def test_remove_copy_fields_from_data(self):
     index = IndexQ(test_config["indexqbase"], "test_reindexer", size=0)
     for dir in ["_todo_dir", "_done_dir"]:
         [os.remove(x) for x in index.get_all_as_list(dir=dir)]
     reindexer = Reindexer(source=self.solr, source_coll="source_coll", dest=index)
     reindexer.reindex()
     from_files = self.get_all_json_from_indexq(index)
     excluded_fields = reindexer._ignore_fields
     for doc in from_files:
         for field in excluded_fields:
             if field in doc:
                 print(doc)

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import get_all_as_list [as 别名]
 def test_complete_dir_rotate(self):
     log = logging.getLogger()
     rotate_func = lambda: '{}/{}/{}'.format(dt.now().year, dt.now().month, dt.now().day)
     index = IndexQ(test_config['indexqbase'], 'testq', size = 1, log = log,
                    rotate_complete = rotate_func)
     dir_set = rotate_func()
     docs = self.rand_docs.get_docs(69)
     for item in self.docs[1:10]:
         index.add(item, finalize=True)
     files = []
     for item in index.get_all_as_list():
         files.append(index.complete(item))
     [self.assertTrue(os.path.exists(x)) for x in files]

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import get_all_as_list [as 别名]
 def test_by_get_all_default_compression(self):
     size = 1
     files = 2
     index = IndexQ(test_config['indexqbase'], 'testq', size=size)
     buff = []
     docs = []
     for _ in range(files):
         doc = index.add(self.docs,finalize=True)
         docs.append(doc)
         sleep(1)
     index = IndexQ(test_config['indexqbase'], 'testq', mode='out')
     indexdocs = index.get_all_as_list()
     self.assertEqual(docs,indexdocs)
     [os.remove(doc) for doc in docs]

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import get_all_as_list [as 别名]
 def test_solr_to_indexq(self):
     """
     Will export documents from Solr and put them into an IndexQ. 
     """
     index = IndexQ(test_config["indexqbase"], "test_reindexer", size=0)
     for dir in ["_todo_dir", "_done_dir"]:
         [os.remove(x) for x in index.get_all_as_list(dir=dir)]
     self._index_docs(5000, self.colls[0])
     reindexer = Reindexer(source=self.solr, source_coll="source_coll", dest=index)
     reindexer.reindex()
     from_files = self.get_all_json_from_indexq(index)
     from_solr = self.solr.query("source_coll", {"q": "*:*", "rows": 5000}).docs
     from_solr = reindexer._trim_fields(from_solr)
     self.assertEqual(sorted(from_files, key=lambda x: x["id"]), sorted(from_solr, key=lambda x: x["id"]))

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import get_all_as_list [as 别名]
 def test_dequeue_100(self):
     size = 1
     files = 100
     rdocs = self.rand_docs.get_docs(500)
     index = IndexQ(test_config['indexqbase'], 'testq', size=size)
     buff = []
     docs = []
     for dir in ['_todo_dir','_done_dir']:
         [os.remove(x) for x in index.get_all_as_list(dir=dir)]
     for _ in range(files):
         doc = index.add(rdocs,finalize=True)
         docs.append(doc)
     index = IndexQ(test_config['indexqbase'], 'testq')
     indexdocs = []
     for x in index.get_todo_items():
         indexdocs.append(x)
     self.assertEqual(docs,indexdocs)
     [os.remove(doc) for doc in docs]

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import get_all_as_list [as 别名]
    def test_thread_pool_high(self):
        '''
        Index data using multiple threads.
        Verity that each thread
        '''
        docs = self.rand_docs.get_docs(25000)
        index = IndexQ(test_config['indexqbase'],
                       'testq',
                       size=.1,
                       devel=True)
        for dir in ['_todo_dir', '_done_dir']:
            [os.remove(x) for x in index.get_all_as_list(dir=dir)]
        threads = 25

        with ThreadPool(threads) as p:
            p.map(index.add, docs)
        index.add(finalize=True)
        d = index.get_all_json_from_indexq()
        self.assertEqual(len(d), len(docs))
        self.assertEqual(sorted(d, key=lambda x: x['id']),
                         sorted(docs, key=lambda x: x['id']))

# 需要导入模块: from SolrClient import IndexQ [as 别名]
# 或者: from SolrClient.IndexQ import get_all_as_list [as 别名]
 def setUp(self):
     index = IndexQ(test_config['indexqbase'], 'testq')
     for dir in ['_todo_dir', '_done_dir']:
         [os.remove(x) for x in index.get_all_as_list(dir=dir)]