Python Connector.join方法代码示例

# 需要导入模块: from mongo_connector.connector import Connector [as 别名]
# 或者: from mongo_connector.connector.Connector import join [as 别名]
    def test_connector_minimum_privileges(self):
        """Test the Connector works with a user with minimum privileges."""
        if not (db_user and db_password):
            raise SkipTest("Need to set a user/password to test this.")
        client = self.repl_set.client()
        minimum_user = "read_local_and_included_databases"
        minimum_pwd = "password"
        client.admin.add_user(
            minimum_user,
            minimum_pwd,
            roles=[
                {"role": "read", "db": "test"},
                {"role": "read", "db": "wildcard"},
                {"role": "read", "db": "local"},
            ],
        )

        client.test.test.insert_one({"replicated": 1})
        client.test.ignored.insert_one({"replicated": 0})
        client.ignored.ignored.insert_one({"replicated": 0})
        client.wildcard.test.insert_one({"replicated": 1})
        conn = Connector(
            mongo_address=self.repl_set.primary.uri,
            auth_username=minimum_user,
            auth_key=minimum_pwd,
            namespace_options={"test.test": True, "wildcard.*": True},
        )
        conn.start()
        try:
            assert_soon(conn.doc_managers[0]._search)
        finally:
            conn.join()

# 需要导入模块: from mongo_connector.connector import Connector [as 别名]
# 或者: from mongo_connector.connector.Connector import join [as 别名]
    def test_start_with_auth(self):
        dm = DocManager()
        connector = Connector(
            mongo_address=self.cluster.uri,
            doc_managers=[dm],
            auth_username=db_user,
            auth_key=db_password
        )
        connector.start()

        # Insert some documents into the sharded cluster.  These
        # should go to the DocManager, and the connector should not
        # have an auth failure.
        self.cluster.client().test.test.insert({'auth_failure': False})
        assert_soon(lambda: len(dm._search()) > 0)

        connector.join()

# 需要导入模块: from mongo_connector.connector import Connector [as 别名]
# 或者: from mongo_connector.connector.Connector import join [as 别名]
    def test_connector(self):
        """Test whether the connector initiates properly
        """
        conn = Connector(
            mongo_address=self.repl_set.uri,
            **connector_opts
        )
        conn.start()

        while len(conn.shard_set) != 1:
            time.sleep(2)
        conn.join()

        self.assertFalse(conn.can_run)
        time.sleep(5)
        for thread in conn.shard_set.values():
            self.assertFalse(thread.running)

# 需要导入模块: from mongo_connector.connector import Connector [as 别名]
# 或者: from mongo_connector.connector.Connector import join [as 别名]
    def test_connector(self):
        """Test whether the connector initiates properly
        """
        conn = Connector(mongo_address=self.repl_set.uri, **connector_opts)
        conn.start()
        assert_soon(lambda: bool(conn.shard_set))

        # Make sure get_mininum_mongodb_version returns the current version.
        self.assertEqual(
            Version.from_client(self.repl_set.client()), get_mininum_mongodb_version()
        )

        conn.join()

        # Make sure the connector is shutdown correctly
        self.assertFalse(conn.can_run)
        for thread in conn.shard_set.values():
            self.assertFalse(thread.running)

# 需要导入模块: from mongo_connector.connector import Connector [as 别名]
# 或者: from mongo_connector.connector.Connector import join [as 别名]
    def test_connector(self):
        """Test whether the connector initiates properly
        """
        if not self.flag:
            self.fail("Shards cannot be added to mongos")

        conn = Connector(MAIN_ADDRESS, CONFIG, None, ['test.test'],
                      '_id', None, None)
        conn.start()

        while len(conn.shard_set) != 1:
            time.sleep(2)
        conn.join()

        self.assertFalse(conn.can_run)
        time.sleep(5)
        for thread in conn.shard_set.values():
            self.assertFalse(thread.running)

# 需要导入模块: from mongo_connector.connector import Connector [as 别名]
# 或者: from mongo_connector.connector.Connector import join [as 别名]
    def test_connector(self):
        """Test whether the connector initiates properly
        """
        conn = Connector(
            address='%s:%d' % (mongo_host, self.primary_p),
            oplog_checkpoint='config.txt',
            target_url=None,
            ns_set=['test.test'],
            u_key='_id',
            auth_key=None
        )
        conn.start()

        while len(conn.shard_set) != 1:
            time.sleep(2)
        conn.join()

        self.assertFalse(conn.can_run)
        time.sleep(5)
        for thread in conn.shard_set.values():
            self.assertFalse(thread.running)

# 需要导入模块: from mongo_connector.connector import Connector [as 别名]
# 或者: from mongo_connector.connector.Connector import join [as 别名]
class MongoReplicaSetTestCase(MongoTestCase):

    def setUp(self):
        self.repl_set = self.replica_set_class().start()
        self.conn = self.repl_set.client()

        try:
            os.unlink("oplog.timestamp")
        except OSError:
            pass
        self._remove()
        self.connector = Connector(
            mongo_address=self.repl_set.uri,
            doc_managers=(self.mongo_doc,),
            namespace_options={
                'test.test': {'gridfs': True},
                'rename.me': 'new.target',
                'rename.me2': 'new2.target2'
            },
            **connector_opts
        )

        self.connector.start()
        assert_soon(lambda: len(self.connector.shard_set) > 0)
        assert_soon(lambda: sum(1 for _ in self._search()) == 0)

    def drop_all_databases(self):
        for name in self.mongo_conn.database_names():
            if name not in ["local", "admin"]:
                self.mongo_conn.drop_database(name)
        for name in self.conn.database_names():
            if name not in ["local", "admin"]:
                self.conn.drop_database(name)

    def tearDown(self):
        self.connector.join()
        self.drop_all_databases()
        self.repl_set.stop()

# 需要导入模块: from mongo_connector.connector import Connector [as 别名]
# 或者: from mongo_connector.connector.Connector import join [as 别名]
class TestConnectorSharded(unittest.TestCase):

    def setUp(self):
        if db_user and db_password:
            auth_args = dict(auth_username=db_user, auth_key=db_password)
        else:
            auth_args = {}
        self.cluster = ShardedClusterSingle().start()
        self.dm = DocManager()
        self.connector = Connector(
            mongo_address=self.cluster.uri,
            doc_managers=[self.dm],
            **auth_args
        )
        self.connector.start()

    def tearDown(self):
        self.connector.join()
        try:
            os.unlink('oplog.timestamp')
        except OSError:
            pass
        self.cluster.stop()

# 需要导入模块: from mongo_connector.connector import Connector [as 别名]
# 或者: from mongo_connector.connector.Connector import join [as 别名]
    def test_connector(self):
        """Test whether the connector initiates properly
        """
        if not self.flag:
            self.fail("Shards cannot be added to mongos")

        conn = Connector(
            address=MAIN_ADDRESS,
            oplog_checkpoint=CONFIG,
            target_url=None,
            ns_set=['test.test'],
            u_key='_id',
            auth_key=None
        )
        conn.start()

        while len(conn.shard_set) != 1:
            time.sleep(2)
        conn.join()

        self.assertFalse(conn.can_run)
        time.sleep(5)
        for thread in conn.shard_set.values():
            self.assertFalse(thread.running)

# 需要导入模块: from mongo_connector.connector import Connector [as 别名]
# 或者: from mongo_connector.connector.Connector import join [as 别名]
class TestSolr(SolrTestCase):
    """ Tests Solr
    """

    @classmethod
    def setUpClass(cls):
        SolrTestCase.setUpClass()
        cls.repl_set = ReplicaSet().start()
        cls.conn = cls.repl_set.client()

    @classmethod
    def tearDownClass(cls):
        """ Kills cluster instance
        """
        cls.repl_set.stop()

    def setUp(self):
        self._remove()
        try:
            os.unlink("oplog.timestamp")
        except OSError:
            pass
        open("oplog.timestamp", "w").close()
        docman = DocManager('http://%s/solr' % solr_pair,
                            auto_commit_interval=0)
        self.connector = Connector(
            mongo_address=self.repl_set.uri,
            ns_set=['test.test'],
            doc_managers=(docman,),
            gridfs_set=['test.test']
        )
        retry_until_ok(self.conn.test.test.drop)
        retry_until_ok(self.conn.test.test.files.drop)
        retry_until_ok(self.conn.test.test.chunks.drop)
        self._remove()
        self.connector.start()
        assert_soon(lambda: len(self.connector.shard_set) > 0)

    def tearDown(self):
        self.connector.join()

    def test_insert(self):
        """Tests insert
        """

        self.conn['test']['test'].insert({'name': 'paulie'})
        assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0)
        result_set_1 = list(self.solr_conn.search('paulie'))
        self.assertEqual(len(result_set_1), 1)
        result_set_2 = self.conn['test']['test'].find_one()
        for item in result_set_1:
            self.assertEqual(item['_id'], str(result_set_2['_id']))
            self.assertEqual(item['name'], result_set_2['name'])

    def test_remove(self):
        """Tests remove
        """
        self.conn['test']['test'].insert({'name': 'paulie'})
        assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
        self.conn['test']['test'].remove({'name': 'paulie'})
        assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)

    def test_insert_file(self):
        """Tests inserting a gridfs file
        """
        fs = GridFS(self.conn['test'], 'test')
        test_data = "test_insert_file test file"
        id = fs.put(test_data, filename="test.txt", encoding='utf8')
        assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0)

        res = list(self.solr_conn.search('test_insert_file'))
        self.assertEqual(len(res), 1)
        doc = res[0]
        self.assertEqual(doc['filename'], "test.txt")
        self.assertEqual(doc['_id'], str(id))
        self.assertEqual(doc['content'][0].strip(), test_data.strip())

    def test_remove_file(self):
        """Tests removing a gridfs file
        """
        fs = GridFS(self.conn['test'], 'test')
        id = fs.put("test file", filename="test.txt", encoding='utf8')
        assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
        fs.delete(id)
        assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)

    def test_update(self):
        """Test update operations on Solr.

        Need to have the following defined in schema.xml:

        <field name="a" type="int" indexed="true" stored="true" />
        <field name="b.0.c" type="int" indexed="true" stored="true" />
        <field name="b.10.c" type="int" indexed="true" stored="true" />
        <field name="b.0.e" type="int" indexed="true" stored="true" />
        <field name="b.1.d" type="int" indexed="true" stored="true" />
        <field name="b.1.f" type="int" indexed="true" stored="true" />
        <field name="b.2.e" type="int" indexed="true" stored="true" />
        """
        docman = self.connector.doc_managers[0]
#.........这里部分代码省略.........

# 需要导入模块: from mongo_connector.connector import Connector [as 别名]
# 或者: from mongo_connector.connector.Connector import join [as 别名]
class TestSynchronizer(unittest.TestCase):
    """ Tests Solr
    """

    def runTest(self):
        """ Runs tests
        """
        unittest.TestCase.__init__(self)

    @classmethod
    def setUpClass(cls):
        os.system('rm %s; touch %s' % (CONFIG, CONFIG))
        cls.flag = start_cluster()
        if cls.flag:
            cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN']),
                replicaSet="demo-repl")
            # Creating a Solr object with an invalid URL 
            # doesn't create an exception
            cls.solr_conn = Solr('http://localhost:8983/solr')
            try:
                cls.solr_conn.commit()
            except (SolrError, MissingSchema):
                cls.err_msg = "Cannot connect to Solr!"
                cls.flag = False
            if cls.flag:    
                cls.solr_conn.delete(q='*:*')
        else:
            cls.err_msg = "Shards cannot be added to mongos"        

    @classmethod
    def tearDownClass(cls):
        """ Kills cluster instance
        """
        kill_all()


    def setUp(self):
        if not self.flag:
            self.fail(self.err_msg)

        self.connector = Connector(
            address=('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN'])),
            oplog_checkpoint=CONFIG,
            target_url='http://localhost:8983/solr',
            ns_set=['test.test'],
            u_key='_id',
            auth_key=None,
            doc_manager='mongo_connector/doc_managers/solr_doc_manager.py'
        )
        self.connector.start()
        while len(self.connector.shard_set) == 0:
            time.sleep(1)
        count = 0
        while (True):
            try:
                self.conn['test']['test'].remove(safe=True)
                break
            except (AutoReconnect, OperationFailure):
                time.sleep(1)
                count += 1
                if count > 60:
                    unittest.SkipTest('Call to remove failed too '
                    'many times in setup')
        while (len(self.solr_conn.search('*:*')) != 0):
            time.sleep(1)

    def tearDown(self):
        self.connector.join()

    def test_shard_length(self):
        """Tests the shard_length to see if the shard set was recognized
        """

        self.assertEqual(len(self.connector.shard_set), 1)

    def test_initial(self):
        """Tests search and assures that the databases are clear.
        """

        while (True):
            try:
                self.conn['test']['test'].remove(safe=True)
                break
            except OperationFailure:
                continue

        self.solr_conn.delete(q='*:*')
        self.assertEqual(self.conn['test']['test'].find().count(), 0)
        self.assertEqual(len(self.solr_conn.search('*:*')), 0)

    def test_insert(self):
        """Tests insert
        """

        self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
        while (len(self.solr_conn.search('*:*')) == 0):
            time.sleep(1)
        result_set_1 = self.solr_conn.search('paulie')
        self.assertEqual(len(result_set_1), 1)
        result_set_2 = self.conn['test']['test'].find_one()
#.........这里部分代码省略.........

# 需要导入模块: from mongo_connector.connector import Connector [as 别名]
# 或者: from mongo_connector.connector.Connector import join [as 别名]
class TestSynchronizer(unittest.TestCase):
    """ Tests Solr
    """

    def runTest(self):
        """ Runs tests
        """
        unittest.TestCase.__init__(self)

    @classmethod
    def setUpClass(cls):
        os.system('rm %s; touch %s' % (CONFIG, CONFIG))
        cls.flag = start_cluster()
        if cls.flag:
            cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN']),
                replicaSet="demo-repl")
            # Creating a Solr object with an invalid URL 
            # doesn't create an exception
            cls.solr_conn = Solr('http://localhost:8983/solr')
            try:
                cls.solr_conn.commit()
            except (SolrError, MissingSchema):
                cls.err_msg = "Cannot connect to Solr!"
                cls.flag = False
            if cls.flag:    
                cls.solr_conn.delete(q='*:*')
        else:
            cls.err_msg = "Shards cannot be added to mongos"        

    @classmethod
    def tearDownClass(cls):
        """ Kills cluster instance
        """
        kill_all()


    def setUp(self):
        if not self.flag:
            self.fail(self.err_msg)

        self.connector = Connector(('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN'])),
            CONFIG, 'http://localhost:8983/solr', ['test.test'], '_id',
            None, 
            'mongo_connector/doc_managers/solr_doc_manager.py')
        self.connector.start()
        while len(self.connector.shard_set) == 0:
            time.sleep(1)
        count = 0
        while (True):
            try:
                self.conn['test']['test'].remove(safe=True)
                break
            except (AutoReconnect, OperationFailure):
                time.sleep(1)
                count += 1
                if count > 60:
                    unittest.SkipTest('Call to remove failed too '
                    'many times in setup')
        while (len(self.solr_conn.search('*:*')) != 0):
            time.sleep(1)

    def tearDown(self):
        self.connector.doc_manager.auto_commit = False
        time.sleep(2)
        self.connector.join()

    def test_shard_length(self):
        """Tests the shard_length to see if the shard set was recognized
        """

        self.assertEqual(len(self.connector.shard_set), 1)

    def test_initial(self):
        """Tests search and assures that the databases are clear.
        """

        while (True):
            try:
                self.conn['test']['test'].remove(safe=True)
                break
            except OperationFailure:
                continue

        self.solr_conn.delete(q='*:*')
        self.assertEqual(self.conn['test']['test'].find().count(), 0)
        self.assertEqual(len(self.solr_conn.search('*:*')), 0)

    def test_insert(self):
        """Tests insert
        """

        self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
        while (len(self.solr_conn.search('*:*')) == 0):
            time.sleep(1)
        result_set_1 = self.solr_conn.search('paulie')
        self.assertEqual(len(result_set_1), 1)
        result_set_2 = self.conn['test']['test'].find_one()
        for item in result_set_1:
            self.assertEqual(item['_id'], str(result_set_2['_id']))
            self.assertEqual(item['name'], result_set_2['name'])
#.........这里部分代码省略.........

# 需要导入模块: from mongo_connector.connector import Connector [as 别名]
# 或者: from mongo_connector.connector.Connector import join [as 别名]
class TestSynchronizer(unittest.TestCase):
    """ Tests Solr
    """

    @classmethod
    def setUpClass(cls):
        _, cls.secondary_p, cls.primary_p = start_replica_set('test-solr')
        cls.conn = MongoClient(mongo_host, cls.primary_p,
                               replicaSet='test-solr')
        cls.solr_conn = Solr('http://%s/solr' % solr_pair)
        cls.solr_conn.delete(q='*:*')

    @classmethod
    def tearDownClass(cls):
        """ Kills cluster instance
        """
        kill_replica_set('test-solr')

    def setUp(self):
        try:
            os.unlink("config.txt")
        except OSError:
            pass
        open("config.txt", "w").close()
        self.connector = Connector(
            address='%s:%s' % (mongo_host, self.primary_p),
            oplog_checkpoint='config.txt',
            target_url='http://localhost:8983/solr',
            ns_set=['test.test'],
            u_key='_id',
            auth_key=None,
            doc_manager='mongo_connector/doc_managers/solr_doc_manager.py',
            auto_commit_interval=0
        )
        self.connector.start()
        assert_soon(lambda: len(self.connector.shard_set) > 0)
        retry_until_ok(self.conn.test.test.remove)
        assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 0)

    def tearDown(self):
        self.connector.join()

    def test_shard_length(self):
        """Tests the shard_length to see if the shard set was recognized
        """

        self.assertEqual(len(self.connector.shard_set), 1)

    def test_insert(self):
        """Tests insert
        """

        self.conn['test']['test'].insert({'name': 'paulie'})
        assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0)
        result_set_1 = list(self.solr_conn.search('paulie'))
        self.assertEqual(len(result_set_1), 1)
        result_set_2 = self.conn['test']['test'].find_one()
        for item in result_set_1:
            self.assertEqual(item['_id'], str(result_set_2['_id']))
            self.assertEqual(item['name'], result_set_2['name'])

    def test_remove(self):
        """Tests remove
        """
        self.conn['test']['test'].insert({'name': 'paulie'})
        assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
        self.conn['test']['test'].remove({'name': 'paulie'})
        assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)

    def test_update(self):
        """Test update operations on Solr.

        Need to have the following defined in schema.xml:

        <field name="a" type="int" indexed="true" stored="true" />
        <field name="b.0.c" type="int" indexed="true" stored="true" />
        <field name="b.0.e" type="int" indexed="true" stored="true" />
        <field name="b.1.d" type="int" indexed="true" stored="true" />
        <field name="b.1.f" type="int" indexed="true" stored="true" />
        """
        docman = self.connector.doc_managers[0]

        # Insert
        self.conn.test.test.insert({"a": 0})
        assert_soon(lambda: sum(1 for _ in docman._search("*:*")) == 1)

        def check_update(update_spec):
            updated = self.conn.test.test.find_and_modify(
                {"a": 0},
                update_spec,
                new=True
            )
            # Stringify _id to match what will be retrieved from Solr
            updated['_id'] = str(updated['_id'])
            # Flatten the MongoDB document to match Solr
            updated = docman._clean_doc(updated)
            # Allow some time for update to propagate
            time.sleep(1)
            replicated = list(docman._search("a:0"))[0]
            # Remove add'l fields until these are stored in a separate Solr core
#.........这里部分代码省略.........