Python HTTPConnectionPool.closeCachedConnections方法代码示例

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class StateTestCase(unittest.TestCase):
    def setUp(self):
        pnconf_uuid_set = copy(pnconf)
        pnconf_uuid_set.uuid = 'someuuid'
        self.pool = HTTPConnectionPool(reactor, persistent=False)
        self.pubnub = PubNubTwisted(pnconf_uuid_set, reactor=reactor, pool=self.pool)

    def tearDown(self):
        return self.pool.closeCachedConnections()

    def assert_valid_state_envelope(self, envelope):
        self.assertIsInstance(envelope, TwistedEnvelope)
        self.assertIsInstance(envelope.result, PNSetStateResult)
        self.assertEqual(envelope.result.state, state)

    @inlineCallbacks
    @pn_vcr.use_cassette(
        'tests/integrational/fixtures/twisted/state/single_channel.yaml',
        filter_query_parameters=['uuid'])
    def test_state_single_channel(self):
        envelope = yield self.pubnub.set_state().channels(channel).state(state).deferred()
        self.assert_valid_state_envelope(envelope)
        returnValue(envelope)

    @inlineCallbacks
    @pn_vcr.use_cassette(
        'tests/integrational/fixtures/twisted/state/multiple_channels.yaml',
        filter_query_parameters=['uuid'])
    def test_state_multiple_channels(self):
        envelope = yield self.pubnub.set_state().channels(channels).state(state).deferred()
        self.assert_valid_state_envelope(envelope)
        returnValue(envelope)

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class HTTP11DownloadHandler(object):
    def __init__(self, settings):
        self.settings = settings
        self.ssl_methods = settings.get('DOWNLOAD_HANDLER_SSL_METHODS')
        self.context_factories = [CrawlmiClientContextFactory(method) for method in self.ssl_methods]
        self.pool = HTTPConnectionPool(reactor, persistent=True)
        self.pool.maxPersistentPerHost = settings.get_int('CONCURRENT_REQUESTS_PER_DOMAIN')
        self.pool._factory.noisy = False

    def download_request(self, request):
        '''Return a deferred for the HTTP download.'''
        dfd = None
        for context_factory in self.context_factories:
            if dfd is None:
                dfd = self._download(request, context_factory)
            else:
                def _failure(failure):
                    failure.trap(SSL.Error)
                    return self._download(request, context_factory)
                dfd.addErrback(_failure)
        return dfd

    def _download(self, request, context_factory):
        agent = CrawlmiAgent(
            context_factory,
            self.settings.get_float('DOWNLOAD_TIMEOUT', 180, request),
            self.settings.get_int('DOWNLOAD_SIZE_LIMIT', 0, request),
            request.meta.get('bind_address'),
            self.pool)
        return agent.download_request(request)

    def close(self):
        return self.pool.closeCachedConnections()

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class SFTPFuncTest(unittest.TestCase):
    @defer.inlineCallbacks
    def setUp(self):
        self.active_connections = []
        self.pool = HTTPConnectionPool(reactor, persistent=True)
        self.swift = get_swift_client(CONFIG, pool=self.pool)
        self.tmpdir = tempfile.mkdtemp()
        _, self.sftp = self.get_client()
        yield clean_swift(self.swift)

    @defer.inlineCallbacks
    def tearDown(self):
        shutil.rmtree(self.tmpdir)
        for (transport, conn) in self.active_connections:
            try:
                conn.close()
            except:
                pass
            try:
                transport.close()
            except:
                pass

        yield clean_swift(self.swift)
        yield self.pool.closeCachedConnections()

    def get_client(self):
        transport, conn = get_sftp_client_with_transport(CONFIG)
        self.active_connections.append((transport, conn))
        return transport, conn

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class HTTP11DownloadHandler(object):
    lazy = False

    def __init__(self, settings):
        self._pool = HTTPConnectionPool(reactor, persistent=True)
        self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
        self._pool._factory.noisy = False

        self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
        self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
        # try method-aware context factory
        try:
            self._contextFactory = self._contextFactoryClass(method=self._sslMethod)
        except TypeError:
            # use context factory defaults
            self._contextFactory = self._contextFactoryClass()
            msg = """
 '%s' does not accept `method` argument (type OpenSSL.SSL method,\
 e.g. OpenSSL.SSL.SSLv23_METHOD).\
 Please upgrade your context factory class to handle it or ignore it.""" % (
                settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
            warnings.warn(msg)
        self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
        self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
        self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
        self._disconnect_timeout = 1

    def download_request(self, request, spider):
        """Return a deferred for the HTTP download"""
        agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool,
            maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
            warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
            fail_on_dataloss=self._fail_on_dataloss)
        return agent.download_request(request)

    def close(self):
        d = self._pool.closeCachedConnections()
        # closeCachedConnections will hang on network or server issues, so
        # we'll manually timeout the deferred.
        #
        # Twisted issue addressing this problem can be found here:
        # https://twistedmatrix.com/trac/ticket/7738.
        #
        # closeCachedConnections doesn't handle external errbacks, so we'll
        # issue a callback after `_disconnect_timeout` seconds.
        delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, [])

        def cancel_delayed_call(result):
            if delayed_call.active():
                delayed_call.cancel()
            return result

        d.addBoth(cancel_delayed_call)
        return d

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class CouchStateTests(CouchDBTestCase):

    def setUp(self):
        CouchDBTestCase.setUp(self)
        self.db = self.couch_server.create('user-' + uuid4().hex)
        self.addCleanup(self.delete_db, self.db.name)
        self.pool = HTTPConnectionPool(reactor, persistent=False)
        self.agent = Agent(reactor, pool=self.pool)

    @defer.inlineCallbacks
    def tearDown(self):
        yield self.pool.closeCachedConnections()

    @restricted_listing
    @defer.inlineCallbacks
    def test__check_db_schema_version_wrong_schema_version_raises(self):
        wrong_schema_version = SCHEMA_VERSION + 1
        self.db.create(
            {'_id': CONFIG_DOC_ID, SCHEMA_VERSION_KEY: wrong_schema_version})
        with pytest.raises(WrongCouchSchemaVersionError):
            yield _check_db_schema_version(
                self.couch_url, self.db.name, None, agent=self.agent)

    @restricted_listing
    @defer.inlineCallbacks
    def test_check_schema_versions_wrong_schema_version_raises(self):
        wrong_schema_version = SCHEMA_VERSION + 1
        self.db.create(
            {'_id': CONFIG_DOC_ID, SCHEMA_VERSION_KEY: wrong_schema_version})
        expected_msg = 'Error checking CouchDB schema versions: ' \
                       'FirstError.*WrongCouchSchemaVersionError()'
        with pytest.raises(Exception, match=expected_msg):
            yield check_schema_versions(self.couch_url, agent=self.agent)

    @restricted_listing
    @defer.inlineCallbacks
    def test__check_db_schema_version_missing_config_doc_raises(self):
        self.db.create({})
        with pytest.raises(MissingCouchConfigDocumentError):
            yield _check_db_schema_version(
                self.couch_url, self.db.name, None, agent=self.agent)

    @restricted_listing
    @defer.inlineCallbacks
    def test_check_schema_versions_missing_config_doc_raises(self):
        self.db.create({})
        expected_msg = 'Error checking CouchDB schema versions: ' \
                       'FirstError.*MissingCouchConfigDocumentError()'
        with pytest.raises(Exception, match=expected_msg):
            yield check_schema_versions(self.couch_url, agent=self.agent)

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class FTPFuncTest(unittest.TestCase):
    @defer.inlineCallbacks
    def setUp(self):
        self.pool = HTTPConnectionPool(reactor, persistent=True)
        self.swift = get_swift_client(conf, pool=self.pool)
        self.tmpdir = tempfile.mkdtemp()
        self.ftp = get_ftp_client(conf)
        yield clean_swift(self.swift)

    @defer.inlineCallbacks
    def tearDown(self):
        shutil.rmtree(self.tmpdir)
        self.ftp.close()
        yield clean_swift(self.swift)
        yield self.pool.closeCachedConnections()

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class AbstractBaseApiTest(TestAsServer):
    """
    Tests for the Tribler HTTP API should create a subclass of this class.
    """
    @inlineCallbacks
    def setUp(self):
        yield super(AbstractBaseApiTest, self).setUp()
        self.connection_pool = HTTPConnectionPool(reactor, False)

    @inlineCallbacks
    def tearDown(self):
        yield self.close_connections()
        yield super(AbstractBaseApiTest, self).tearDown()

    def close_connections(self):
        return self.connection_pool.closeCachedConnections()

    def setUpPreSession(self):
        super(AbstractBaseApiTest, self).setUpPreSession()
        self.config.set_http_api_enabled(True)
        self.config.set_http_api_retry_port(True)
        self.config.set_tunnel_community_enabled(False)

        # Make sure we select a random port for the HTTP API
        min_base_port = 1000 if not os.environ.get("TEST_BUCKET", None) \
            else int(os.environ['TEST_BUCKET']) * 2000 + 2000
        self.config.set_http_api_port(get_random_port(min_port=min_base_port, max_port=min_base_port + 2000))

    def do_request(self, endpoint, req_type, post_data, raw_data):
        try:
            req_type = req_type.encode('utf-8')
        except AttributeError:
            pass
        agent = Agent(reactor, pool=self.connection_pool)
        return agent.request(req_type, 'http://localhost:%s/%s' % (self.session.config.get_http_api_port(), endpoint),
                             Headers({'User-Agent': ['Tribler ' + version_id],
                                      "Content-Type": ["text/plain; charset=utf-8"]}),
                             POSTDataProducer(post_data, raw_data))

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class HereNowTest(unittest.TestCase):
    def setUp(self):
        self.pool = HTTPConnectionPool(reactor, persistent=False)
        self.pubnub = PubNubTwisted(pnconf, reactor=reactor, pool=self.pool)

    def tearDown(self):
        return self.pool.closeCachedConnections()

    class PNHereNowChannelData(object):
        def __init__(self, channel_name, occupancy, occupants):
            self.channel_name = channel_name
            self.occupancy = occupancy
            self.occupants = occupants

    def assert_valid_here_now_envelope(self, envelope, result_channels):
        def get_uuids(here_now_channel_data):
            return [here_now_channel_data.channel_name,
                    here_now_channel_data.occupancy,
                    map(lambda x: x.uuid, here_now_channel_data.occupants)]

        self.assertIsInstance(envelope, TwistedEnvelope)
        self.assertIsInstance(envelope.result, PNHereNowResult)
        self.assertEqual(map(get_uuids, envelope.result.channels), result_channels)

    @inlineCallbacks
    @pn_vcr.use_cassette(
        'tests/integrational/fixtures/twisted/here_now/global.yaml',
        filter_query_parameters=['uuid'])
    def test_global_here_now(self):
        envelope = yield self.pubnub.here_now() \
            .include_uuids(True) \
            .deferred()

        self.assert_valid_here_now_envelope(envelope,
                                            [[u'twisted-test-1', 1, [u'00de2586-7ad8-4955-b5f6-87cae3215d02']],
                                             [u'twisted-test', 1, [u'00de2586-7ad8-4955-b5f6-87cae3215d02']]])
        returnValue(envelope)

    @inlineCallbacks
    @pn_vcr.use_cassette(
        'tests/integrational/fixtures/twisted/here_now/single.yaml',
        filter_query_parameters=['uuid'])
    def test_here_now_single_channel(self):
        envelope = yield self.pubnub.here_now() \
            .channels(channel) \
            .include_uuids(True) \
            .deferred()

        self.assert_valid_here_now_envelope(envelope, [['twisted-test', 1, [u'00de2586-7ad8-4955-b5f6-87cae3215d02']]])
        returnValue(envelope)

    @inlineCallbacks
    @pn_vcr.use_cassette(
        'tests/integrational/fixtures/twisted/here_now/multiple.yaml',
        filter_query_parameters=['uuid'])
    def test_here_now_multiple_channels(self):
        envelope = yield self.pubnub.here_now() \
            .channels(channels) \
            .include_uuids(True) \
            .deferred()

        self.assert_valid_here_now_envelope(envelope,
                                            [[u'twisted-test-1', 1, [u'00de2586-7ad8-4955-b5f6-87cae3215d02']]])
        returnValue(envelope)

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class TorrentChecker(TaskManager):

    def __init__(self, session):
        super(TorrentChecker, self).__init__()
        self._logger = logging.getLogger(self.__class__.__name__)
        self.tribler_session = session

        self._should_stop = False

        self.tracker_check_lc = self.register_task("tracker_check", LoopingCall(self.check_random_tracker))
        self.torrent_check_lc = self.register_task("torrent_check", LoopingCall(self.check_random_torrent))

        self._session_list = {'DHT': []}

        # Track all session cleanups
        self.session_stop_defer_list = []

        self.socket_mgr = self.udp_port = None
        self.connection_pool = None

    def initialize(self):
        self.tracker_check_lc.start(TRACKER_SELECTION_INTERVAL, now=False)
        self.torrent_check_lc.start(TORRENT_SELECTION_INTERVAL, now=False)
        self.connection_pool = HTTPConnectionPool(reactor, False)
        self.socket_mgr = UdpSocketManager()
        self.create_socket_or_schedule()

    def listen_on_udp(self):
        return reactor.listenUDP(0, self.socket_mgr)

    def create_socket_or_schedule(self):
        """
        This method attempts to bind to a UDP port. If it fails for some reason (i.e. no network connection), we try
        again later.
        """
        try:
            self.udp_port = self.listen_on_udp()
        except socket.error as exc:
            self._logger.error("Error when creating UDP socket in torrent checker: %s", exc)
            self.register_task("listen_udp_port", reactor.callLater(10, self.create_socket_or_schedule))

    def shutdown(self):
        """
        Shutdown the torrent health checker.

        Once shut down it can't be started again.
        :returns A deferred that will fire once the shutdown has completed.
        """
        self._should_stop = True

        if self.udp_port:
            self.session_stop_defer_list.append(maybeDeferred(self.udp_port.stopListening))
            self.udp_port = None

        if self.connection_pool:
            self.session_stop_defer_list.append(self.connection_pool.closeCachedConnections())

        self.shutdown_task_manager()

        # kill all the tracker sessions.
        # Wait for the defers to all have triggered by using a DeferredList
        for tracker_url in self._session_list.keys():
            for session in self._session_list[tracker_url]:
                self.session_stop_defer_list.append(session.cleanup())

        return DeferredList(self.session_stop_defer_list)

    def check_random_tracker(self):
        """
        Calling this method will fetch a random tracker from the database, select some torrents that have this
        tracker, and perform a request to these trackers.
        """
        tracker_url = self.get_valid_next_tracker_for_auto_check()
        if tracker_url is None:
            self._logger.warn(u"No tracker to select from, skip")
            return succeed(None)

        self._logger.debug(u"Start selecting torrents on tracker %s.", tracker_url)

        # get the torrents that should be checked
        infohashes = []
        with db_session:
            tracker = self.tribler_session.lm.mds.TrackerState.get(url=tracker_url)
            if tracker:
                torrents = tracker.torrents
                for torrent in torrents:
                    dynamic_interval = TORRENT_CHECK_RETRY_INTERVAL * (2 ** tracker.failures)
                    if torrent.last_check + dynamic_interval < int(time.time()):
                        infohashes.append(torrent.infohash)

        if len(infohashes) == 0:
            # We have no torrent to recheck for this tracker. Still update the last_check for this tracker.
            self._logger.info("No torrent to check for tracker %s", tracker_url)
            self.update_tracker_info(tracker_url, True)
            return succeed(None)
        elif tracker_url != u'DHT' and tracker_url != u'no-DHT':
            try:
                session = self._create_session_for_request(tracker_url, timeout=30)
            except MalformedTrackerURLException as e:
                # Remove the tracker from the database
#.........这里部分代码省略.........

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class TreqIntegrationTests(TestCase):
    baseurl = HTTPBIN_URL
    get = with_baseurl(treq.get)
    head = with_baseurl(treq.head)
    post = with_baseurl(treq.post)
    put = with_baseurl(treq.put)
    patch = with_baseurl(treq.patch)
    delete = with_baseurl(treq.delete)

    def setUp(self):
        self.pool = HTTPConnectionPool(reactor, False)

    def tearDown(self):
        def _check_fds(_):
            # This appears to only be necessary for HTTPS tests.
            # For the normal HTTP tests then closeCachedConnections is
            # sufficient.
            fds = set(reactor.getReaders() + reactor.getReaders())
            if not [fd for fd in fds if isinstance(fd, Client)]:
                return

            return deferLater(reactor, 0, _check_fds, None)

        return self.pool.closeCachedConnections().addBoth(_check_fds)

    @inlineCallbacks
    def assert_data(self, response, expected_data):
        body = yield treq.json_content(response)
        self.assertIn('data', body)
        self.assertEqual(body['data'], expected_data)

    @inlineCallbacks
    def assert_sent_header(self, response, header, expected_value):
        body = yield treq.json_content(response)
        self.assertIn(header, body['headers'])
        self.assertEqual(body['headers'][header], expected_value)

    @inlineCallbacks
    def test_get(self):
        response = yield self.get('/get')
        self.assertEqual(response.code, 200)
        yield print_response(response)

    @inlineCallbacks
    def test_get_headers(self):
        response = yield self.get('/get', {'X-Blah': ['Foo', 'Bar']})
        self.assertEqual(response.code, 200)
        yield self.assert_sent_header(response, 'X-Blah', 'Foo, Bar')
        yield print_response(response)

    @inlineCallbacks
    def test_get_302_absolute_redirect(self):
        response = yield self.get(
            '/redirect-to?url={0}/get'.format(self.baseurl))
        self.assertEqual(response.code, 200)
        yield print_response(response)

    @todo_relative_redirect
    @inlineCallbacks
    def test_get_302_relative_redirect(self):
        response = yield self.get('/relative-redirect/1')
        self.assertEqual(response.code, 200)
        yield print_response(response)

    @inlineCallbacks
    def test_get_302_redirect_disallowed(self):
        response = yield self.get('/redirect/1', allow_redirects=False)
        self.assertEqual(response.code, 302)
        yield print_response(response)

    @inlineCallbacks
    def test_head(self):
        response = yield self.head('/get')
        body = yield treq.content(response)
        self.assertEqual('', body)
        yield print_response(response)

    @inlineCallbacks
    def test_head_302_absolute_redirect(self):
        response = yield self.head(
            '/redirect-to?url={0}/get'.format(self.baseurl))
        self.assertEqual(response.code, 200)
        yield print_response(response)

    @todo_relative_redirect
    @inlineCallbacks
    def test_head_302_relative_redirect(self):
        response = yield self.head('/relative-redirect/1')
        self.assertEqual(response.code, 200)
        yield print_response(response)

    @inlineCallbacks
    def test_head_302_redirect_disallowed(self):
        response = yield self.head('/redirect/1', allow_redirects=False)
        self.assertEqual(response.code, 302)
        yield print_response(response)

    @inlineCallbacks
    def test_post(self):
        response = yield self.post('/post', 'Hello!')
#.........这里部分代码省略.........

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class TreqIntegrationTests(TestCase):
    baseurl = HTTPBIN_URL
    get = with_baseurl(treq.get)
    head = with_baseurl(treq.head)
    post = with_baseurl(treq.post)
    put = with_baseurl(treq.put)
    patch = with_baseurl(treq.patch)
    delete = with_baseurl(treq.delete)

    def setUp(self):
        self.pool = HTTPConnectionPool(reactor, False)

    def tearDown(self):
        def _check_fds(_):
            # This appears to only be necessary for HTTPS tests.
            # For the normal HTTP tests then closeCachedConnections is
            # sufficient.
            fds = set(reactor.getReaders() + reactor.getReaders())
            if not [fd for fd in fds if isinstance(fd, Client)]:
                return

            return deferLater(reactor, 0, _check_fds, None)

        return self.pool.closeCachedConnections().addBoth(_check_fds)

    @inlineCallbacks
    def assert_data(self, response, expected_data):
        body = yield treq.json_content(response)
        self.assertIn('data', body)
        self.assertEqual(body['data'], expected_data)

    @inlineCallbacks
    def assert_sent_header(self, response, header, expected_value):
        body = yield treq.json_content(response)
        self.assertIn(header, body['headers'])
        self.assertEqual(body['headers'][header], expected_value)

    @inlineCallbacks
    def test_get(self):
        response = yield self.get('/get')
        self.assertEqual(response.code, 200)
        yield print_response(response)

    @inlineCallbacks
    def test_get_headers(self):
        response = yield self.get('/get', {'X-Blah': ['Foo', 'Bar']})
        self.assertEqual(response.code, 200)
        yield self.assert_sent_header(response, 'X-Blah', 'Foo, Bar')
        yield print_response(response)

    @inlineCallbacks
    def test_get_302_redirect_allowed(self):
        response = yield self.get('/redirect/1')
        self.assertEqual(response.code, 200)
        yield print_response(response)

    @inlineCallbacks
    def test_get_302_redirect_disallowed(self):
        response = yield self.get('/redirect/1', allow_redirects=False)
        self.assertEqual(response.code, 302)
        yield print_response(response)

    @inlineCallbacks
    def test_head(self):
        response = yield self.head('/get')
        body = yield treq.content(response)
        self.assertEqual('', body)
        yield print_response(response)

    @inlineCallbacks
    def test_head_302_redirect_allowed(self):
        response = yield self.head('/redirect/1')
        self.assertEqual(response.code, 200)
        yield print_response(response)

    @inlineCallbacks
    def test_head_302_redirect_disallowed(self):
        response = yield self.head('/redirect/1', allow_redirects=False)
        self.assertEqual(response.code, 302)
        yield print_response(response)

    @inlineCallbacks
    def test_post(self):
        response = yield self.post('/post', 'Hello!')
        self.assertEqual(response.code, 200)
        yield self.assert_data(response, 'Hello!')
        yield print_response(response)

    @inlineCallbacks
    def test_post_headers(self):
        response = yield self.post(
            '/post',
            '{msg: "Hello!"}',
            headers={'Content-Type': ['application/json']}
        )

        self.assertEqual(response.code, 200)
        yield self.assert_sent_header(
            response, 'Content-Type', 'application/json')
        yield self.assert_data(response, '{msg: "Hello!"}')
#.........这里部分代码省略.........

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class WebClient(object):
    """
    Client to communicate with the webservice.
    
    Manages a connection pool, supports using a proxy.
    """
    
    def __init__(self, server, secret, proxy = None):
        """
        :param server: Url of the server.
        :param secret: Passpharse. Only clients with the same secret can interact, 
          even when using the same server.
        :param proxy: URL to the proxy. An empty string or no proxy. `None` to check
          the environment variable `http_proxy`.
        """
        self.server = server
        self.encryption_key = _make_key(secret)
        self.proxy = proxy
        self.pool = HTTPConnectionPool(reactor, persistent=True)
        self.pool.maxPersistentPerHost = 1024
    
    
    def close(self):
        """
        Closes the connection pool.
        """
        return self.pool.closeCachedConnections()
    
    def public_ip(self):
        """
        Returns our IP as it is seen from the server.
        """
        def cb(headers):
            return headers["X-Request-From"]
        url = self._url(0, 0)
        d = httpclient.request("GET", url, return_headers=True, pool=self.pool, proxy=self.proxy)
        
        d.addCallback(lambda headers:headers["X-Request-From"][0])
        #d.addCallback(cb)
        return d
    
    def put(self, key, value):
        """
        Store new key-value pair.
        @return deferred new version
        """
        key = _encrypt_key(self.encryption_key, key)
        value = _encrypt_value(self.encryption_key, value)
        d = self._post_request(key, "JUNGEST", value)
        d.addCallback(lambda r:r["record_version"])
        return d
    
    
    def get(self, key, version, wait=False):
        """
        Returns the value for the given key and version.
        If `wait` is `True` then we wait for the key & version
        to be stored. The deferred can be canceled.
        """
        d = self._get(key, version, wait)
        d.addCallback(lambda r:r["value"])
        return d
                    

    def get_jungest(self, key, wait=False):
        """
        Returns a tuple with the jungest version and value for the given key.
        """
        d = self._get(key, "JUNGEST", wait)
        d.addCallback(lambda r:(r["record_version"], r["value"]))
        return d
    

    def get_oldest(self, key, wait=False):
        """
        Returns a tuple with the oldest version and value for the given key.
        """
        d = self._get(key, "OLDEST", wait)
        d.addCallback(lambda r:(r["record_version"], r["value"]))
        return d
    
    def _url(self, record_id, record_version):
        record_version = str(record_version)
        url = "{base}/rec/{id}/{version}".format(
                    base=self.server,
                    id=urllib.quote(record_id, ''), 
                    version=urllib.quote(record_version, ''))
        return url


    def _get(self, key, version, wait=False):
        
        def make_request():
            d = self._get_request(record_id, version, timeout = timeout)
            d.addCallbacks(got_response, got_failure)
            return d
        
        def got_response(response):
            if "value" in response:
                response["value"] = _decrypt_value(self.encryption_key, response["value"])
#.........这里部分代码省略.........

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class TestEventsEndpoint(AbstractApiTest):

    @inlineCallbacks
    def setUp(self):
        yield super(TestEventsEndpoint, self).setUp()
        self.events_deferred = Deferred()
        self.connection_pool = HTTPConnectionPool(reactor, False)
        self.socket_open_deferred = self.tribler_started_deferred.addCallback(self.open_events_socket)
        self.messages_to_wait_for = 0

    @inlineCallbacks
    def tearDown(self):
        yield self.close_connections()

        # Wait to make sure the HTTPChannel is closed, see https://twistedmatrix.com/trac/ticket/2447
        yield deferLater(reactor, 0.3, lambda: None)

        yield super(TestEventsEndpoint, self).tearDown()

    def on_event_socket_opened(self, response):
        response.deliverBody(EventDataProtocol(self.messages_to_wait_for, self.events_deferred, response))

    def open_events_socket(self, _):
        agent = Agent(reactor, pool=self.connection_pool)
        return agent.request('GET', 'http://localhost:%s/events' % self.session.config.get_http_api_port(),
                             Headers({'User-Agent': ['Tribler ' + version_id]}), None)\
            .addCallback(self.on_event_socket_opened)

    def close_connections(self):
        return self.connection_pool.closeCachedConnections()

    @trial_timeout(20)
    def test_search_results(self):
        """
        Testing whether the event endpoint returns search results when we have search results available
        """
        def verify_search_results(results):
            self.assertEqual(len(results), 2)

        self.messages_to_wait_for = 2

        def send_notifications(_):
            self.session.lm.api_manager.root_endpoint.events_endpoint.start_new_query()

            results_dict = {"keywords": ["test"], "result_list": [('a',) * 10]}
            self.session.notifier.notify(SIGNAL_CHANNEL, SIGNAL_ON_SEARCH_RESULTS, None, results_dict)
            self.session.notifier.notify(SIGNAL_TORRENT, SIGNAL_ON_SEARCH_RESULTS, None, results_dict)

        self.socket_open_deferred.addCallback(send_notifications)

        return self.events_deferred.addCallback(verify_search_results)

    @trial_timeout(20)
    def test_events(self):
        """
        Testing whether various events are coming through the events endpoints
        """
        self.messages_to_wait_for = 21

        def send_notifications(_):
            self.session.lm.api_manager.root_endpoint.events_endpoint.start_new_query()
            results_dict = {"keywords": ["test"], "result_list": [('a',) * 10]}
            self.session.notifier.notify(SIGNAL_TORRENT, SIGNAL_ON_SEARCH_RESULTS, None, results_dict)
            self.session.notifier.notify(SIGNAL_CHANNEL, SIGNAL_ON_SEARCH_RESULTS, None, results_dict)
            self.session.notifier.notify(NTFY_UPGRADER, NTFY_STARTED, None, None)
            self.session.notifier.notify(NTFY_UPGRADER_TICK, NTFY_STARTED, None, None)
            self.session.notifier.notify(NTFY_UPGRADER, NTFY_FINISHED, None, None)
            self.session.notifier.notify(NTFY_WATCH_FOLDER_CORRUPT_TORRENT, NTFY_INSERT, None, None)
            self.session.notifier.notify(NTFY_NEW_VERSION, NTFY_INSERT, None, None)
            self.session.notifier.notify(NTFY_CHANNEL, NTFY_DISCOVERED, None, None)
            self.session.notifier.notify(NTFY_TORRENT, NTFY_DISCOVERED, None, {'a': 'Invalid character \xa1'})
            self.session.notifier.notify(NTFY_TORRENT, NTFY_DELETE, None, {'a': 'b'})
            self.session.notifier.notify(NTFY_TORRENT, NTFY_FINISHED, 'a' * 10, None)
            self.session.notifier.notify(NTFY_TORRENT, NTFY_ERROR, 'a' * 10, 'This is an error message')
            self.session.notifier.notify(NTFY_MARKET_ON_ASK, NTFY_UPDATE, None, {'a': 'b'})
            self.session.notifier.notify(NTFY_MARKET_ON_BID, NTFY_UPDATE, None, {'a': 'b'})
            self.session.notifier.notify(NTFY_MARKET_ON_ASK_TIMEOUT, NTFY_UPDATE, None, {'a': 'b'})
            self.session.notifier.notify(NTFY_MARKET_ON_BID_TIMEOUT, NTFY_UPDATE, None, {'a': 'b'})
            self.session.notifier.notify(NTFY_MARKET_ON_TRANSACTION_COMPLETE, NTFY_UPDATE, None, {'a': 'b'})
            self.session.notifier.notify(NTFY_MARKET_ON_PAYMENT_RECEIVED, NTFY_UPDATE, None, {'a': 'b'})
            self.session.notifier.notify(NTFY_MARKET_ON_PAYMENT_SENT, NTFY_UPDATE, None, {'a': 'b'})
            self.session.notifier.notify(SIGNAL_RESOURCE_CHECK, SIGNAL_LOW_SPACE, None, {})
            self.session.notifier.notify(NTFY_CREDIT_MINING, NTFY_ERROR, None, {"message": "Some credit mining error"})
            self.session.lm.api_manager.root_endpoint.events_endpoint.on_tribler_exception("hi")

        self.socket_open_deferred.addCallback(send_notifications)

        return self.events_deferred

    @trial_timeout(20)
    def test_family_filter_search(self):
        """
        Testing the family filter when searching for torrents and channels
        """
        self.messages_to_wait_for = 2

        def send_searches(_):
            events_endpoint = self.session.lm.api_manager.root_endpoint.events_endpoint

            channels = [['a', ] * 10, ['a', ] * 10]
#.........这里部分代码省略.........

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class TestEventsEndpoint(AbstractApiTest):

    @inlineCallbacks
    def setUp(self):
        yield super(TestEventsEndpoint, self).setUp()
        self.events_deferred = Deferred()
        self.connection_pool = HTTPConnectionPool(reactor, False)
        self.socket_open_deferred = self.tribler_started_deferred.addCallback(self.open_events_socket)
        self.messages_to_wait_for = 0

    @inlineCallbacks
    def tearDown(self):
        yield self.close_connections()

        # Wait to make sure the HTTPChannel is closed, see https://twistedmatrix.com/trac/ticket/2447
        yield deferLater(reactor, 0.3, lambda: None)

        yield super(TestEventsEndpoint, self).tearDown()

    def on_event_socket_opened(self, response):
        response.deliverBody(EventDataProtocol(self.messages_to_wait_for, self.events_deferred, response))

    def open_events_socket(self, _):
        agent = Agent(reactor, pool=self.connection_pool)
        return agent.request(b'GET', 'http://localhost:%s/events' % self.session.config.get_http_api_port(),
                             Headers({'User-Agent': ['Tribler ' + version_id]}), None) \
            .addCallback(self.on_event_socket_opened)

    def close_connections(self):
        return self.connection_pool.closeCachedConnections()

    @trial_timeout(20)
    def test_events(self):
        """
        Testing whether various events are coming through the events endpoints
        """
        self.messages_to_wait_for = 21

        def send_notifications(_):
            self.session.notifier.notify(NTFY_UPGRADER, NTFY_STARTED, None, None)
            self.session.notifier.notify(NTFY_UPGRADER_TICK, NTFY_STARTED, None, None)
            self.session.notifier.notify(NTFY_UPGRADER, NTFY_FINISHED, None, None)
            self.session.notifier.notify(NTFY_WATCH_FOLDER_CORRUPT_TORRENT, NTFY_INSERT, None, None)
            self.session.notifier.notify(NTFY_NEW_VERSION, NTFY_INSERT, None, None)
            self.session.notifier.notify(NTFY_CHANNEL, NTFY_DISCOVERED, None, None)
            self.session.notifier.notify(NTFY_TORRENT, NTFY_DISCOVERED, None, {'a': 'Invalid character \xa1'})
            self.session.notifier.notify(NTFY_TORRENT, NTFY_FINISHED, 'a' * 10, None)
            self.session.notifier.notify(NTFY_TORRENT, NTFY_ERROR, 'a' * 10, 'This is an error message')
            self.session.notifier.notify(NTFY_MARKET_ON_ASK, NTFY_UPDATE, None, {'a': 'b'})
            self.session.notifier.notify(NTFY_MARKET_ON_BID, NTFY_UPDATE, None, {'a': 'b'})
            self.session.notifier.notify(NTFY_MARKET_ON_ASK_TIMEOUT, NTFY_UPDATE, None, {'a': 'b'})
            self.session.notifier.notify(NTFY_MARKET_ON_BID_TIMEOUT, NTFY_UPDATE, None, {'a': 'b'})
            self.session.notifier.notify(NTFY_MARKET_ON_TRANSACTION_COMPLETE, NTFY_UPDATE, None, {'a': 'b'})
            self.session.notifier.notify(NTFY_MARKET_ON_PAYMENT_RECEIVED, NTFY_UPDATE, None, {'a': 'b'})
            self.session.notifier.notify(NTFY_MARKET_ON_PAYMENT_SENT, NTFY_UPDATE, None, {'a': 'b'})
            self.session.notifier.notify(SIGNAL_RESOURCE_CHECK, SIGNAL_LOW_SPACE, None, {})
            self.session.notifier.notify(NTFY_CREDIT_MINING, NTFY_ERROR, None, {"message": "Some credit mining error"})
            self.session.notifier.notify(NTFY_TUNNEL, NTFY_REMOVE, Circuit(1234, None), 'test')
            self.session.notifier.notify(SIGNAL_GIGACHANNEL_COMMUNITY, SIGNAL_ON_SEARCH_RESULTS, None,
                                         {"query": "test", "results": []})
            self.session.lm.api_manager.root_endpoint.events_endpoint.on_tribler_exception("hi")

        self.socket_open_deferred.addCallback(send_notifications)

        return self.events_deferred

# 需要导入模块: from twisted.web.client import HTTPConnectionPool [as 别名]
# 或者: from twisted.web.client.HTTPConnectionPool import closeCachedConnections [as 别名]
class TestScaling(unittest.TestCase):
    def setUp(self):
        self.pool = HTTPConnectionPool(reactor, False)
        self.identity = IdentityV2(
            auth=auth, username=username, password=password,
            endpoint=endpoint, pool=self.pool
        )

    def tearDown(self):
        def _check_fds(_):
            fds = set(reactor.getReaders() + reactor.getReaders())
            if not [fd for fd in fds if isinstance(fd, Client)]:
                return
            return deferLater(reactor, 0, _check_fds, None)
        return self.pool.closeCachedConnections().addBoth(_check_fds)

    def test_scaling_up(self):
        scaling_group_body = autoscale.create_scaling_group_dict(
            image_ref=image_ref, flavor_ref=flavor_ref,
            name="my-group-configuration"
        )

        self.scaling_group = autoscale.ScalingGroup(
            group_config=scaling_group_body,
            pool=self.pool
        )

        self.scaling_policy = autoscale.ScalingPolicy(
            scale_by=2,
            scaling_group=self.scaling_group
        )

        rcs = TestResources()
        d = (
            self.identity.authenticate_user(
                rcs,
                resources={
                    "otter": ("autoscale", "http://localhost:9000/v1.0/{0}"),
                    "loadbalancers": ("cloudLoadBalancers",),
                },
                region=region,
            ).addCallback(print_token_and_ep)
            .addCallback(self.scaling_group.start, self)
            .addCallback(dump_groups)
            .addCallback(self.scaling_policy.start, self)
            .addCallback(self.scaling_policy.execute)
            .addCallback(
                self.scaling_group.wait_for_N_servers, 2, timeout=1800
            ).addCallback(
                lambda _: self.scaling_group.get_scaling_group_state(rcs)
            ).addCallback(dump_state)
        )
        return d
    test_scaling_up.timeout = 1800

    def test_scaling_down(self):
        """
        Verify that a basic scale down operation completes as expected.
        """
        scaling_group_body = autoscale.create_scaling_group_dict(
            image_ref=image_ref, flavor_ref=flavor_ref,
            name="tr-scaledown-conf",
        )

        self.scaling_group = autoscale.ScalingGroup(
            group_config=scaling_group_body,
            pool=self.pool
        )

        self.scaling_policy_up_2 = autoscale.ScalingPolicy(
            scale_by=2,
            scaling_group=self.scaling_group
        )
        self.scaling_policy_down_1 = autoscale.ScalingPolicy(
            scale_by=-1,
            scaling_group=self.scaling_group
        )

        rcs = TestResources()
        d = (
            self.identity.authenticate_user(
                rcs,
                resources={
                    "otter": ("autoscale", "http://localhost:9000/v1.0/{0}"),
                    "loadbalancers": ("cloudLoadBalancers",),
                },
                region=region
            ).addCallback(print_token_and_ep)
            .addCallback(self.scaling_group.start, self)
            .addCallback(self.scaling_policy_up_2.start, self)
            .addCallback(self.scaling_policy_up_2.execute)
            .addCallback(self.scaling_group.wait_for_N_servers,
                         2, timeout=1800)
            .addCallback(
                lambda _: self.scaling_group.get_scaling_group_state(rcs))
            .addCallback(dump_state)
            .addCallback(lambda _: rcs)
            .addCallback(self.scaling_policy_down_1.start, self)
            .addCallback(self.scaling_policy_down_1.execute)
            .addCallback(self.scaling_group.wait_for_N_servers,
#.........这里部分代码省略.........