Python CacheControl.get方法代码示例

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
    def test_not_modified_releases_connection(self, server):
        sess = CacheControl(requests.Session())
        etag_url = urljoin(server.application_url, "/etag")
        sess.get(etag_url)

        resp = Mock(status=304, headers={})

        # This is how the urllib3 response is created in
        # requests.adapters
        response_mod = "requests.adapters.HTTPResponse.from_httplib"

        with patch(response_mod, Mock(return_value=resp)):
            sess.get(etag_url)
            assert resp.read.called
            assert resp.release_conn.called

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
class TestStream(object):

    def setup(self):
        self.sess = CacheControl(requests.Session())

    def test_stream_is_cached(self, url):
        resp_1 = self.sess.get(url + 'stream')
        content_1 = resp_1.content

        resp_2 = self.sess.get(url + 'stream')
        content_2  = resp_1.content

        assert not resp_1.from_cache
        assert resp_2.from_cache
        assert content_1 == content_2

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
class JSONLocator(Locator):
    def __init__(self, url=PYPI_JSON_URL):
        self.url = url
        self.session = CacheControl(requests.session())
    def versions(self, distribution):
        url = "{}/{}/json".format(self.url, distribution)
        response = self.session.get(url)
        ret = []
        j = response.json()['releases']
        return [v for v, d in j.items() if len(d) > 0]
    def get(self, distribution, version):
        url = "{}/{}/json".format(self.url, distribution)
        response = self.session.get(url)
        # Reformat the data...
        return response.json()['releases'][version]

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
def all_sites(sitemap_url='http://library.link/harvest/sitemap.xml'):
    '''
    >>> from librarylink.util import all_sites
    >>> [ s.host for s in all_sites() if 'denverlibrary' in s.host ]
    ['link.denverlibrary.org']
    '''
    #FIXME: Avoid accumulating all the nodes, which will require improvements to xml.treesequence
    @coroutine
    def sink(accumulator):
        while True:
            e = yield
            loc = next(select_name(e, 'loc'))
            lastmod = next(select_name(e, 'lastmod'))
            s = liblink_site()
            s.sitemap = loc.xml_value
            s.url, _, tail = s.sitemap.partition('harvest/sitemap.xml')
            s.base_url = s.url #Legacy property name
            #Early warning for funky URLs breaking stuff downstream
            assert not tail
            protocol, s.host, path, query, fragment = iri.split_uri_ref(s.sitemap)
            s.lastmod = lastmod.xml_value
            accumulator.append(s)

    nodes = []
    ts = xml.treesequence(('sitemapindex', 'sitemap'), sink(nodes))
    if hasattr (all_sites, 'cachedir'):
        sess = CacheControl(requests.Session(), cache=FileCache(all_sites.cachedir))
    else:
        sess = CacheControl(requests.Session())
    result = sess.get(sitemap_url)
    ts.parse(result.text)
    yield from nodes

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
class LDClient(object):

    def __init__(self, apiKey, config=Config.default()):
        self._apiKey = apiKey
        self._config = config
        self._session = CacheControl(requests.Session())

    def get_flag(self, key, user, default=False):
        try:
            return self._get_flag(key, user, default)
        except:
            logging.exception('Unhandled exception in get_flag. Returning default value for flag.')
            return default

    def _get_flag(self, key, user, default):
        hdrs = {'Authorization': 'api_key ' + self._apiKey,
             'User-Agent': 'PythonClient/' + __version__}
        uri = self._config._base_uri + '/api/eval/features/' + key
        r = self._session.get(uri, headers=hdrs, timeout = (self._config._connect, self._config._read))
        dict = r.json()
        val = _evaluate(dict, user)
        if val is None:
            return default
        else:
            return val

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
    def get_reader(self):
        sess = CacheControl(requests.Session(),
                            cache=FileCache(gettempdir()))
        req = sess.get(self.file)

        # if the response is not 200, an exception will be raised
        req.raise_for_status()

        return io.BufferedReader(io.BytesIO(req.content))

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
class TestHeuristicWith3xxResponse(object):

    def setup(self):

        class DummyHeuristic(BaseHeuristic):

            def update_headers(self, resp):
                return {"x-dummy-header": "foobar"}

        self.sess = CacheControl(Session(), heuristic=DummyHeuristic())

    def test_heuristic_applies_to_301(self, url):
        the_url = url + "permanent_redirect"
        resp = self.sess.get(the_url)
        assert "x-dummy-header" in resp.headers

    def test_heuristic_applies_to_304(self, url):
        the_url = url + "conditional_get"
        resp = self.sess.get(the_url)
        assert "x-dummy-header" in resp.headers

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
class reQuiver(object):

    def __init__(self):
        self._raw_endpoint = "http://quiver.archerdx.com/results?query="
        self._sesh = CacheControl(requests.Session())

    def query(self, query):

        if len(query) == 0:
            raise EmptyQueryStringException()

        q_string = self._raw_endpoint + str(query)
        response = self._sesh.get(q_string)

        if response.status_code != 200:
            raise NetworkErrorException(response.status_code)

        soup = BeautifulSoup(response.content, "html.parser")

        # parse the panels
        panels = soup.find(panel_table_filter)
        panels_list  = []
        
        if panels is not None:
            for row in panels.find_all("tr"):
                cells = row.find_all("td")
                
                if len(cells) == 2:
                    link = cells[0].a['href']
                    genes = [clean_string(gene) for gene in cells[1].string.split()]
                    panels_list.append(QuiverFushionPlexPanel(link, genes))

        # parse the fusions
        fusions = soup.find_all(fusion_table_filter)
        fusions_list = []

        if fusions is not None:
            for fusion in fusions:
                table = fusion.find('table')
                for row in table.find_all('tr'):
                    cells = row.find_all('td')
                    if len(cells) != 2:

                        # get the link
                        link = cells[0].a['href']
                        original_annotation = clean_string(cells[1].string)
                        disease = cells[2].string.strip()
                        pubmed_link = cells[3].a['href']
                        evidence_count = int(cells[4].string)

                        fusions_list.append(QuiverGeneFushion(link, original_annotation, disease,
                                            pubmed_link, evidence_count))

        return QuiverResultSet(panels_list, fusions_list, query)

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
def get_cached_session():
    sess = CacheControl(requests.Session(),
                    cache=FileCache(CACHE_DIR), heuristic=LastModifiedNoDate(require_date=False))

    original_get = sess.get
    def wrapped_get(*args, **kwargs):
        try:
            return original_get(*args, **kwargs)
        except (OSError, IOError) as e:
            return requests.get(*args, **kwargs)
    sess.get = wrapped_get
    return sess

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=None, json=False):
    """
    Returns a byte-string retrieved from the url provider.
    """

    # request session
    cache_dir = sickbeard.CACHE_DIR or _getTempDir()
    session = CacheControl(sess=session, cache=caches.FileCache(os.path.join(cache_dir, "sessions")))

    # request session headers
    req_headers = {"User-Agent": USER_AGENT, "Accept-Encoding": "gzip,deflate"}
    if headers:
        req_headers.update(headers)
    session.headers.update(req_headers)

    # request session ssl verify
    session.verify = False

    # request session paramaters
    session.params = params

    try:
        # Remove double-slashes from url
        parsed = list(urlparse.urlparse(url))
        parsed[2] = re.sub("/{2,}", "/", parsed[2])  # replace two or more / with one
        url = urlparse.urlunparse(parsed)

        # request session proxies
        if sickbeard.PROXY_SETTING:
            logger.log("Using proxy for url: " + url, logger.DEBUG)
            session.proxies = {"http": sickbeard.PROXY_SETTING, "https": sickbeard.PROXY_SETTING}

        # decide if we get or post data to server
        if post_data:
            resp = session.post(url, data=post_data, timeout=timeout)
        else:
            resp = session.get(url, timeout=timeout)

        if not resp.ok:
            logger.log(
                u"Requested url "
                + url
                + " returned status code is "
                + str(resp.status_code)
                + ": "
                + clients.http_error_code[resp.status_code],
                logger.DEBUG,
            )
            return

    except requests.exceptions.HTTPError, e:
        logger.log(u"HTTP error " + str(e.errno) + " while loading URL " + url, logger.WARNING)
        return

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
def fetch_file(url, encoding=None):
    s = requests.session()
    s = CacheControl(s, cache=FileCache(os.path.expanduser('~/.tst/cache')))

    try:
        response = s.get(url, headers={})
    except requests.ConnectionError:
        _assert(False, "Connection failed... check your internet connection")

    _assert(response.ok, "%s\nFile request failed: %s (%d)" % (url, response.reason, response.status_code))
    if encoding:
        response.encoding = encoding

    return response.text

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
class TwistedHttpFeatureRequester(FeatureRequester):

    def __init__(self, api_key, config):
        self._api_key = api_key
        self._session = CacheControl(txrequests.Session())
        self._config = config

    def get(self, key, callback):
        d = self.toggle(key)
        d.addBoth(callback)
        return d

    def toggle(self, key):
        @defer.inlineCallbacks
        def run(should_retry):
            # noinspection PyBroadException
            try:
                val = yield self._toggle(key)
                defer.returnValue(val)
            except ProtocolError as e:
                inner = e.args[1]
                if inner.errno == errno.ECONNRESET and should_retry:
                    log.warning(
                        'ProtocolError exception caught while getting flag. Retrying.')
                    d = yield run(False)
                    defer.returnValue(d)
                else:
                    log.exception(
                        'Unhandled exception. Returning default value for flag.')
                    defer.returnValue(None)
            except Exception:
                log.exception(
                    'Unhandled exception. Returning default value for flag.')
                defer.returnValue(None)

        return run(True)

    @defer.inlineCallbacks
    def _toggle(self, key):
        hdrs = _headers(self._api_key)
        uri = self._config.base_uri + '/api/eval/features/' + key
        r = yield self._session.get(uri, headers=hdrs, timeout=(self._config.connect, self._config.read))
        r.raise_for_status()
        feature = r.json()
        defer.returnValue(feature)

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
class TestHeuristicWithoutWarning(object):

    def setup(self):

        class NoopHeuristic(BaseHeuristic):
            warning = Mock()

            def update_headers(self, resp):
                return {}

        self.heuristic = NoopHeuristic()
        self.sess = CacheControl(Session(), heuristic=self.heuristic)

    def test_no_header_change_means_no_warning_header(self, url):
        the_url = url + "optional_cacheable_request"
        resp = self.sess.get(the_url)

        assert not self.heuristic.warning.called

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
def _get_filehandle(filepath_or, *args, **kwargs):
    """Open file if `filepath_or` looks like a string/unicode/bytes, else
    pass through.
    """
    if _is_string_or_bytes(filepath_or):
        if requests.compat.urlparse(filepath_or).scheme in {'http', 'https'}:
            sess = CacheControl(requests.Session(),
                                cache=FileCache(gettempdir()))
            req = sess.get(filepath_or, **kwargs)

            # if the response is not 200, an exception will be raised
            req.raise_for_status()

            fh, own_fh = BytesIO(req.content), True
        else:
            fh, own_fh = open(filepath_or, *args, **kwargs), True
    else:
        fh, own_fh = filepath_or, False
    return fh, own_fh

# 需要导入模块: from cachecontrol import CacheControl [as 别名]
# 或者: from cachecontrol.CacheControl import get [as 别名]
def Request(
	url,
	method="GET",
	headers=DEFAULT_HEADERS,
	additional_headers=None,
	data=None,
	session=None,
	allow_redirects=True,
	timeout=10,
	load_cookies=True,
	mobile=False
):
	if additional_headers:
		headers.update(additional_headers)
	try:
		session = CacheControl(session)
	except Exception as e:
		pass
		# Error("Init web cache failed!!!", e)
	if mobile:
		headers["User-Agents"] = MOBILE_IOS_AGENTS
	xbmc.log("Requests headers: {0}".format(json.dumps(headers)), 1)
	if session:
		session.headers.update(headers)
		domain = re.search("https*\://(.+?)($|/)", url).group(1)
		if load_cookies:
			LoadCookies(session, cookies_name=domain)
		if data:
			response = session.post(url, data=data, allow_redirects=allow_redirects, timeout=timeout, verify=False)
		else:
			if method == "HEAD":
				response = session.head(url, allow_redirects=allow_redirects, timeout=timeout, verify=False)
			else:
				response = session.get(url, allow_redirects=allow_redirects, timeout=timeout, verify=False)
		response.encoding = "utf8"
		SaveCookies(session, cookies_name=domain)
		return response
	else:
		if method == "HEAD":
			return requests.head(url, headers=headers, allow_redirects=allow_redirects, timeout=timeout, verify=False)
		else:
			return requests.get(url, headers=headers, allow_redirects=allow_redirects, timeout=timeout, verify=False)