Python sessions.Session类代码示例

 def __init__(self, test, test_result, dns_resolve=True):
     _Session.__init__(self)
     self.verify = not os.getenv('BYPASS_SSL_CHECK')
     self.test = test
     self.test_result = test_result
     self.loads_status = None, None, None, None
     self.dns_resolve = dns_resolve

    def test_session_persistent_params(self):

        params = {'a': 'a_test'}

        s = Session()
        s.params = params

        # Make 2 requests from Session object, should send header both times
        r1 = s.get(httpbin('get'))
        assert params['a'] in r1.content


        params2 = {'b': 'b_test'}

        r2 = s.get(httpbin('get'), params=params2)
        assert params['a'] in r2.content
        assert params2['b'] in r2.content


        params3 = {'b': 'b_test', 'a': None, 'c': 'c_test'}

        r3 = s.get(httpbin('get'), params=params3)

        assert not params['a'] in r3.content
        assert params3['b'] in r3.content
        assert params3['c'] in r3.content

 def curl(self, method, endpoint, params=None):
     url = '{scheme}://{host}{endpoint}'.format(scheme=self.SCHEME, host=self.HOST, endpoint=endpoint)
     params = params or {}
     session = Session()
     request = Request(method, url, params=params)
     request = request.prepare()
     request.headers.update({
         'X-Application-Key': self.KEY,
     })
     response = session.send(request)
     return GforceResponse(response)

    def test_session_persistent_headers(self):

        heads = {'User-agent': 'Mozilla/5.0'}

        s = Session()
        s.headers = heads
        # Make 2 requests from Session object, should send header both times
        r1 = s.get(httpbin('user-agent'))

        assert heads['User-agent'] in r1.content
        r2 = s.get(httpbin('user-agent'))

        assert heads['User-agent'] in r2.content
        self.assertEqual(r2.status_code, 200)

class MeiPai:
    def __init__(self):
        self.session = Session()
        self.session.headers = headers

    def search(self, query, search_type='mv', page=1):
        query = query.strip()
        # topic
        if query.startswith('#'):
            topic = query.strip('#')
            return self.get_videos_by_topic(topic)
        cache_exists, result = check_cache_and_return_result(query=query, search_type=search_type, page=page)
        if cache_exists:
            return result
        url = 'http://www.meipai.com/search/{search_type}?'.format(search_type=search_type) + \
              urlencode({'q': query, 'page': page})
        resp = self.session.request('GET', url)
        html = BeautifulSoup(resp.content, 'html.parser')
        if search_type == 'mv':
            video_links = [div.attrs['data-video'].strip() for div in html.find_all(class_='content-l-video')]

            # associated_words = self.word_association(query)
            # print("你是否还想搜索：" + ",".join(associated_words))
            result = video_links
        elif search_type == 'topic':
            result = [div.text.strip().strip('#') for div in html.find_all(class_='tcard-name')]
        else:
            result = []

        cache_search_result(query, search_type, page, result)
        return result

    def get_videos_by_topic(self, topic_name):
        """
        get top videos by topic
        :param topic_name:
        :return:
        """
        topic = Topic(topic_name)
        topic_id = topic.topic_id
        url = "http://www.meipai.com/topics/hot_timeline?page=1&count=24&tid={topic_id}".format(topic_id=topic_id)
        resp = self.session.request('GET', url)
        result = json.loads(resp.text)
        return [media['video'] for media in result['medias']]

    # get associated words
    def word_association(self, word):
        url = 'http://www.meipai.com/search/word_assoc?' + urlencode({'q': word})
        resp = self.session.request('GET', url)
        return json.loads(resp.text)

class EMDRUploader(Thread):
    def __init__(self, statsCollector):
        Thread.__init__(self)
        self._queue = Queue()
        self.setDaemon(True)
        self._session = Session()
        self._session.headers.update({
            "User-Agent": "CRESTMarketTrawler/{0} ([email protected])".format(VERSION)
        })
        self._pool = Pool(size=10)
        self.statsCollector = statsCollector

    def notify(self, regionID, typeID, orders):
        self._queue.put((timestampString(), regionID, typeID, orders))
        self.statsCollector.tally("emdr_send_queued")
        queueSize = self._queue.qsize()
        self.statsCollector.datapoint("emdr_queue_size", queueSize)
        if queueSize > 100:
            logger.error("EMDR submit queue is about {0} items long!".format(queueSize))
        elif queueSize > 10:
            logger.warn("EMDR submit queue is about {0} items long!".format(queueSize))

    def run(self):
        def submit(generationTime, regionID, typeID, orders):
            uudif = json.dumps(EMDROrdersAdapter(generationTime, regionID, typeID, orders))
            res = self._session.post("http://upload.eve-emdr.com/upload/", data=uudif)
            self.statsCollector.tally("emdr_sent")
            if res.status_code != 200:
                logger.error("Error {0} submitting to EMDR: {1}".format(res.status_code, res.content))
                self.statsCollector.tally("emdr_errored")

        while True:
            (generationTime, regionID, typeID, orders) = self._queue.get()
            self._pool.spawn(submit, generationTime, regionID, typeID, orders)

def fetch_data(dl_type=None, **kwargs):
    """ 
    Fetch Receipts, Expenditures, and Committees. 
    dl_type is one of those three choices. 
    kwargs depend on the choice. 
    Receipts and Expenditures need start_date and end_date for search.
    Committees need a name_start kwarg to pass into the search.
    
    Seems like the maximum that you can get is about 250,000 records at a time.
    """
    s = Session()
    post_data = {
        '__EVENTTARGET': 'ctl00$ContentPlaceHolder1$btnText',
        'ctl00$pnlMenu_CollapsiblePanelExtender_ClientState': 'true',
        'ctl00$AccordionStateBoardMenu_AccordionExtender_ClientState': '0',
        'ctl00$mtbSearch': '',
        'ctl00$AccordionPaneStateBoardMenu_content$AccordionMainContent_AccordionExtender_ClientState': '-1',
        'hiddenInputToUpdateATBuffer_CommonToolkitScripts': '1',
        '__EVENTARGUMENT': '',
        '__VIEWSTATEGENERATOR': 'E8D1F59A'
    }
    if dl_type == 'Receipts':
        CONT_GET_PARAMS['RcvDate'] = kwargs['start_date']
        CONT_GET_PARAMS['RcvDateThru'] = kwargs['end_date']
        url = '%s/DownloadList.aspx?%s' % (BASE_URL, urlencode(CONT_GET_PARAMS))
    elif dl_type == 'Committees':
        COMM_GET_PARAMS['Name'] = kwargs['name_start']
        url = '%s/DownloadList.aspx?%s' % (BASE_URL, urlencode(COMM_GET_PARAMS))
    elif dl_type == 'Expenditures':
        EXP_GET_PARAMS['ExpendedDate'] = kwargs['start_date']
        EXP_GET_PARAMS['ExpendedDateThru'] = kwargs['end_date']
        url = '%s/DownloadList.aspx?%s' % (BASE_URL, urlencode(EXP_GET_PARAMS))
    elif dl_type == 'Candidates':
        url = 'http://www.elections.state.il.us/ElectionInformation/CandDataFile.aspx?id=%s' % kwargs['election_id']
    g = s.get(url)
    if 'Unexpected errors occurred trying to populate page' in g.content:
        return None
    soup = BeautifulSoup(g.content)
    view_state = soup.find('input', attrs={'id': '__VIEWSTATE'}).get('value')
    event_val = soup.find('input', attrs={'id': '__EVENTVALIDATION'}).get('value')
    post_data['__VIEWSTATE'] = view_state
    post_data['__EVENTVALIDATION'] = event_val
    dl_page = s.post(url, data=post_data)
    if dl_page.status_code == 200:
        return dl_page.content
    else:
        return None

    def __init__(self, device_key):
        self.base_url = '{0}/{1}/'.format(self.API_URL, self.API_VERSION)

        self.session = Session()
        self.session.auth = KeyAuth(device_key)
        self.session.headers.update({
            'Content-Type': 'application/json'
        })

 def delete(self, url, **kwargs):
     return Session.delete(
         self,
         self._get_resource_uri(url),
         **self._set_default_timeout(
             **kwargs
         )
     )

def configure_http_session(size=20, max_retries=1, _session=None):
    """
    Return a :class:`requests.Session` object configured with
    a :class:`requests.adapters.HTTPAdapter` (connection pool)
    for http and https connections.

    :param size: The connection pool and maximum size.
    :type size: int

    :param max_retries: The maximum number of retries for each connection.
    :type max_retries: int

    :param _session: Test-only hook to provide a pre-configured session.
    """
    if _session is not None:
        return _session

    adapter = HTTPAdapter(
        pool_connections=size,
        pool_maxsize=size,
        max_retries=max_retries,
    )
    session = Session()
    session.mount("http://", adapter)
    session.mount("https://", adapter)
    session.max_redirects = 1
    session.verify = certifi.where()
    return session

 def patch(self, url, data=None, **kwargs):
     return Session.patch(
         self,
         self._get_resource_uri(url),
         data,
         **self._set_default_timeout(
             **kwargs
         )
     )

 def send(self, request, **kwargs):
     request.url = resolve(request.url)
     # started
     start = datetime.datetime.utcnow()
     res = _Session.send(self, request, **kwargs)
     res.started = start
     res.method = request.method
     _measure(res)
     return res

class APIClient(BaseAPIClient):
    verify = True
    base_url = None

    def __init__(self, *args, **kwargs):
        self.session = Session()
        spec = self.call(SpecEndpoint())
        super(APIClient, self).__init__(*args, spec=spec, **kwargs)

    def make_request(self, endpoint, request):
        request.url = self.base_url + request.url
        prepared = self.session.prepare_request(request)
        return self.session.send(prepared,
                                 stream=False,
                                 timeout=None,
                                 verify=self.verify,
                                 cert=None,
                                 proxies={},
                                 allow_redirects=True)

 def __init__(self, statsCollector):
     Thread.__init__(self)
     self._queue = Queue()
     self.setDaemon(True)
     self._session = Session()
     self._session.headers.update({
         "User-Agent": "CRESTMarketTrawler/{0} ([email protected])".format(VERSION)
     })
     self._pool = Pool(size=10)
     self.statsCollector = statsCollector

 def __init__(self, statsCollector):
     Thread.__init__(self)
     self._queue = Queue(EMDR_QUEUE_SIZE)
     self.setDaemon(True)
     self._session = Session()
     self._session.headers.update({
         "User-Agent": USER_AGENT_STRING
     })
     self._pool = Pool(size=10)
     self.statsCollector = statsCollector