Python grequests.map方法代码示例

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def _get_async(self, keys, frmt='json', params={}):
        # does not work under pyhon3 so local import
        import grequests
        session = self._get_session()
        try:
            # build the requests
            urls = self._get_all_urls(keys, frmt)
            self.logging.debug("grequests.get processing")
            rs = (grequests.get(url, session=session, params=params)  for key,url in zip(keys, urls))
            # execute them
            self.logging.debug("grequests.map call")
            ret = grequests.map(rs, size=min(self.settings.CONCURRENT, len(keys)))
            self.last_response = ret
            self.logging.debug("grequests.map call done")
            return ret
        except Exception as err:
            self.logging.warning("Error caught in async. " + err.message)
            return [] 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def get_image_urls(girl_urls):

    girl_list = []

    # 建立5个并发连接
    rs = (grequests.get(url) for url in girl_urls)
    responses = grequests.map(rs, size = 5)

    for response in responses:
        parsed_body = html.fromstring(response.text)
        girl_title  = parsed_body.xpath('//title/text()')
        image_urls = parsed_body.xpath('//li[@class="slide "]/img/@src | //li[@class="slide "]/img/@delay')

        # print image_urls
        girl_dict = {girl_title[0] : image_urls}
        girl_list.append(girl_dict)
    
    print "get_girl_urls done!!!"
    return girl_list 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def filter_projects_by_license(projects, headers, licenses):
    import grequests

    reqs = [grequests.get(constants.REPO_URL.format(full_name=p.full_name), headers=headers)
            for p in projects]
    resps = grequests.map(reqs, exception_handler=request_exception_handler)

    filtered_projects = []
    for i, project in enumerate(projects):
        resp = resps[i]
        if not resp or resp.status_code != 200:
            logging.warning("ignoring %s because no info could be fetched", project.full_name)
            continue

        project_license = resp.json().get("license")
        if not project_license or not project_license.get("spdx_id"):
            continue
        license_id = project_license.get("spdx_id")
        if license_id in licenses:
            project.license = license_id
            filtered_projects.append(project)
    return filtered_projects 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def exploitpoc(self):
        for poctype in [
            'cms',
            'system',
            'industrial',
            'information',
                'hardware']:
            tasks = [
                grequests.post(
                    "http://tools.hexlt.org/api/" +
                    poctype,
                    json={
                        "url": self.url,
                        "type": type}) for type in self.poclist[poctype]]
            res = grequests.map(tasks, size=30)
            for i in res:
                result = i.json()
                if result['status']:
                    self.result.append(result['pocresult'])
        return self.result  # exp利用成功以列表形式返回结果,否则返回空列表 [] 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def get_url(url, abort_on_error=False, is_json=True, fetch_timeout=5, auth=None, post_data=None):
    """
    @param post_data: If not None, do a POST request, with the passed data (which should be in the correct string format already)
    """
    headers = {'Connection': 'close', }  # no keepalive
    if auth:
        # auth should be a (username, password) tuple, if specified
        headers['Authorization'] = http_basic_auth_str(auth[0], auth[1])

    try:
        if post_data is not None:
            if is_json:
                headers['content-type'] = 'application/json'
            r = grequests.map((grequests.post(url, data=post_data, timeout=fetch_timeout, headers=headers, verify=False),))[0]
        else:
            r = grequests.map((grequests.get(url, timeout=fetch_timeout, headers=headers, verify=False),))[0]
        if r is None:
            raise Exception("result is None")
    except Exception as e:
        raise Exception("Got get_url request error: %s" % e)
    else:
        if r.status_code != 200 and abort_on_error:
            raise Exception("Bad status code returned: '%s'. result body: '%s'." % (r.status_code, r.text))
    return r.json() if r.text and is_json else r.text 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def get_volume(self):
        """

        Args:

        Returns:
          pandas.DataFrame: pandas.DataFrame with volumes of symbols

        """
        dfs = map(self.iex_get_volume, [[batch] for batch in self.batches])
        self.volume = pd.concat(dfs)
        # with Pool() as pool:
        #     self.volume = pd.concat(
        #         pool.starmap(self.iex_get_volume, [
        #                      [batch] for batch in self.batches])
        #     ) 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def get_responses(payloads):
        """

        Args:
          payloads(list): list of payloads for GET request

        Returns:
          list: list of dictionaries with data from JSON responses

        """
        batch_url = "{base}stock/market/batch?".format(base=iex_url_base)
        rs = (grequests.get(batch_url, params=payload) for payload in payloads)
        result = grequests.map(rs)
        try:
            outputs = [r.json() for r in result]
        except AttributeError:
            outputs = []
        return outputs 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def get_girl_urls(page_urls):

    girl_urls = []

    # 采用grequests，建立5个并发连接
    rs = (grequests.get(url) for url in page_urls)
    responses = grequests.map(rs, size = 5)
    
    for response in responses:
        parsed_body = html.fromstring(response.text)
        girl = parsed_body.xpath('//div[@class="grid_title"]/a/@href')
        girl_urls.extend(girl)
    
    return girl_urls 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def get_images(girl_list):

    count = 1
    # 图片的默认存储目录
    start_dir = '/home/pein/Pictures/'

    for girl in girl_list:
        dir_name = start_dir + girl.keys()[0]
        urls = girl.values()[0]

        if not os.path.exists(dir_name):
            os.makedirs(dir_name)

        rs = (grequests.get(url) for url in urls)
        responses = grequests.map(rs)

        image_dict = dict(zip(urls, responses))
        for url in image_dict:
            print url
            with open(dir_name + '/' + url.split('/')[-1], 'wb') as f:
                r = image_dict[url]
                f.write(r.content)

        print
        print count, girl.keys()[0] + "  done!!!"

        count += 1
        print 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def _read_tenth_batch(pathurls):
        import grequests
        urls = [grequests.get(ILSVRCTenth._tenthpath(pathurl)) for pathurl in pathurls]
        resps = grequests.map(urls)
        result_dict = {}
        for url, resp in zip(pathurls, resps):
            if not resp or resp.status_code // 100 != 2:
                continue
            result_dict[url] = resp.content
        return result_dict 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def send_request(self, rs):
        responses = grequests.map(rs)
        for res in responses:
            if not res:
                print responses
                raise Exception
        return responses 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def run_requests(n, batch_size):
    wav = open('resources/test.wav', 'rb').read()
    requests = [make_request(id, wav) for id in range(0, n)]

    return grequests.map(requests, size=batch_size) 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def test_001_test(self):
        """ Test hitting the 'test' endpoint of OpenDCRE in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/test'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def test_002_test_read(self):
        """ Test reading a voltage sensor in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/read/voltage/rack_1/40000000/0666'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 500) 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import map [as 别名]
def test_003_test_read(self):
        """ Test reading a fan speed sensor in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/read/fan_speed/rack_1/40000000/0666'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 500)