Python grequests.get方法代码示例

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def _get_async(self, keys, frmt='json', params={}):
        # does not work under pyhon3 so local import
        import grequests
        session = self._get_session()
        try:
            # build the requests
            urls = self._get_all_urls(keys, frmt)
            self.logging.debug("grequests.get processing")
            rs = (grequests.get(url, session=session, params=params)  for key,url in zip(keys, urls))
            # execute them
            self.logging.debug("grequests.map call")
            ret = grequests.map(rs, size=min(self.settings.CONCURRENT, len(keys)))
            self.last_response = ret
            self.logging.debug("grequests.map call done")
            return ret
        except Exception as err:
            self.logging.warning("Error caught in async. " + err.message)
            return [] 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def get_image_urls(girl_urls):

    girl_list = []

    # 建立5个并发连接
    rs = (grequests.get(url) for url in girl_urls)
    responses = grequests.map(rs, size = 5)

    for response in responses:
        parsed_body = html.fromstring(response.text)
        girl_title  = parsed_body.xpath('//title/text()')
        image_urls = parsed_body.xpath('//li[@class="slide "]/img/@src | //li[@class="slide "]/img/@delay')

        # print image_urls
        girl_dict = {girl_title[0] : image_urls}
        girl_list.append(girl_dict)
    
    print "get_girl_urls done!!!"
    return girl_list 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def get_data(self):
        idxs = np.arange(len(self.train_list))
        if self.shuffle:
            self.rng.shuffle(idxs)

        caches = {}
        for i, k in enumerate(idxs):
            path = self.train_list[k]
            label = self.lb_list[k]

            if i % self.preload == 0:
                try:
                    caches = ILSVRCTenth._read_tenth_batch(self.train_list[idxs[i:i+self.preload]])
                except Exception as e:
                    logging.warning('tenth local cache failed, err=%s' % str(e))

            content = caches.get(path, '')
            if not content:
                content = ILSVRCTenth._read_tenth(path)

            img = cv2.imdecode(np.fromstring(content, dtype=np.uint8), cv2.IMREAD_COLOR)
            yield [img, label] 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def filter_projects_by_license(projects, headers, licenses):
    import grequests

    reqs = [grequests.get(constants.REPO_URL.format(full_name=p.full_name), headers=headers)
            for p in projects]
    resps = grequests.map(reqs, exception_handler=request_exception_handler)

    filtered_projects = []
    for i, project in enumerate(projects):
        resp = resps[i]
        if not resp or resp.status_code != 200:
            logging.warning("ignoring %s because no info could be fetched", project.full_name)
            continue

        project_license = resp.json().get("license")
        if not project_license or not project_license.get("spdx_id"):
            continue
        license_id = project_license.get("spdx_id")
        if license_id in licenses:
            project.license = license_id
            filtered_projects.append(project)
    return filtered_projects 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def get_url(url, abort_on_error=False, is_json=True, fetch_timeout=5, auth=None, post_data=None):
    """
    @param post_data: If not None, do a POST request, with the passed data (which should be in the correct string format already)
    """
    headers = {'Connection': 'close', }  # no keepalive
    if auth:
        # auth should be a (username, password) tuple, if specified
        headers['Authorization'] = http_basic_auth_str(auth[0], auth[1])

    try:
        if post_data is not None:
            if is_json:
                headers['content-type'] = 'application/json'
            r = grequests.map((grequests.post(url, data=post_data, timeout=fetch_timeout, headers=headers, verify=False),))[0]
        else:
            r = grequests.map((grequests.get(url, timeout=fetch_timeout, headers=headers, verify=False),))[0]
        if r is None:
            raise Exception("result is None")
    except Exception as e:
        raise Exception("Got get_url request error: %s" % e)
    else:
        if r.status_code != 200 and abort_on_error:
            raise Exception("Bad status code returned: '%s'. result body: '%s'." % (r.status_code, r.text))
    return r.json() if r.text and is_json else r.text 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def parse_and_print_data(data, metric, factors=None, filter=None,
                         template=None, template_params=None):
    if template_params is None:
        template_params = {}

    if not factors:
        factors = {'Tbps': 1000000000000, 'Gbps': 1000000000, 'Mbps': 1000000,
                   'Kbps': 1000, 'Bps': 1}

    factor = parse_factor(
        data.get('meta').get('yValueFormatString'),
        factors
    )
    for traffic in data.get('data'):
        timestamp = int(traffic.get('x') / 1000)

        if not filter(timestamp):
            continue

        value = abs(traffic.get('y')) * factor
        print(template.format(
            metric=metric, value=value, time=timestamp, **template_params)
        ) 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def get_responses(payloads):
        """

        Args:
          payloads(list): list of payloads for GET request

        Returns:
          list: list of dictionaries with data from JSON responses

        """
        batch_url = "{base}stock/market/batch?".format(base=iex_url_base)
        rs = (grequests.get(batch_url, params=payload) for payload in payloads)
        result = grequests.map(rs)
        try:
            outputs = [r.json() for r in result]
        except AttributeError:
            outputs = []
        return outputs 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def scan_process(q_targets, q_results, args, target_process_done):
    reload(socket)
    signal.signal(signal.SIGINT, exit_func)
    s = Scanner(q_results, args.timeout * 60, args=args)
    while True:
        try:
            target = q_targets.get(timeout=0.2)
        except Exception as e:
            if target_process_done.value:
                break
            else:
                continue

        if 'target' in target:
            ret = s.init_from_url(target['target'])
        elif 'file' in target:
            ret = s.init_from_log_file(target['file'])
        else:
            continue

        if ret:
            host, results = s.scan(threads=args.t)
            if results:
                q_results.put((host, results)) 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def Get(*args, **kwargs) -> requests.models.Response:
        """Send a GET request.

        Args:
            *args: arguments for requests.get(); particularly the URL
            **kwargs: optional arguments for requests.get()

        Returns:
            requests.models.Response: response
        """
        return requests.get(
            *args,
            headers=Net.HEADERS,
            timeout=float(castero.config.Config['request_timeout']),
            proxies={
                'http': castero.config.Config['proxy_http'],
                'https': castero.config.Config['proxy_https'],
            },
            **kwargs
        ) 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def _process_get_request(self, url, session, frmt, data=None, **kwargs):
        try:
            res = session.get(url, **kwargs)
            self.last_response = res
            res = self._interpret_returned_request(res, frmt)
            return res
        except Exception:
            return None 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def http_get(self, query, frmt='json', params={}, **kargs):
        """

        * query is the suffix that will be appended to the main url attribute.
        * query is either a string or a list of strings.
        * if list is larger than ASYNC_THRESHOLD, use asynchronous call.

        """
        if isinstance(query, list) and len(query) > self.settings.ASYNC_THRESHOLD:
            self.logging.debug("Running async call for a list")
            return self.get_async(query, frmt, params=params, **kargs)

        if isinstance(query, list) and len(query) <= self.settings.ASYNC_THRESHOLD:
            self.logging.debug("Running sync call for a list")
            return [self.get_one(key, frmt, params=params, **kargs) for key in query]
            #return self.get_sync(query, frmt)

        # OTHERWISE
        self.logging.debug("Running http_get (single call mode)")
        #return self.get_one(**{'frmt': frmt, 'query': query, 'params':params})


        # if user provide a content, let us use it, otherwise, it will be the
        # same as the frmt provided
        content = kargs.get("content", self.content_types[frmt])

        # if user provide a header, we use it otherwise, we use the header from
        # bioservices and the content defined here above
        headers = kargs.get("headers")
        if headers is None:
            headers = {}
            headers['User-Agent'] = self.getUserAgent()
            if content is None:
                headers['Accept'] = self.content_types[frmt]
            else:
                headers['Accept'] = content
        kargs.update({"headers": headers})

        return self.get_one(query, frmt=frmt, params=params,  **kargs) 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def get_one(self, query=None, frmt='json', params={}, **kargs):
        """

        if query starts with http:// do not use self.url
        """
        self._calls()
        url = self._build_url(query)

        if url.count('//') >1:
            self.logging.warning("URL of the services contains a double //." +
                "Check your URL and remove trailing /")
        self.logging.debug(url)
        try:
            kargs['params'] = params
            kargs['timeout'] = self.TIMEOUT
            # Used only in biomart with cosmic database
            # See doc/source/biomart.rst for an example
            if hasattr(self, 'authentication'):
                kargs['auth'] = self.authentication

            #res = self.session.get(url, **{'timeout':self.TIMEOUT, 'params':params})
            res = self.session.get(url, **kargs)

            self.last_response = res
            res = self._interpret_returned_request(res, frmt)
            try:
                # for python 3 compatibility
                res = res.decode()
            except:
                pass
            return res
        except Exception as err:
            self.logging.critical(err)
            self.logging.critical("""Query unsuccesful. Maybe too slow response.
    Consider increasing it with settings.TIMEOUT attribute {}""".format(self.settings.TIMEOUT)) 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def http_post(self, query, params=None, data=None,
                    frmt='xml', headers=None, files=None, content=None, **kargs):
        # query and frmt are bioservices parameters. Others are post parameters
        # NOTE in requests.get you can use params parameter
        # BUT in post, you use data
        # only single post implemented for now unlike get that can be asynchronous
        # or list of queries


        # if user provide a header, we use it otherwise, we use the header from
        # bioservices and the content defined here above
        if headers is None:
            headers = {}
            headers['User-Agent'] = self.getUserAgent()
            if content is None:
                headers['Accept'] = self.content_types[frmt]
            else:
                headers['Accept'] = content

        self.logging.debug("Running http_post (single call mode)")
        kargs.update({'query':query})
        kargs.update({'headers':headers})
        kargs.update({'files':files})
        kargs.update({'params':params})
        kargs.update({'data':data})
        kargs.update({'frmt':frmt})
        return self.post_one(**kargs) 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def get_response(url):

    headers = {
        "headers" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.94 Safari/537.36"
    }
    
    response = requests.get(url, headers = headers)
    return response

# 获取每个页面的url 

# 需要导入模块: import grequests [as 别名]
# 或者: from grequests import get [as 别名]
def get_girl_urls(page_urls):

    girl_urls = []

    # 采用grequests，建立5个并发连接
    rs = (grequests.get(url) for url in page_urls)
    responses = grequests.map(rs, size = 5)
    
    for response in responses:
        parsed_body = html.fromstring(response.text)
        girl = parsed_body.xpath('//div[@class="grid_title"]/a/@href')
        girl_urls.extend(girl)
    
    return girl_urls