Python urllib2.request方法代码示例

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import request [as 别名]
def _make_request(self, opener, request, timeout=None):
        """Make the API call and return the response. This is separated into
           it's own function, so we can mock it easily for testing.

        :param opener:
        :type opener:
        :param request: url payload to request
        :type request: urllib.Request object
        :param timeout: timeout value or None
        :type timeout: float
        :return: urllib response
        """
        timeout = timeout or self.timeout
        try:
            return opener.open(request, timeout=timeout)
        except HTTPError as err:
            exc = handle_error(err)
            return exc 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import request [as 别名]
def _get_data(url):
    """Helper function to get data over http or from a local file"""
    if url.startswith('http://'):
        # Try Python 2, use Python 3 on exception
        try:
            resp = urllib.urlopen(url)
            encoding = resp.headers.dict.get('content-encoding', 'plain')
        except AttributeError:
            resp = urllib.request.urlopen(url)
            encoding = resp.headers.get('content-encoding', 'plain')
        data = resp.read()
        if encoding == 'plain':
            pass
        elif encoding == 'gzip':
            data = StringIO(data)
            data = gzip.GzipFile(fileobj=data).read()
        else:
            raise RuntimeError('unknown encoding')
    else:
        with open(url, 'r') as fid:
            data = fid.read()
        fid.close()

    return data 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import request [as 别名]
def _get_data(url):
    """Helper function to get data over http or from a local file"""
    if url.startswith('http://'):
        # Try Python 2, use Python 3 on exception
        try:
            resp = urllib.urlopen(url)
            encoding = resp.headers.dict.get('content-encoding', 'plain')
        except AttributeError:
            resp = urllib.request.urlopen(url)
            encoding = resp.headers.get('content-encoding', 'plain')
        data = resp.read()
        if encoding == 'plain':
            pass
        elif encoding == 'gzip':
            data = StringIO(data)
            data = gzip.GzipFile(fileobj=data).read()
        else:
            raise RuntimeError('unknown encoding')
    else:
        with open(url, 'r') as fid:
            data = fid.read()

    return data 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import request [as 别名]
def make_request(self,
                     method,
                     request_body=None,
                     query_params=None,
                     request_headers=None):
        method = method.upper()
        if request_headers:
            self._set_headers(request_headers)
            request_body = json.dumps(request_body) if request_body else None
            query_params = query_params if query_params else None
            opener = urllib.build_opener()
            request = urllib.Request(self._build_url(query_params),
                                     data=request_body)
            for key, value in self.request_headers.iteritems():
                request.add_header(key, value)
            request.get_method = lambda: method
            self._response = opener.open(request)
            self._set_response(self._response)
            self._reset() 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import request [as 别名]
def _update_headers(self, request_headers):
        """Update the headers for the request

        :param request_headers: headers to set for the API call
        :type request_headers: dictionary
        :return: dictionary
        """
        self.request_headers.update(request_headers) 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import request [as 别名]
def GET(path):
    url = getURL()
    opener = getOpener()
    request = urllib.Request(url)
    response = opener.open(request)
    setValues(response) 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import request [as 别名]
def POST(path):
    url = getURL()
    opener = getOpener()
    request = urllib.Request(url, GLOBALS['POSTDATA'])
    response = opener.open(request)
    setValues(response) 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import request [as 别名]
def extractArticlePublishedDate(articleLink, html = None):

    print("Extracting date from " + articleLink)

    articleDate = None

    try:
        articleDate = _extractFromURL(articleLink)

        if html is None:
            request = urllib.Request(articleLink)
            # Using a browser user agent, decreases the change of sites blocking this request - just a suggestion
            # request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36')
            html = urllib.build_opener().open(request).read()

        parsedHTML = BeautifulSoup(html,"lxml")

        possibleDate = _extractFromLDJson(parsedHTML)
        if possibleDate is None:
            possibleDate = _extractFromMeta(parsedHTML)
        if possibleDate is None:
            possibleDate = _extractFromHTMLTag(parsedHTML)


        articleDate = possibleDate

    except Exception as e:
        print("Exception in extractArticlePublishedDate for " + articleLink)
        print(e.args)   

    return articleDate 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import request [as 别名]
def _make_request(self, opener, request, timeout=None):
        if 200 <= self.response_code < 299:  # if successful code
            return MockResponse(self.response_code)
        else:
            raise handle_error(MockException(self.response_code)) 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import request [as 别名]
def test__urllib_headers(self, maker):
        self.client._update_headers({'X-test': 'Test'})
        self.client.get()
        request = maker.call_args[0][1]
        self.assertIn('X-test', request.headers) 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import request [as 别名]
def test__urllib_method(self, maker):
        self.client.delete()
        request = maker.call_args[0][1]
        self.assertEqual(request.get_method(), 'DELETE') 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import request [as 别名]
def download_file():
    big_file_url = "http://www.python.org/ftp/python/3.3.2/Python-3.3.2.tgz"

    # Download the file
    if sys.version_info[0] == 3:
        f = urllib.request.urlopen(big_file_url)
    else:
        f = urllib.urlopen(big_file_url)

    # Get the total length of the file
    scale = int(f.headers["content-length"])
    chunk_size = 500

    bar = CustomProgressBar(length=50,
                            left_limit='[',
                            right_limit=']',
                            head_repr=None,
                            empty_repr=' ',
                            filled_repr='|',
                            start=0,
                            scale_start=0,
                            scale_end=scale)

    print("Downloading a big txt file: ")

    print_flag = 0
    # Load all the data chunk by chunk
    while not bar.completed():
        f.read(chunk_size)
        bar.increase(chunk_size)

        # Don't print always
        if print_flag == 100:
            bar.show_progress_bar()
            print_flag = 0
        else:
            print_flag += 1

    bar.show_progress_bar()
    print("")
    print("Finished :)")