Python urllib.request方法代码示例

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def get_response(url):
	headers = {}
	headers['User-Agent'] = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'
	req = urllib.request.Request(
		url = url,
		headers = headers
	)
	#try: 
	response = urllib.request.urlopen(req)
	data = response.read()
	#except: # 抓取出错
	#	return None
	if response.info().get('Content-Encoding') == 'gzip':
		data = ungzip(data)
	elif response.info().get('Content-Encoding') == 'deflate':
		data = undeflate(data)
	response.data = data
	return response



## 抓取网页html 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def save_picture(recipes_raw, url):
    recipe = recipes_raw[url]
    path_save = path.join(
        config.path_img, '{}.jpg'.format(URL_to_filename(url)))
    if not path.isfile(path_save):
        if 'picture_link' in recipe:
            link = recipe['picture_link']
            if link is not None:
                try:
                    if 'epicurious' in url:
                        img_url = 'https://{}'.format(link[2:])
                        urllib.request.urlretrieve(img_url, path_save)
                    else:
                        urllib.request.urlretrieve(link, path_save)
                except:
                    print('Could not download image from {}'.format(link)) 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def test_server(server, qtbot, path, content, expected):
    with qtbot.waitSignal(server.new_request, timeout=100):
        url = 'http://localhost:{}{}'.format(server.port, path)
        try:
            response = urllib.request.urlopen(url)
        except urllib.error.HTTPError as e:
            # "Though being an exception (a subclass of URLError), an HTTPError
            # can also function as a non-exceptional file-like return value
            # (the same thing that urlopen() returns)."
            # ...wat
            print(e.read().decode('utf-8'))
            raise

    data = response.read().decode('utf-8')

    assert server.get_requests() == [server.ExpectedRequest('GET', path)]
    assert (content in data) == expected 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def get_filesize(url, timeout=15):
    '''
    Fetches file's size of a file over HTTP.
    
    :param url: Url address.
    :type url: string
    :param timeout: Timeout in seconds. Default is 15.
    :type timeout: int
    :returns: Size in bytes.
    :rtype: int
    '''
    try:
        urlObj = urllib.request.urlopen(url, timeout=timeout)
        file_size = int(urlObj.headers["Content-Length"])
    except (IndexError, KeyError, TypeError, urllib.error.HTTPError, urllib.error.URLError):
        return 0
        
    return file_size 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def request(self, method: str, url: str, body=None, headers=None,
                **client_options):
        client = self.create_client(**client_options)
        url, headers, body = client.sign(url, method, body, headers)
        request = urllib.request.Request(url, body, headers, method=method)
        try:
            return urllib.request.urlopen(request)
        except urllib.error.HTTPError as e:
            logger = logging.getLogger(__name__ + '.StashTeam.request')
            logger.exception(
                '[%s %s] %s\nrequest headers: %r\nrequest body: %r\n'
                'client_options: %r\nresponse status: %r\n'
                'response headers: %r\nresponse body: %r',
                method, url, e, headers, body, client_options,
                e.code, dict(e.headers), e.read()
            )
            raise 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def register(self, identity: Identity, public_key: PKey) -> None:
        team = self.team
        if not (isinstance(team, identity.team_type) and
                cast(str, identity.identifier).startswith(team.server_url)):
            return
        data = json.dumps({
            'text': format_openssh_pubkey(public_key)
        })
        try:
            self.request(
                identity, 'POST', self.REGISTER_URL.format(self.team), data,
                headers={'Content-Type': 'application/json'}
            )
        except urllib.error.HTTPError as e:
            if e.code == 409:
                errors = json.loads(e.read().decode('utf-8'))['errors']
                raise DuplicatePublicKeyError(errors[0]['message'])
            raise 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def feature_popup_content(request):
    url = request.POST.get("url", None)

    if url is not None:
        host = "{uri.hostname}".format(uri=urlparse(url))
        try:
            if host in settings.ALLOWED_POPUP_HOSTS:
                if url is not None:
                    f = urllib.request.urlopen(url)
                    return HttpResponse(f.read())
            else:
                raise Exception()
        except:
            return HttpResponseNotFound()
    else:
        return HttpResponseNotFound() 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def get(self, rest_query):
        """
        Sends a GET request to the Alyx server. Will raise an exception on any status_code
        other than 200, 201.
        For the dictionary contents and list of endpoints, refer to:
        https://alyx.internationalbrainlab.org/docs

        :param rest_query: example: '/sessions?user=Hamish'.
        :type rest_query: str

        :return: (dict/list) json interpreted dictionary from response
        """
        rep = self._generic_request(requests.get, rest_query)
        if isinstance(rep, dict) and list(rep.keys()) == ['count', 'next', 'previous', 'results']:
            if len(rep['results']) < rep['count']:
                rep = _PaginatedResponse(self, rep)
            else:
                rep = rep['results']
        return rep 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def patch(self, rest_query, data=None):
        """
        Sends a PATCH request to the Alyx server.
        For the dictionary contents, refer to:
        https://alyx.internationalbrainlab.org/docs

        :param rest_query: (required)the endpoint as full or relative URL
        :type rest_query: str
        :param data: json encoded string or dictionary
        :type data: None, dict or str

        :return: response object
        """
        if isinstance(data, dict):
            data = json.dumps(data)
        return self._generic_request(requests.patch, rest_query, data=data) 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def getResource(self, url, uuid):
        """Provides the resource"""
        if self.__cacheDir is None:
            return None

        if url in self.__urlToFileName:
            return self.__urlToFileName[url]

        fName = self.__cacheDir + hashlib.md5(url.encode('utf-8')).hexdigest()
        if fName in self.__threads:
            # Reject double request
            return None

        thread = ResourceRetriever()
        self.__threads[fName] = thread
        self.__connectThread(thread)
        thread.get(url, fName, uuid)
        return None 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def _url_status(url):
    parse_obj = urllib.parse.urlparse(url)

    timer = 1
    for i in range(6):
        try:
            connection = http.client.HTTPConnection(parse_obj.netloc)
            connection.request('HEAD', parse_obj.path)
            break
        except Exception as e:
            print(url, e, 'sleep', timer)
            time.sleep(timer)
            timer *= 2
    else:
        return e

    response = connection.getresponse()
    connection.close()
    return response.status 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def get_url(url):
    req_headers = {
        'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.A.B.C Safari/525.13',
        'Referer': url
    }

    request = urllib.request.Request(url, headers=req_headers)
    opener = urllib.request.build_opener()
    timer = 1
    for i in range(1):
        try:
            return opener.open(request).read()
        except:
            time.sleep(timer)
            timer *= 2
    raise IOError("Unable to download `%s`."%url) 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def download_files(self, urls, env, headers=None, dir=None):

        if dir is None:
            dir = env.get('cache-dir')
        
        dest_directory = os.path.join(dir, "tmp_" + str(uuid.uuid4()))
        
        if not os.path.exists(dest_directory):
            os.makedirs(dest_directory)

        for data_url in urls:
            if isinstance(data_url, tuple):
                data_url, download_file = data_url
            else:
                download_file = data_url.split('/')[-1]

            download_path = os.path.join(dest_directory, download_file)

            if headers:
                opener = urllib.request.build_opener()
                opener.addheaders = headers
                urllib.request.install_opener(opener)

            try:
                urllib.request.urlretrieve(data_url, filename=download_path, 
                                           reporthook=self._report_hook(download_file), data=None)
            except Exception as e:
                raise VergeMLError("Could not download {}: {}".format(data_url, e))
            finally:
                if headers:
                    urllib.request.install_opener(urllib.request.build_opener())

        return dest_directory 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def read_file_function(self, filepath):
        if filepath.startswith('http://'):
            from urllib.request import urlopen
            s = urlopen(filepath, timeout=5).read().decode('utf-8')
            return re.sub('\r+\n*', '\n', s)

        if self.base_path:
            filepath = os.path.join(self.base_path, filepath)
        filepath = os.path.abspath(filepath)
        view = CompileKspThread.find_view_by_filename(filepath, self.base_path)
        if view is None:
            s = codecs.open(filepath, 'r', 'utf-8').read()
            return re.sub('\r+\n*', '\n', s)
        else:
            return view.substr(sublime.Region(0, view.size())) 

# 需要导入模块: import urllib [as 别名]
# 或者: from urllib import request [as 别名]
def jsonload(url):
	'''Load a web page and return the resulting JSON object'''
	request = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
	with urllib.request.urlopen(request, timeout=TIMEOUT) as webpage:
		data = str(webpage.read(), 'UTF-8')
		data = json.loads(data)
	return data