Python urllib2.Request类代码示例

    def modified_run(self):
        import sys

        try:
            try:
                from urllib2 import HTTPHandler, build_opener
                from urllib2 import urlopen, Request
                from urllib import urlencode
            except ImportError:
                from urllib.request import HTTPHandler, build_opener
                from urllib.request import urlopen, Request
                from urllib.parse import urlencode

            os_ver = platform.system()
            py_ver = "_".join(str(x) for x in sys.version_info)
            now_ver = __version__.replace(".", "_")

            code = "os:{0},py:{1},now:{2}".format(os_ver, py_ver, now_ver)
            action = command_subclass.action
            cid = getnode()
            payload = {"v": "1", "tid": "UA-61791314-1", "cid": str(cid), "t": "event", "ec": action, "ea": code}

            url = "http://www.google-analytics.com/collect"
            data = urlencode(payload).encode("utf-8")
            request = Request(url, data=data)
            request.get_method = lambda: "POST"
            connection = urlopen(request)
        except:
            pass
        orig_run(self)

  def hit_endpoint(self, url, data_dict={}, http_method='GET'):
    """
    A reusable method that actually performs the request to the specified Atlas API endpoint.
    """

    if self.verbose == 'true':
      print "HIT_ENDPOINT"

    data_dict.update({ "access_token" : self.access_token })
    if self.verbose == 'true':
      print "  Added access_token to data_dict (inside hit_endpoint)"

    if self.verbose == 'true':
      print "  Constructing request URL"
    request = Request(url, urllib.urlencode(data_dict))
    
    if self.verbose == 'true':
      print "    Setting request http_method: %s" % http_method
    request.get_method = lambda: http_method
    
    try:
      if self.verbose == 'true':
        print "  Opening Request URL: %s?%s" % (request.get_full_url(),request.get_data())
      response = urlopen(request)
    except URLError, e:
      raise SystemExit(e)

 def report_now(self, registry=None, timestamp=None):
     if self.autocreate_database and not self._did_create_database:
         self._create_database()
     timestamp = timestamp or int(round(self.clock.time()))
     metrics = (registry or self.registry).dump_metrics()
     post_data = []
     for key, metric_values in metrics.items():
         if not self.prefix:
             table = key
         else:
             table = "%s.%s" % (self.prefix, key)
         values = ",".join(["%s=%s" % (k, v if type(v) is not str \
                                            else '"{}"'.format(v))
                           for (k, v) in metric_values.items()])
         line = "%s %s %s" % (table, values, timestamp)
         post_data.append(line)
     post_data = "\n".join(post_data)
     path = "/write?db=%s&precision=s" % self.database
     url = "%s://%s:%s%s" % (self.protocol, self.server, self.port, path)
     request = Request(url, post_data.encode("utf-8"))
     if self.username:
         auth = _encode_username(self.username, self.password)
         request.add_header("Authorization", "Basic %s" % auth.decode('utf-8'))
     try:
         response = urlopen(request)
         response.read()
     except URLError as err:
         LOG.warning("Cannot write to %s: %s",
                     self.server, err.reason)

    def test_http_doubleslash(self):
        # Checks that the presence of an unnecessary double slash in a url doesn't break anything
        # Previously, a double slash directly after the host could cause incorrect parsing of the url
        h = urllib2.AbstractHTTPHandler()
        o = h.parent = MockOpener()

        data = ""
        ds_urls = [
            "http://example.com/foo/bar/baz.html",
            "http://example.com//foo/bar/baz.html",
            "http://example.com/foo//bar/baz.html",
            "http://example.com/foo/bar//baz.html",
        ]

        for ds_url in ds_urls:
            ds_req = Request(ds_url, data)

            # Check whether host is determined correctly if there is no proxy
            np_ds_req = h.do_request_(ds_req)
            self.assertEqual(np_ds_req.unredirected_hdrs["Host"],"example.com")

            # Check whether host is determined correctly if there is a proxy
            ds_req.set_proxy("someproxy:3128",None)
            p_ds_req = h.do_request_(ds_req)
            self.assertEqual(p_ds_req.unredirected_hdrs["Host"],"example.com")

    def _send_webhook_msg(self, ip, port, payload_str, url_path='',
                          content_len=-1, content_type='application/json',
                          get_method=None):
        headers = {
            'content-type': content_type,
        }

        if not payload_str:
            content_len = None
            payload = None
        else:
            payload = bytes(payload_str, encoding='utf-8')

        if content_len == -1:
            content_len = len(payload)

        if content_len is not None:
            headers['content-length'] = str(content_len)

        url = 'http://{ip}:{port}/{path}'.format(ip=ip, port=port,
                                                 path=url_path)

        req = Request(url, data=payload, headers=headers)


        if get_method is not None:
            req.get_method = get_method

        return urlopen(req)

def test_compare_triples():
    for mime, fext in MIME_TYPES.items():
        dump_path = path.join(DUMP_DIR, path.basename(mime))

        for url in URLs:
            if six.PY2:
                fname = '%s.%s' % (path.basename(urlparse.urlparse(url).path), fext)
            else:
                fname = '%s.%s' % (path.basename(urlparse(url).path), fext)

            fname = path.join(dump_path, fname)

            req = Request(url)
            req.add_header('Accept', mime)
            res = urlopen(req)

            g_fdp.parse(data=res.read(), format=mime)
            g_dump.parse(fname, format=mime)

            both, first, second = graph_diff(g_fdp, g_dump)
            n_first = len(first)
            # n_second = len(second)
            # n_both = len(both)

            assert_equals(
               n_first, 0, '{} triple(s) different from reference:\n\n{}===\n{}\n'.format(
                  n_first, first.serialize(format='turtle'), second.serialize(format='turtle')))

	def get(self, bugid):
		"""Get an ElementTree representation of a bug.

		@param bugid: bug id
		@type  bugid: int

		@rtype: ElementTree
		"""
		if not self.authenticated and not self.skip_auth:
			self.auth()

		qparams = config.params['show'].copy()
		qparams['id'] = bugid

		req_params = urlencode(qparams, True)
		req_url = urljoin(self.base,  config.urls['show'])
		req_url += '?' + req_params
		req = Request(req_url, None, config.headers)
		if self.httpuser and self.httppassword:
			base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1]
			req.add_header("Authorization", "Basic %s" % base64string)
		resp = self.opener.open(req)

		fd = StringIO(resp.read())
		# workaround for ill-defined XML templates in bugzilla 2.20.2
		parser = ForcedEncodingXMLTreeBuilder(encoding = 'utf-8')
		etree = ElementTree.parse(fd, parser)
		bug = etree.find('.//bug')
		if bug and bug.attrib.has_key('error'):
			return None
		else:
			return etree

def fetch(url, auth=None):
    """Fetch URL, optional with HTTP Basic Authentication."""

    if not (url.startswith('http://') or url.startswith('https://')):
        try:
            with io.open(url, 'r', encoding='utf-8', errors='replace') as fp:
                return u''.join(fp.readlines())
        except OSError as e:
            raise AcrylamidException(e.args[0])

    req = Request(url)
    if auth:
        req.add_header('Authorization', 'Basic ' + b64encode(auth))

    try:
        r = urlopen(req)
    except HTTPError as e:
        raise AcrylamidException(e.msg)

    if r.getcode() == 401:
        user = input('Username: ')
        passwd = getpass.getpass()
        fetch(url, user + ':' + passwd)
    elif r.getcode() == 200:
        try:
            enc = re.search('charset=(.+);?', r.headers.get('Content-Type', '')).group(1)
        except AttributeError:
            enc = 'utf-8'
        return u'' + r.read().decode(enc)

    raise AcrylamidException('invalid status code %i, aborting.' % r.getcode())

    def __init__(self, url, method="GET", headers=None, data=None,
                 downloadTo=None, closeConnection=False, proxy=None,
                 redirectedFrom=None, unredirectedHeaders=None, **kw):
        """
        """
        headers = headers or dict()
        urllib2_Request.__init__(
                self, str(url), data=data, headers=headers,
                origin_req_host=kw.get("origin_req_host", redirectedFrom),
                unverifiable=kw.get("unverifiable", False),
            )
        Message.__init__(self, url, method, self.headers)
        self.host = self._url.host
        self.port = self._url.port
        self.setProxy(proxy)

        assert isinstance(self.headers, util.InsensitiveDict)
        unredirectedHeaders = unredirectedHeaders or dict()
        self.unredirectedHeaders = util.InsensitiveDict(unredirectedHeaders)

        self.closeConnection = closeConnection is True

        self.downloadTo = downloadTo
        self.redirectedTo = None
        self.redirectedFrom = tuple()
        self.response = defer.Deferred()

    def makeRequest(self, method, path, params=None):
        if not params:
            params = {}
        params['key'] = self.api_key
        params['token'] = self.oauth_token
        
        url = self.base_url + path
        data = None

        if method == 'GET':
            url += '?' + urlencode(params)
        elif method in ['DELETE', 'POST', 'PUT']:
            data = urlencode(params).encode('utf-8')

        request = Request(url)
        if method in ['DELETE', 'PUT']:
            request.get_method = lambda: method

        try:
            if data:
                response = urlopen(request, data=data)
            else:
                response = urlopen(request)
        except HTTPError as e:
            print(e)
            print(e.read())
            result = None
        else:
            result = json.loads(response.read().decode('utf-8'))

        return result

	def auth(self):
		"""Authenticate a session.
		"""
		if self.try_auth():
			return

		# prompt for username if we were not supplied with it
		if not self.user:
			self.log('No username given.')
			self.user = self.get_input('Username: ')

		# prompt for password if we were not supplied with it
		if not self.password:
			self.log('No password given.')
			self.password = getpass.getpass()

		# perform login
		qparams = config.params['auth'].copy()
		qparams['Bugzilla_login'] = self.user
		qparams['Bugzilla_password'] = self.password

		req_url = urljoin(self.base, config.urls['auth'])
		req = Request(req_url, urlencode(qparams), config.headers)
		if self.httpuser and self.httppassword:
			base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1]
			req.add_header("Authorization", "Basic %s" % base64string)
		resp = self.opener.open(req)
		if resp.info().has_key('Set-Cookie'):
			self.authenticated = True
			if not self.forget:
				self.cookiejar.save()
				os.chmod(self.cookiejar.filename, 0700)
			return True
		else:
			raise RuntimeError("Failed to login")

	def try_auth(self):
		"""Check whether the user is already authenticated."""
		if self.authenticated:
			return True

		# try seeing if we really need to request login
		if not self.forget:
			try:
				self.cookiejar.load()
			except IOError:
				pass

		req_url = urljoin(self.base, config.urls['auth'])
		req_url += '?GoAheadAndLogIn=1'
		req = Request(req_url, None, config.headers)
		if self.httpuser and self.httppassword:
			base64string = base64.encodestring('%s:%s' % (self.httpuser, self.httppassword))[:-1]
			req.add_header("Authorization", "Basic %s" % base64string)
		resp = self.opener.open(req)
		re_request_login = re.compile(r'<title>.*Log in to Bugzilla</title>')
		if not re_request_login.search(resp.read()):
			self.log('Already logged in.')
			self.authenticated = True
			return True
		return False

    def download_checked(cls, url, target_file, expected_digest):
        if os.path.exists(target_file):
            # file already exists, see if we can use it.
            if PBFile.md5_digest(target_file) == expected_digest:
                # local file is ok
                return
            else:
                os.unlink(target_file)

        user_agent = ("pbunder/%s " % (cls.my_version) +
                      "(http://github.com/zeha/pbundler/issues)")

        try:
            req = Request(url)
            req.add_header("User-Agent", user_agent)
            req.add_header("Accept", "*/*")
            with file(target_file, 'wb') as f:
                sock = urlopen(req)
                try:
                    f.write(sock.read())
                finally:
                    sock.close()

        except Exception as ex:
            raise PBundlerException("Downloading %s failed (%s)" % (url, ex))

        local_digest = PBFile.md5_digest(target_file)
        if local_digest != expected_digest:
            os.unlink(target_file)
            msg = ("Downloading %s failed (MD5 Digest %s did not match expected %s)" %
                   (url, local_digest, expected_digest))
            raise PBundlerException(msg)
        else:
            # local file is ok
            return

def _download_as_child(url, if_modified_since):
	from httplib import HTTPException
	from urllib2 import urlopen, Request, HTTPError, URLError
	try:
		#print "Child downloading", url
		if url.startswith('http:') or url.startswith('https:') or url.startswith('ftp:'):
			req = Request(url)
			if url.startswith('http:') and if_modified_since:
				req.add_header('If-Modified-Since', if_modified_since)
			src = urlopen(req)
		else:
			raise Exception(_('Unsupported URL protocol in: %s') % url)

		try:
			sock = src.fp._sock
		except AttributeError:
			sock = src.fp.fp._sock	# Python 2.5 on FreeBSD
		while True:
			data = sock.recv(256)
			if not data: break
			os.write(1, data)

		sys.exit(RESULT_OK)
	except (HTTPError, URLError, HTTPException) as ex:
		if isinstance(ex, HTTPError) and ex.code == 304: # Not modified
			sys.exit(RESULT_NOT_MODIFIED)
		print >>sys.stderr, "Error downloading '" + url + "': " + (str(ex) or str(ex.__class__.__name__))
		sys.exit(RESULT_FAILED)

    def parseFeed(cls, url, lastModified=None, etag=None):
        '''
        Return tuple of feed object, last-modified, etag.
        '''
        req = Request(normalize_url(url))
        if lastModified:
            req.add_header('if-modified-since', lastModified)
        if etag:
            req.add_header('if-none-match', etag)
        resp = None
        try:
            resp = urlopen(req, None, 10)
        except HTTPError as error:
            # HTTP 304 not modifed raise an exception
            resp = error

        # url of local file returns empty code
        if resp.code and resp.code != 200:
            return None

        feedDoc = etree.parse(resp)
        feedType = None
        for ft in cls._feedTypes:
            if ft.accept(feedDoc):
                feedType = ft
                break
        if not feedType:
            raise ValueError('Cannot handle ' + feedDoc.getroot().tag)
        return (feedType(url, feedDoc),
                resp.headers.get('last-modified'),
                resp.headers.get('etag'))