Python urllib.parse方法代码示例

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def find_external_links(url, page):
    """Find rel="homepage" and rel="download" links in `page`, yielding URLs"""

    for match in REL.finditer(page):
        tag, rel = match.groups()
        rels = set(map(str.strip, rel.lower().split(',')))
        if 'homepage' in rels or 'download' in rels:
            for match in HREF.finditer(tag):
                yield urllib.parse.urljoin(url, htmldecode(match.group(1)))

    for tag in ("<th>Home Page", "<th>Download URL"):
        pos = page.find(tag)
        if pos != -1:
            match = HREF.search(page, pos)
            if match:
                yield urllib.parse.urljoin(url, htmldecode(match.group(1))) 

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def _download_svn(self, url, filename):
        url = url.split('#', 1)[0]  # remove any fragment for svn's sake
        creds = ''
        if url.lower().startswith('svn:') and '@' in url:
            scheme, netloc, path, p, q, f = urllib.parse.urlparse(url)
            if not netloc and path.startswith('//') and '/' in path[2:]:
                netloc, path = path[2:].split('/', 1)
                auth, host = splituser(netloc)
                if auth:
                    if ':' in auth:
                        user, pw = auth.split(':', 1)
                        creds = " --username=%s --password=%s" % (user, pw)
                    else:
                        creds = " --username=" + auth
                    netloc = host
                    parts = scheme, netloc, url, p, q, f
                    url = urllib.parse.urlunparse(parts)
        self.info("Doing subversion checkout from %s to %s", url, filename)
        os.system("svn checkout%s -q %s %s" % (creds, url, filename))
        return filename 

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def _vcs_split_rev_from_url(url, pop_prefix=False):
        scheme, netloc, path, query, frag = urllib.parse.urlsplit(url)

        scheme = scheme.split('+', 1)[-1]

        # Some fragment identification fails
        path = path.split('#', 1)[0]

        rev = None
        if '@' in path:
            path, rev = path.rsplit('@', 1)

        # Also, discard fragment
        url = urllib.parse.urlunsplit((scheme, netloc, path, query, ''))

        return url, rev 

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def _encode_auth(auth):
    """
    A function compatible with Python 2.3-3.3 that will encode
    auth from a URL suitable for an HTTP header.
    >>> str(_encode_auth('username%3Apassword'))
    'dXNlcm5hbWU6cGFzc3dvcmQ='

    Long auth strings should not cause a newline to be inserted.
    >>> long_auth = 'username:' + 'password'*10
    >>> chr(10) in str(_encode_auth(long_auth))
    False
    """
    auth_s = urllib.parse.unquote(auth)
    # convert to bytes
    auth_bytes = auth_s.encode()
    # use the legacy interface for Python 2.3 support
    encoded_bytes = base64.encodestring(auth_bytes)
    # convert back to a string
    encoded = encoded_bytes.decode()
    # strip the trailing carriage return
    return encoded.replace('\n', '') 

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def apply_config(self):
        """
        Apply global configuration object based on the configuration values.

        At this point this only applies to the JSON library which is used.
        """
        if not self.__config:
            # parse() hasn't been called yet. We should probably throw here
            return

        # Set json library based on the config value. If "auto" is provided this means we use
        # default behavior which is try to use ujson and if that's not available fall back to
        # stdlib json
        json_library = self.json_library
        current_json_library = scalyr_util.get_json_lib()

        if json_library != "auto" and json_library != current_json_library:
            self.__logger.debug(
                'Changing JSON library from "%s" to "%s"'
                % (current_json_library, json_library)
            )
            scalyr_util.set_json_lib(json_library) 

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def __init__(
        self,
        k8s_log_configs,
        logger,
        rename_no_original,
        rename_logfile=None,
        parse_format="json",
    ):
        """
        @param k8s_log_configs: The config snippets from the configuration
        @param logger: A scalyr logger
        @param rename_no_original: A bool, used to prevent the original log file name from being added to the attributes.
        @param rename_logfile: A value for renaming a logfile - can contain variable substitutions
        @param parse_format: The parse format of this log config
        """

        if rename_logfile is None:
            rename_logfile = "/${container_runtime}/${container_name}.log"

        self.__k8s_log_configs = k8s_log_configs
        self._logger = logger
        self.__rename_logfile = rename_logfile
        self.__rename_no_original = rename_no_original
        self.__parse_format = parse_format 

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def _http_request(url, headers=None, data=None, method="GET", timeout=None):
    import socket
    from six.moves import urllib

    headers = headers or {}
    url_parts = urllib.parse.urlparse(url)
    conn = _HTTPConnection(url_parts.scheme, url_parts.netloc, timeout)
    params = urllib.parse.urlencode(data) if data else ""
    try:
        conn.request(method, url_parts.path, params, headers)
    except socket.error as e:
        if e.errno == errno.ECONNREFUSED:
            raise HTTPConnectionError(url)
        raise
    else:
        return HTTPResponse(conn.getresponse()) 

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def mtranslate_google(word):
	import html.parser
	import urllib.request
	import urllib.parse

	agent = {'User-Agent':
	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36"}

	def unescape(text):
		parser = html.parser.HTMLParser()
		return (parser.unescape(text))

	def translate(to_translate, to_language="auto", from_language="auto"):
		base_link = "http://translate.google.com/m?hl=%s&sl=%s&q=%s"

		to_translate = urllib.parse.quote(to_translate)
		link = base_link % (to_language, from_language, to_translate)
		request = urllib.request.Request(link, headers=agent)
		raw_data = urllib.request.urlopen(request).read()

		data = raw_data.decode("utf-8")
		expr = r'class="t0">(.*?)<'
		re_result = re.findall(expr, data)

		if (len(re_result) == 0):
			result = ""
		else:
			result = unescape(re_result[0])
		return (result)

	return [[word, translate(word, lang_to, lang_from)]], ['', '']

# reverso.net 

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def parse_requirement_arg(spec):
    try:
        return Requirement.parse(spec)
    except ValueError:
        raise DistutilsError(
            "Not a URL, existing file, or requirement spec: %r" % (spec,)
        ) 

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def egg_info_for_url(url):
    parts = urllib.parse.urlparse(url)
    scheme, server, path, parameters, query, fragment = parts
    base = urllib.parse.unquote(path.split('/')[-1])
    if server == 'sourceforge.net' and base == 'download':  # XXX Yuck
        base = urllib.parse.unquote(path.split('/')[-2])
    if '#' in base:
        base, fragment = base.split('#', 1)
    return base, fragment 

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def from_url(cls, url):
        "Construct a (possibly null) ContentChecker from a URL"
        fragment = urllib.parse.urlparse(url)[-1]
        if not fragment:
            return ContentChecker()
        match = cls.pattern.search(fragment)
        if not match:
            return ContentChecker()
        return cls(**match.groupdict()) 

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def _download_url(self, scheme, url, tmpdir):
        # Determine download filename
        #
        name, fragment = egg_info_for_url(url)
        if name:
            while '..' in name:
                name = name.replace('..', '.').replace('\\', '_')
        else:
            name = "__downloaded__"  # default if URL has no path contents

        if name.endswith('.egg.zip'):
            name = name[:-4]  # strip the extra .zip before download

        filename = os.path.join(tmpdir, name)

        # Download the file
        #
        if scheme == 'svn' or scheme.startswith('svn+'):
            return self._download_svn(url, filename)
        elif scheme == 'git' or scheme.startswith('git+'):
            return self._download_git(url, filename)
        elif scheme.startswith('hg+'):
            return self._download_hg(url, filename)
        elif scheme == 'file':
            return urllib.request.url2pathname(urllib.parse.urlparse(url)[2])
        else:
            self.url_ok(url, True)  # raises error if not allowed
            return self._attempt_download(url, filename) 

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def local_open(url):
    """Read a local path, with special support for directories"""
    scheme, server, path, param, query, frag = urllib.parse.urlparse(url)
    filename = urllib.request.url2pathname(path)
    if os.path.isfile(filename):
        return urllib.request.urlopen(url)
    elif path.endswith('/') and os.path.isdir(filename):
        files = []
        for f in os.listdir(filename):
            filepath = os.path.join(filename, f)
            if f == 'index.html':
                with open(filepath, 'r') as fp:
                    body = fp.read()
                break
            elif os.path.isdir(filepath):
                f += '/'
            files.append('<a href="{name}">{name}</a>'.format(name=f))
        else:
            tmpl = ("<html><head><title>{url}</title>"
                "</head><body>{files}</body></html>")
            body = tmpl.format(url=url, files='\n'.join(files))
        status, message = 200, "OK"
    else:
        status, message, body = 404, "Path not found", "Not found"

    headers = {'content-type': 'text/html'}
    body_stream = six.StringIO(body)
    return urllib.error.HTTPError(url, status, message, headers, body_stream) 

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def _generate_auth_token(self, uri, sas_name, sas_value):
        sas = base64.b64decode(sas_value)
        expiry = str(int(time.time() + default_sas_expiry))
        string_to_sign = (uri + "\n" + expiry).encode("utf-8")
        signed_hmac_sha256 = hmac.HMAC(sas, string_to_sign, hashlib.sha256)
        signature = urllib.parse.quote(base64.b64encode(signed_hmac_sha256.digest()))
        return "SharedAccessSignature sr={}&sig={}&se={}&skn={}".format(
            uri, signature, expiry, sas_name
        ) 

# 需要导入模块: from six.moves import urllib [as 别名]
# 或者: from six.moves.urllib import parse [as 别名]
def load_yahoo_quote(ticker, begindate, enddate, info='quote'):
    '''
    This function load the corresponding history/divident/split from Yahoo.
    The "begindate" and "enddate" are in the format of YYYYMMDD.
    The "info" can be "quote" for price, "divident" for divident events,
    or "split" for split events.
    '''
    # Check to make sure that the cookie and crumb has been loaded
    global _cookie, _crumb
    if _cookie == None or _crumb == None:
        _get_cookie_crumb()

    # Prepare the parameters and the URL
    tb = time.mktime((int(begindate[0:4]), int(
        begindate[4:6]), int(begindate[6:8]), 4, 0, 0, 0, 0, 0))
    te = time.mktime((int(enddate[0:4]), int(
        enddate[4:6]), int(enddate[6:8]), 18, 0, 0, 0, 0, 0))

    param = dict()
    param['period1'] = int(tb)
    param['period2'] = int(te)
    param['interval'] = '1d'
    if info == 'quote':
        param['events'] = 'history'
    elif info == 'dividend':
        param['events'] = 'div'
    elif info == 'split':
        param['events'] = 'split'
    param['crumb'] = _crumb
    params = urllib.parse.urlencode(param)
    url = 'https://query1.finance.yahoo.com/v7/finance/download/{}?{}'.format(
        ticker, params)
    # print(url)
    req = urllib.request.Request(url, headers=_headers)

    # Perform the query
    # There is no need to enter the cookie here, as it is automatically handled by opener
    f = urllib.request.urlopen(req, timeout=5)
    alines = f.read().decode('utf-8')
    # print(alines)
    return alines.split('\n')