Python parse.urljoin方法代码示例

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def maybe_download_mnist_file(file_name, datadir=None, force=False):
    try:
        from urllib.parse import urljoin
        from urllib.request import urlretrieve
    except ImportError:
        from urlparse import urljoin
        from urllib import urlretrieve

    if not datadir:
        datadir = tempfile.gettempdir()
    dest_file = os.path.join(datadir, file_name)

    if force or not os.path.isfile(file_name):
        url = urljoin('http://yann.lecun.com/exdb/mnist/', file_name)
        urlretrieve(url, dest_file)
    return dest_file 

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def instance_endpoint_by_id(self, id=None):
        """
        Returns the endpoint designating some instantiated API resource of the
        same kind. If no `id` is provided, will use the `id` of the currently
        instantiated resource. If this is called from a static object, then
        returns `None`.
        """
        _id = self._get_id(id=id)

        if _id:
            # CASE 1: The class end point might have a formatting parameter
            # NOTE: This is for the weird case of submissions of an assignment
            try:
                tmp = self.class_endpoint.format(_id)
                if tmp != self.class_endpoint:
                    return tmp
            except IndexError: # means formatting didn't work
                pass

            # CASE 2: The class end point has not formatting parameter
            # NOTE: Trailing slash important (API bug)
            return urljoin(self.class_endpoint, "{}/".format(_id)) 

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def __init__(self, board_url_or_id):
        board_id = str(board_url_or_id)
        self.fetcher = HuaBanFetcher()
        if "http" in board_id:
            board_id = re.findall(r'boards/(\d+)/', board_id)[0]
        self.id = board_id
        path = "/boards/{board_id}/".format(
            board_id=board_id,
        )
        self.base_url = urljoin(BASE_URL, path)
        self.further_pin_url_tpl = urljoin(
            self.base_url,
            "?{random_string}"
            "&max={pin_id}"
            "&limit=20"
            "&wfl=1"
        )

        # uninitialized properties
        self.pin_count = None
        self.title = None
        self.description = None
        self._pins = []
        self._init_board() 

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def http_head(self, path, **kwargs):
        """
        Make a HEAD request to the k8s server.
        """
        try:

            url = urljoin(self.url, path)
            response = self.session.head(url, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem retrieving headers from " \
                "the Kubernetes API server. URL: {}".format(url)
            logger.error(message)
            raise KubeException(message) from err

        return response 

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def http_post(self, path, data=None, json=None, **kwargs):
        """
        Make a POST request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.post(url, data=data, json=json, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem posting data to " \
                      "the Kubernetes API server. URL: {}, " \
                      "data: {}, json: {}".format(url, data, json)
            logger.error(message)
            raise KubeException(message) from err

        return response 

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def http_put(self, path, data=None, **kwargs):
        """
        Make a PUT request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.put(url, data=data, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem putting data to " \
                      "the Kubernetes API server. URL: {}, " \
                      "data: {}".format(url, data)
            logger.error(message)
            raise KubeException(message) from err

        return response 

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def get_links_from_url(url):
    """Download the page at `url` and parse it for links.

    Returned links have had the fragment after `#` removed, and have been made
    absolute so, e.g. the URL 'gen.html#tornado.gen.coroutine' becomes
    'http://www.tornadoweb.org/en/stable/gen.html'.
    """
    try:
        response = yield httpclient.AsyncHTTPClient().fetch(url)
        print('fetched %s' % url)

        html = response.body if isinstance(response.body, str) \
            else response.body.decode()
        urls = [urljoin(url, remove_fragment(new_url))
                for new_url in get_links(html)]
    except Exception as e:
        print('Exception: %s %s' % (e, url))
        raise gen.Return([])

    raise gen.Return(urls) 

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def get_links_from_url(url):
    """Download the page at `url` and parse it for links.

    Returned links have had the fragment after `#` removed, and have been made
    absolute so, e.g. the URL 'gen.html#tornado.gen.coroutine' becomes
    'http://www.tornadoweb.org/en/stable/gen.html'.
    """
    try:
        response = yield httpclient.AsyncHTTPClient().fetch(url)#获取到
        print('fetched %s' % url)


        html = response.body if isinstance(response.body, str) \
            else response.body.decode()
        urls = [urljoin(url, remove_fragment(new_url))
                for new_url in get_links(html)]
    except Exception as e:
        print('Exception: %s %s' % (e, url))
        raise gen.Return([])

    raise gen.Return(urls) 

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def test_static_cache_headers(conf, requests_session):
    """Test that all scripts included from self-repair have long lived cache headers"""
    req = requests_session.get(conf.getoption("server") + "/en-US/repair")
    req.raise_for_status()
    document = html5lib.parse(req.content, treebuilder="dom")
    scripts = document.getElementsByTagName("script")
    for script in scripts:
        src = script.getAttribute("src")
        url = urljoin(conf.getoption("server"), src)
        script_req = requests_session.get(url)
        script_req.raise_for_status()
        cache_control = parse_cache_control(script_req.headers["cache-control"])
        assert cache_control["public"], f"Cache-control: public for {url}"
        ONE_YEAR = 31_536_000
        assert cache_control["max-age"] >= ONE_YEAR, f"Cache-control: max-age > 1 year for {url}"
        assert cache_control["immutable"], f"Cache-control: immutable for {url}" 

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def create_new_user(requests_session, server, headers):
    # Get a list of groups and grab the ID of the first one
    response = requests_session.get(urljoin(server, "/api/v3/group/"), headers=headers)
    group_id = response.json()["results"][0]["id"]
    group_name = response.json()["results"][0]["name"]
    fake = Faker()

    # Create a user, assigning them to the group we obtained
    user_data = {
        "first_name": fake.first_name(),
        "last_name": fake.last_name(),
        "email": fake.company_email(),
        "groups": {"id": group_id, "name": group_name},
    }
    response = requests_session.post(
        urljoin(server, "/api/v3/user/"), headers=headers, data=user_data
    )

    return {
        "id": response.json()["id"],
        "first_name": response.json()["first_name"],
        "last_name": response.json()["last_name"],
        "group_id": group_id,
    } 

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def new_recipe(requests_session, action_id, server, headers):
    urllib3.disable_warnings()

    # Create a recipe
    recipe_data = {
        "action_id": action_id,
        "arguments": '{"learnMoreMessage":"This field may not be blank.","learnMoreUrl":"This field may not be blank.","message":"This field may not be blank.","postAnswerUrl":"This field may not be blank.","surveyId":"'
        + str(uuid.uuid4())
        + '","thanksMessage":"This field may not be blank."}',
        "name": "test recipe",
        "extra_filter_expression": "counter == 0",
        "enabled": "false",
    }

    response = requests_session.post(
        urljoin(server, "/api/v3/recipe/"), data=recipe_data, headers=headers
    )
    data = response.json()
    return {"id": data["id"], "latest_revision_id": data["latest_revision"]["id"]} 

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def __init__(self, account, log_level=None, api_base_url=GODADDY_API_BASE_URL, api_version=GODADDY_API_VERSION):
        """Create a new `godaddypy.Client` object

        :type account: godaddypy.Account
        :param account: The godaddypy.Account object to create auth headers with.
        """

        # Logging setup
        self.logger = logging.getLogger('GoDaddyPy.Client')
        # Explicit override of logging level
        if log_level is not None:
            self.logger.setLevel(log_level)

        # Templates
        self.API_TEMPLATE = urljoin(api_base_url, api_version)
        self.DOMAINS = '/domains'
        self.DOMAIN_INFO = '/domains/{domain}'
        self.RECORDS = '/domains/{domain}/records'
        self.RECORDS_TYPE = '/domains/{domain}/records/{type}'
        self.RECORDS_TYPE_NAME = '/domains/{domain}/records/{type}/{name}'

        self.account = account 

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def as_requests_kwargs(self, base_url: Optional[str] = None) -> Dict[str, Any]:
        """Convert the case into a dictionary acceptable by requests."""
        base_url = self._get_base_url(base_url)
        formatted_path = self.formatted_path.lstrip("/")  # pragma: no mutate
        url = urljoin(base_url + "/", formatted_path)
        # Form data and body are mutually exclusive
        extra: Dict[str, Optional[Body]]
        if self.form_data:
            extra = {"files": self.form_data}
        elif is_multipart(self.body):
            extra = {"data": self.body}
        else:
            extra = {"json": self.body}
        return {
            "method": self.method,
            "url": url,
            "cookies": self.cookies,
            "headers": self.headers,
            "params": self.query,
            **extra,
        } 

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def extract_real_url_from_embedded_url(embedded_url):
    """
    将 embed_real_url_to_embedded_url() 编码后的url转换为原来的带有参数的url
    `cdn_redirect_encode_query_str_into_url`设置依赖于本函数, 详细说明请看配置文件中这个参数的部分

    eg: https://cdn.domain.com/a.php_zm24_.cT1zb21ldGhpbmc=._zm24_.css
        ---> https://foo.com/a.php?q=something (assume it returns an css) (base64 only)
    eg2: https://cdn.domain.com/a/b/_zm24_.bG92ZT1saXZl._zm24_.jpg
        ---> https://foo.com/a/b/?love=live (assume it returns an jpg) (base64 only)
    eg3: https://cdn.domain.com/a/b/_zm24z_.[some long long base64 encoded string]._zm24_.jpg
        ---> https://foo.com/a/b/?love=live[and a long long query string] (assume it returns an jpg) (gzip + base64)
    eg4:https://cdn.domain.com/a  (no change)
        ---> (no query string): https://foo.com/a (assume it returns an png) (no change)
    :param embedded_url: 可能被编码的URL
    :return: 如果传入的是编码后的URL, 则返回解码后的URL, 否则返回None
    :type embedded_url: str
    :rtype: Union[str, None]
    """
    if '._' + cdn_url_query_encode_salt + '_.' not in embedded_url[-15:]:  # check url mark
        return None
    m = regex_extract_base64_from_embedded_url.search(embedded_url)
    b64 = get_group('b64', m)

    # 'https://cdn.domain.com/a.php_zm24_.cT1zb21ldGhpbmc=._zm24_.css'
    # real_request_url_no_query ---> 'https://cdn.domain.com/a.php'
    real_request_url_no_query = embedded_url[:m.span()[0]]

    query_string_byte = base64.urlsafe_b64decode(b64)
    is_gzipped = get_group('gzip', m)
    if is_gzipped:
        query_string_byte = zlib.decompress(query_string_byte)
    query_string = query_string_byte.decode(encoding='utf-8')

    result = urljoin(real_request_url_no_query, '?' + query_string)
    # dbgprint('extract:', embedded_url, 'to', result)
    return result 

# 需要导入模块: from urllib import parse [as 别名]
# 或者: from urllib.parse import urljoin [as 别名]
def assemble_remote_url():
    """
    组装目标服务器URL, 即生成 parse.remote_url 的值
    :rtype: str
    """
    if parse.is_external_domain:
        # 请求的是外部域名 (external domains)
        scheme = 'https://' if parse.is_https else 'http://'
        return urljoin(scheme + parse.remote_domain, parse.remote_path_query)
    else:
        # 请求的是主域名及可以被当做(alias)主域名的域名
        return urljoin(target_scheme + target_domain, parse.remote_path_query)