Python urlobject.URLObject类代码示例

def get_pulls(owner_repo, labels=None, state="open", since=None, org=False):
    url = URLObject("https://api.github.com/repos/{}/issues".format(owner_repo))
    if labels:
        url = url.set_query_param('labels', ",".join(labels))
    if since:
        url = url.set_query_param('since', since.isoformat())
    if state:
        url = url.set_query_param('state', state)
    url = url.set_query_param('sort', 'updated')

    org_fn = None
    if org:
        try:
            with open("people.yaml") as fpeople:
                people = yaml.load(fpeople)
            def_org = "other"
        except IOError:
            people = {}
            def_org = "---"

        def org_fn(issue):
            user_info = people.get(issue["user.login"])
            if not user_info:
                user_info = {"institution": "unsigned"}
            return user_info.get("institution", def_org)

    issues = JPullRequest.from_json(paginated_get(url), org_fn)
    if org:
        issues = sorted(issues, key=operator.itemgetter("org"))

    return issues

def process_url(url, tracking_id):
    amazon_url = URLObject(url)

    amazon_url = amazon_url.with_scheme('https')
    amazon_url = amazon_url.set_query_param('tag', tracking_id)

    return amazon_url

def proxied(url):
    url = URLObject(url)
    netloc = url.netloc or settings.SERVER_NAME
    cache = get_cache()
    if netloc not in cache:
        return url
    return url.with_netloc(cache[netloc])

def get_pulls(labels=None, state="open", since=None, org=False):
    url = URLObject("https://api.github.com/repos/edx/edx-platform/issues")
    if labels:
        url = url.set_query_param('labels', ",".join(labels))
    if since:
        url = url.set_query_param('since', since.isoformat())
    if state:
        url = url.set_query_param('state', state)
    url = url.set_query_param('sort', 'updated')

    org_fn = None
    if org:
        try:
            with open("mapping.yaml") as fmapping:
                user_mapping = yaml.load(fmapping)
            def_org = "other"
        except IOError:
            user_mapping = {}
            def_org = "---"

        def org_fn(issue):
            return user_mapping.get(issue["user.login"], {}).get("institution", def_org)

    issues = JPullRequest.from_json(paginated_get(url), org_fn)
    if org:
        issues = sorted(issues, key=operator.itemgetter("org"))

    return issues

 def test_multiple_parses_are_idempotent(self):
     url = u'http://xn-hllo-bpa.com/path%20withspaces?query=es%25capes&foo=bar#frag%28withescapes%29'
     parse1 = URLObject.parse(url)
     self.assertEqual(unicode(url), unicode(parse1))
     parse2 = URLObject.parse(unicode(parse1))
     self.assertEqual(unicode(url), unicode(parse2))
     self.assertEqual(unicode(parse1), unicode(parse2))

    def as_value(self, data, context):
        # The queries of the current URL, not using sequences here
        # since the order of sorting arguments matter
        url = URLObject(context['request'].get_full_path())
        queries = url.query.dict

        name, orderings = data['with'], data['by']
        query = self.find_query(queries.get(name), orderings, orderings[0])
        url = url.set_query_param(name, query)

        # If this isn't a block tag we probably only want the URL
        if not self._meta.block:
            return url

        label = self.nodelist.render(context)
        if not label.strip():
            raise TemplateSyntaxError("No label was specified")

        parts = []
        for part in query.split(','):
            part = part.strip()
            if part.startswith('-'):
                part = part.lstrip('-')
                # Translators: Used in title of descending sort fields
                text = _("'%(sort_field)s' (desc)")
            else:
                # Translators: Used in title of ascending sort fields
                text = _("'%(sort_field)s' (asc)")
            parts.append(text % {'sort_field': part})
        # Translators: Used for the link/form input title excluding the sort fields
        title = (_('Sort by: %(sort_fields)s') %
                 {'sort_fields': get_text_list(parts, _('and'))})

        extra_context = dict(data, title=title, label=label, url=url, query=query)
        return render_to_string(self.using(data), extra_context, context)

def show_pulls(jrep, labels=None, show_comments=False, state="open", since=None, org=False):
    issues = get_pulls(labels, state, since, org)

    category = None
    for index, issue in enumerate(issues):
        issue.finish_loading()
        if issue.get("org") != category:
            # new category! print category header
            category = issue["org"]
            print("-- {category} ----".format(category=category))

        if 0:
            import pprint
            pprint.pprint(issue.obj)
        print(issue.format(ISSUE_FMT))

        if show_comments:
            comments_url = URLObject(issue['comments_url'])
            comments_url = comments_url.set_query_param("sort", "created")
            comments_url = comments_url.set_query_param("direction", "desc")
            comments = paginated_get(comments_url)
            last_five_comments = reversed(more_itertools.take(5, comments))
            for comment in last_five_comments:
                print(comment.format(COMMENT_FMT))

    # index is now set to the total number of pull requests
    print()
    print("{num} pull requests".format(num=index+1))

    def process_request(self, request):
        token = request.GET.get(self.param_name)
        if not token:
            return

        redirect_url = URLObject(request.get_full_path())
        redirect_url = redirect_url.del_query_param(self.param_name)

        response = redirect(unicode(redirect_url))
        try:
            token_data = tempus_loads(token, max_age=self.max_age)
            tempus = getattr(request, 'tempus', None)
            if tempus:
                current_tempus = tempus.copy()
                current_tempus.update(token_data)
                request.tempus = current_tempus
            else:
                request.tempus = token_data
        except SignatureExpired:
            value = self.__process_func(request, 'expired_func')
            if value:
                return value
        except BadSignature:
            value = self.__process_func(request, 'unsuccess_func')
            if value:
                return value
        else:
            value = self.__process_func(request, 'success_func')
            if value:
                return value

        add_never_cache_headers(response)
        return response

def jira_paginated_get(url, obj_name, session=None, start=0, retries=3, **fields):
    """
    Like ``paginated_get``, but uses JIRA's conventions for a paginated API, which
    are different from Github's conventions.
    """
    session = session or requests.Session()
    url = URLObject(url)
    more_results = True
    while more_results:
        result_url = (
            url.set_query_param("startAt", str(start))
               .set_query_params(**fields)
        )
        for _ in xrange(retries):
            try:
                result_resp = session.get(result_url)
                result = result_resp.json()
                break
            except ValueError:
                continue
        if not result_resp.ok:
            raise requests.exceptions.RequestException(result)
        result = result_resp.json()
        for obj in result[obj_name]:
            yield obj
        returned = len(result[obj_name])
        total = result["total"]
        if start + returned < total:
            start += returned
        else:
            more_results = False

def jira_group_members(groupname, session=None, start=0, retries=3, debug=False):
    """
    JIRA's group members API is horrible. This makes it easier to use.
    """
    session = session or requests.Session()
    url = URLObject("/rest/api/2/group").set_query_param("groupname", groupname)
    more_results = True
    while more_results:
        end = start + 49  # max 50 users per page
        expand = "users[{start}:{end}]".format(start=start, end=end)
        result_url = url.set_query_param("expand", expand)
        for _ in xrange(retries):
            try:
                if debug:
                    print(result_url, file=sys.stderr)
                result_resp = session.get(result_url)
                result = result_resp.json()
                break
            except ValueError:
                continue
        result_resp.raise_for_status()
        result = result_resp.json()
        if not result:
            break
        users = result["users"]["items"]
        for user in users:
            yield user
        returned = len(users)
        total = result["users"]["size"]
        if start + returned < total:
            start += returned
        else:
            more_results = False

def should_transition(issue):
    """
    Return a boolean indicating if the given issue should be transitioned
    automatically from "Needs Triage" to an open status.
    """
    issue_key = to_unicode(issue["key"])
    issue_status = to_unicode(issue["fields"]["status"]["name"])
    project_key = to_unicode(issue["fields"]["project"]["key"])
    if issue_status != "Needs Triage":
        print(
            "{key} has status {status}, does not need to be processed".format(
                key=issue_key, status=issue_status,
            ),
            file=sys.stderr,
        )
        return False

    # Open source pull requests do not skip Needs Triage.
    # However, if someone creates a subtask on an OSPR issue, that subtasks
    # might skip Needs Triage (it just follows the rest of the logic in this
    # function.)
    is_subtask = issue["fields"]["issuetype"]["subtask"]
    if project_key == "OSPR" and not is_subtask:
        print(
            "{key} is an open source pull request, and does not need to be processed.".format(
                key=issue_key
            ),
            file=sys.stderr,
        )
        return False

    user_url = URLObject(issue["fields"]["creator"]["self"])
    user_url = user_url.set_query_param("expand", "groups")

    user_resp = jira_get(user_url)
    if not user_resp.ok:
        raise requests.exceptions.RequestException(user_resp.text)

    user = user_resp.json()
    user_group_map = {g["name"]: g["self"] for g in user["groups"]["items"]}
    user_groups = set(user_group_map)

    exempt_groups = {
        # group name: set of projects that they can create non-triage issues
        "edx-employees": {"ALL"},
        "clarice": {"MOB"},
        "bnotions": {"MOB"},
        "opencraft": {"SOL"},
    }
    for user_group in user_groups:
        if user_group not in exempt_groups:
            continue
        exempt_projects = exempt_groups[user_group]
        if "ALL" in exempt_projects:
            return True
        if project_key in exempt_projects:
            return True

    return False

    def test_with_auth_with_two_args_replaces_whole_auth_string_with_username_and_password(self):
        # Replaces username-only auth string
        url = URLObject('https://[email protected]/')
        assert url.with_auth('zack', '1234') == 'https://zack:[email protected]/'

        # Replaces username and password.
        url = URLObject('https://alice:[email protected]/')
        assert url.with_auth('zack', '1234') == 'https://zack:[email protected]/'

    def clean_url(self):
        url = URLObject(self.cleaned_data["url"])

        # URLObject doesn't handle ipv6 very well yet. In the meantime, ...
        if url.netloc.count(":") > 3:
            raise forms.ValidationError(_("Enter a valid URL."))

        URLValidator()(url.without_auth())
        if url.scheme not in ["http", "https"]:
            raise forms.ValidationError(
                _("Invalid URL scheme: '%s'. Only HTTP and HTTPS are " "supported.") % url.scheme
            )

        if url.netloc.hostname in ["localhost", "127.0.0.1", "::1"]:
            raise forms.ValidationError(_("Enter a valid URL."))

        try:
            validate_ipv46_address(url.netloc.hostname)
        except forms.ValidationError:
            pass
        else:
            raise forms.ValidationError(_("Enter a valid URL."))

        existing = self.user.feeds.filter(url=url)
        if self.instance is not None:
            existing = existing.exclude(pk=self.instance.pk)

        if existing.exists():
            raise forms.ValidationError(_("It seems you're already subscribed to this feed."))

        auth = None
        if url.auth != (None, None):
            auth = url.auth

        # Check this is actually a feed
        with user_lock("feed_check", self.user.pk, timeout=30):
            headers = {"User-Agent": USER_AGENT % "checking feed", "Accept": feedparser.ACCEPT_HEADER}
            try:
                response = requests.get(six.text_type(url.without_auth()), headers=headers, timeout=10, auth=auth)
            except Exception:
                if "SENTRY_DSN" in os.environ:
                    client = Client()
                    client.captureException()
                raise forms.ValidationError(_("Error fetching the feed."))
            if response.status_code != 200:
                raise forms.ValidationError(_("Invalid response code from URL: " "HTTP %s.") % response.status_code)
        try:
            parsed = feedparser.parse(response.content)
        except Exception:
            raise forms.ValidationError(_("Error parsing the feed."))
        if not is_feed(parsed):
            raise forms.ValidationError(_("This URL doesn't seem to be a valid feed."))
        self.cleaned_data["title"] = parsed.feed.title
        # Cache this in case update_favicon needs it and it's not in the
        # scheduler data yet.
        if hasattr(parsed.feed, "link"):
            cache.set(u"feed_link:{0}".format(url), parsed.feed.link, 600)
        return url

    def authorized(self):
        if "next" in request.args:
            next_url = request.args["next"]
        elif self.redirect_url:
            next_url = self.redirect_url
        elif self.redirect_to:
            next_url = url_for(self.redirect_to)
        else:
            next_url = "/"

        # check for error in request args
        error = request.args.get("error")
        if error:
            error_desc = request.args.get("error_description")
            error_uri = request.args.get("error_uri")
            log.warning(
                "OAuth 2 authorization error: %s description: %s uri: %s",
                error, error_desc, error_uri,
            )
            oauth_error.send(self,
                error=error, error_description=error_desc, error_uri=error_uri,
            )
            return redirect(next_url)

        state_key = "{bp.name}_oauth_state".format(bp=self)
        self.session._state = flask.session[state_key]
        del flask.session[state_key]

        secure = request.is_secure or request.headers.get("X-Forwarded-Proto", "http") == "https"
        self.session.redirect_uri = url_for(
            ".authorized", next=request.args.get('next'), _external=True,
            _scheme="https" if secure else "http",
        )

        url = URLObject(request.url)
        if request.headers.get("X-Forwarded-Proto", "http") == "https":
            url = url.with_scheme("https")
        try:
            token = self.session.fetch_token(
                self.token_url,
                authorization_response=url,
                client_secret=self.client_secret,
                **self.token_url_params
            )
        except MissingCodeError as e:
            e.args = (
                e.args[0],
                "The redirect request did not contain the expected parameters. Instead I got: {}".format(
                    json.dumps(request.args)
                )
            )
            raise
        results = oauth_authorized.send(self, token=token) or []
        if not any(ret == False for func, ret in results):
            self.token = token
        return redirect(next_url)

def urls_are_equal(url1, url2):
    """
    Compare to URLs for equality, ignoring the ordering of non-ordered elements.
    """
    url1 = URLObject(url1)
    url2 = URLObject(url2)
    return (
        url1.without_query() == url2.without_query() and
        url1.query_multi_dict == url2.query_multi_dict
    )