Python URLObject.without_auth方法代码示例

# 需要导入模块: from urlobject import URLObject [as 别名]
# 或者: from urlobject.URLObject import without_auth [as 别名]
    def clean_url(self):
        url = URLObject(self.cleaned_data["url"])

        # URLObject doesn't handle ipv6 very well yet. In the meantime, ...
        if url.netloc.count(":") > 3:
            raise forms.ValidationError(_("Enter a valid URL."))

        URLValidator()(url.without_auth())
        if url.scheme not in ["http", "https"]:
            raise forms.ValidationError(
                _("Invalid URL scheme: '%s'. Only HTTP and HTTPS are " "supported.") % url.scheme
            )

        if url.netloc.hostname in ["localhost", "127.0.0.1", "::1"]:
            raise forms.ValidationError(_("Enter a valid URL."))

        try:
            validate_ipv46_address(url.netloc.hostname)
        except forms.ValidationError:
            pass
        else:
            raise forms.ValidationError(_("Enter a valid URL."))

        existing = self.user.feeds.filter(url=url)
        if self.instance is not None:
            existing = existing.exclude(pk=self.instance.pk)

        if existing.exists():
            raise forms.ValidationError(_("It seems you're already subscribed to this feed."))

        auth = None
        if url.auth != (None, None):
            auth = url.auth

        # Check this is actually a feed
        with user_lock("feed_check", self.user.pk, timeout=30):
            headers = {"User-Agent": USER_AGENT % "checking feed", "Accept": feedparser.ACCEPT_HEADER}
            try:
                response = requests.get(six.text_type(url.without_auth()), headers=headers, timeout=10, auth=auth)
            except Exception:
                if "SENTRY_DSN" in os.environ:
                    client = Client()
                    client.captureException()
                raise forms.ValidationError(_("Error fetching the feed."))
            if response.status_code != 200:
                raise forms.ValidationError(_("Invalid response code from URL: " "HTTP %s.") % response.status_code)
        try:
            parsed = feedparser.parse(response.content)
        except Exception:
            raise forms.ValidationError(_("Error parsing the feed."))
        if not is_feed(parsed):
            raise forms.ValidationError(_("This URL doesn't seem to be a valid feed."))
        self.cleaned_data["title"] = parsed.feed.title
        # Cache this in case update_favicon needs it and it's not in the
        # scheduler data yet.
        if hasattr(parsed.feed, "link"):
            cache.set(u"feed_link:{0}".format(url), parsed.feed.link, 600)
        return url

# 需要导入模块: from urlobject import URLObject [as 别名]
# 或者: from urlobject.URLObject import without_auth [as 别名]
 def test_without_auth_removes_entire_auth_string(self):
     # No username or password => no-op.
     url = URLObject('https://github.com/')
     assert url.without_auth() == 'https://github.com/'
     # Username-only.
     url = URLObject('https://[email protected]/')
     assert url.without_auth() == 'https://github.com/'
     # Username and password.
     url = URLObject('https://alice:[email protected]/')
     assert url.without_auth() == 'https://github.com/'

# 需要导入模块: from urlobject import URLObject [as 别名]
# 或者: from urlobject.URLObject import without_auth [as 别名]
    def update_feed(self, url, etag=None, last_modified=None, subscribers=1,
                    backoff_factor=1, previous_error=None, link=None,
                    title=None, hub=None):
        url = URLObject(url)
        # Check if this domain has rate-limiting rules
        ratelimit_key = 'ratelimit:{0}'.format(
            url.netloc.without_auth().without_port())
        retry_at = cache.get(ratelimit_key)
        if retry_at:
            retry_in = (epoch_to_utc(retry_at) - timezone.now()).seconds
            schedule_job(url, schedule_in=retry_in,
                         connection=get_redis_connection())
            return

        if subscribers == 1:
            subscribers_text = '1 subscriber'
        else:
            subscribers_text = '{0} subscribers'.format(subscribers)

        headers = {
            'User-Agent': USER_AGENT % subscribers_text,
            'Accept': feedparser.ACCEPT_HEADER,
        }

        if last_modified:
            headers['If-Modified-Since'] = force_bytes(last_modified)
        if etag:
            headers['If-None-Match'] = force_bytes(etag)

        if settings.TESTS:
            # Make sure requests.get is properly mocked during tests
            if str(type(requests.get)) != "<class 'mock.MagicMock'>":
                raise ValueError("Not Mocked")

        auth = None
        if url.auth != (None, None):
            auth = url.auth

        start = datetime.datetime.now()
        error = None
        try:
            response = requests.get(
                six.text_type(url.without_auth()), headers=headers, auth=auth,
                timeout=UniqueFeed.request_timeout(backoff_factor))
        except (requests.RequestException, socket.timeout, socket.error,
                IncompleteRead, DecodeError) as e:
            logger.debug("Error fetching %s, %s" % (url, str(e)))
            if isinstance(e, IncompleteRead):
                error = UniqueFeed.CONNECTION_ERROR
            elif isinstance(e, DecodeError):
                error = UniqueFeed.DECODE_ERROR
            else:
                error = UniqueFeed.TIMEOUT
            self.backoff_feed(url, error, backoff_factor)
            return
        except LocationParseError:
            logger.debug(u"Failed to parse URL for {0}".format(url))
            self.mute_feed(url, UniqueFeed.PARSE_ERROR)
            return

        elapsed = (datetime.datetime.now() - start).seconds

        ctype = response.headers.get('Content-Type', None)
        if (response.history and
            url != response.url and ctype is not None and (
                ctype.startswith('application') or
                ctype.startswith('text/xml') or
                ctype.startswith('text/rss'))):
            redirection = None
            for index, redirect in enumerate(response.history):
                if redirect.status_code != 301:
                    break
                # Actual redirection is next request's url
                try:
                    redirection = response.history[index + 1].url
                except IndexError:  # next request is final request
                    redirection = response.url

            if redirection is not None and redirection != url:
                self.handle_redirection(url, redirection)

        update = {'last_update': int(time.time())}

        if response.status_code == 410:
            logger.debug(u"Feed gone, {0}".format(url))
            self.mute_feed(url, UniqueFeed.GONE)
            return

        elif response.status_code in [400, 401, 403, 404, 500, 502, 503]:
            self.backoff_feed(url, str(response.status_code), backoff_factor)
            return

        elif response.status_code not in [200, 204, 304]:
            logger.debug(u"{0} returned {1}".format(url, response.status_code))

            if response.status_code == 429:
                # Too Many Requests
                # Prevent next jobs from fetching the URL before retry-after
                retry_in = int(response.headers.get('Retry-After', 60))
                retry_at = timezone.now() + datetime.timedelta(
#.........这里部分代码省略.........