Python requests.head函数代码示例

 def _check_host(self, host):
     """Check is host available"""
     try:
         requests.head(host, timeout=self.timeout)
         return True
     except requests.ConnectionError:
         return False

 def CheckBundleDigest(self, container, uripath, bundle_data):
     """
     Download the Bundle from CloudFiles into a local path
         container - the CloudFiles container in which to find the Vault DB
         bundle_data - a dict containing atlest the 'id'  and 'md5' of the bundle
         localpath - the local path at which to store the downloaded VaultDB
     """
     self.apihost = self._get_container(container)
     try:
         fulluri = uripath + '/BUNDLES/' + bundle_data['name']
         self.ReInit(self.sslenabled, '/' + fulluri)
         self.headers['X-Auth-Token'] = self.authenticator.AuthToken
         self.log.debug('uri: %s', self.Uri)
         self.log.debug('headers: %s', self.Headers)
         try:
             res = requests.head(self.Uri, headers=self.Headers)
         except requests.exceptions.SSLError as ex:
             self.log.error('Requests SSLError: {0}'.format(str(ex)))
             res = requests.head(self.Uri, headers=self.Headers, verify=False)
         if res.status_code == 404:
             raise UserWarning('Server failed to find the specified bundle')
         elif res.status_code >= 300:
             raise UserWarning('Server responded unexpectedly during download (Code: ' + str(res.status_code) + ' )')
         else:
             digest = res.headers['etag'].upper()
             result = (digest == bundle_data['md5'])
             self.log.debug('CloudFiles Bundle Digest (' + digest + ') == Bundle MD5 (' + bundle_data['md5'] + ')? ' + str(result))
             return result
     except LookupError:
         raise UserWarning('Invalid VaultDB Data provided.')

    def fetch_metadata(self):
        """
        Do a HEAD request on self.url to try to get metadata
        (self.length and self.mimetype).

        Note that while this method fills in those attributes, it does *not*
        call self.save() - so be sure to do so after calling this method!

        """
        if not self.url:
            return

        try:
            response = requests.head(self.url, timeout=5)
            if response.status_code == 302:
                response = requests.head(response.headers['location'],
                                         timeout=5)
        except Exception:
            pass
        else:
            if response.status_code != 200:
                return
            self.length = response.headers.get('content-length')
            self.mimetype = response.headers.get('content-type', '')
            if self.mimetype in ('application/octet-stream', ''):
                # We got a not-useful MIME type; guess!
                guess = mimetypes.guess_type(self.url)
                if guess[0] is not None:
                    self.mimetype = guess[0]

    def __init__(self, cfg=None, service_url=None, service_mode=None):
        """

        :param cfg: Configuration for the service
        :param service_url: The service URL
        :param service_mode: set to "private" if imeji instance runs in "private" mode
           (any other value considered as standard imeji instance mode )

        If the imeji instance is not available or does not run, the instantiation will throw an error message.
        """
        self.cfg = cfg or Config()
        self.service_url = service_url or self.cfg.get('service', 'url')
        self.service_mode_private = False or (self.cfg.get('service', 'mode', 'public') == 'private' or service_mode == 'private')
        self.service_unavailable_message = \
            "WARNING : The REST Interface of Imeji at {rest_service} is not available or there is another problem, " \
            "check if the service is running under {imeji_service}" \
                .format(imeji_service=self.service_url, rest_service=self.service_url + '/rest')

        # check if Imeji instance is running and notify the user
        try:
            requests.head(self.service_url)
        except Exception as e:
            raise ImejiError(self.service_unavailable_message, e)

        user = self.cfg.get('service', 'user', default=None)
        password = self.cfg.get('service', 'password', default=None)
        self.session = requests.Session()
        if user and password:
            self.session.auth = (user, password)
        # initialize the request query
        self.total_number_of_results = self.number_of_results = self.offset = self.size = None

 def __init__(self, server, userAgent=_getUserAgent()):
     self.server = server
     self.userAgent = userAgent
     try:
         requests.head(self.server)
     except requests.exceptions.ConnectionError, e:
         raise ValueError("server %s does not look to be alive: %s" % (server, e.message))

 def find_url_title(self, url):
     """Retrieve the title of a given URL"""
     headers = {'User-Agent': 'Wget/1.13.4 (linux-gnu)'}
     if url.find("://") == -1:
         url = "http://" + url
     try:
         # a HEAD first to thwart attacks
         requests.head(url, headers=headers, timeout=5)
         # now the actual request
         resp = requests.get(url, headers=headers)
         html = resp.text
     except requests.RequestException as e:
         self.logger.warning(e)
         return url, e.__doc__
     except ValueError as e:
         self.logger.warning(e)
         return url, "Failed to parse url"
     else:
         resp.close()
         cmphtml = html.lower()
         start = cmphtml.find("<title")
         end = cmphtml.find("</title>")
         if start == -1 or end == -1:
             return resp.url, "Could not find page title!"
         else:
             str.find
             html = html[start+7:end]
             html = html[html.find('>')+1:]
             return resp.url, html.strip()

 def request(self, url, request_method=None, auth_method=None, timeout=None, post_data=None,
             user=None, password=None):
     timeout = float(timeout)
     try:
         if request_method == "1":
             # GET
             if (user or password) and auth_method == "2":
                 req = requests.get(url, auth=(user, password), timeout=timeout, verify=False)
             else:
                 req = requests.get(url, timeout=timeout, verify=False)
         elif request_method == "2":
             # POST
             if (user or password) and auth_method == "2":
                 req = requests.post(url, data=post_data, auth=(user, password), timeout=timeout, verify=False)
             else:
                 req = requests.post(url, data=post_data, timeout=timeout, verify=False)
         elif request_method == "3":
             # HEAD
             if (user or password) and auth_method == "2":
                 req = requests.head(url, auth=(user, password), timeout=timeout, verify=False)
             else:
                 req = requests.head(url, timeout=timeout, verify=False)
         time = req.elapsed
     except Exception as e:
         logging.error(e)
         raise
     try:
         code = req.status_code
         response_time = time.microseconds / 1000
     except Exception as e:
         logging.error(e)
         raise
     data = [int(code), float(response_time)]
     return data

def EXTRACT_MP3_LINKS(url):
    header = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:32.0) Gecko/20100101 Firefox/32.0',}
    page = requests.get(url,header)
    soup = BeautifulSoup(page.content)
    links = soup.findAll("a")
    #print type (links)
    for item in links:
        try:
            if ".mp3" in item['href']:
                try:
                    response=requests.head(item['href'])

                    if response.headers['content-type']=='audio/mpeg':
                        SAVEMP3(item['href'])
                except:
                    pass
                try:
                    response=requests.head(url+item['href'])
                    #print response
                    if response.headers['content-type']=='audio/mpeg':
                        SAVEMP3(url+item['href'])
                except:
                    pass
        except:
            pass

def correct_url(url, rights):
    """
correct_url

link checker and guesser for wikipedia thunbnail URLs

returns a checked (good) URL as a unicode string or None
"""
    urlres = requests.head(url, allow_redirects=True)
    # thubmnail URL looks good (check the link first)
    if (urlres.status_code == requests.codes.ok):
        return url

    # something is not right
    # if the attribute page for the image does not exist, then we
    # won't find a thumbnail, so we may as well give up now
    rightsres = requests.head(rights)
    if (rightsres.status_code != requests.codes.ok):
        return None

    # okay, there should be a good thumbnail here, just not at the
    # URL we tried

    elif (urlres.status_code == 404):
        return correct_url_404(url)
    elif (urlres.status_code == 500):
        return correct_url_500(url)
    # not sure we can get here, something might be very wrong
    else:
        raise Exception("wikipedia thumbnail URL {0} had unexpected" +
                        "status code {1}".format(urlres.status_code, url))

def download_vid(vid):
    html_downloader = main.Downloader()
    html_downloader.get(vid[1], vid[2], "html")

    flv_redirect = app.extract_flv_redirect(vid[2])

    headers = {
        "Accept-Encoding": "gzip,deflate,sdch",
        "Host": "redirector.googlevideo.com",
        "Accept-Language": "en-US,en;q=0.8,fr;q=0.6",
        "User-Agent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.66 ari/537.36",
        "Accept": "*/*",
        "Referer": vid[1],
        "Connection": "keep-alive",
        "Cache-Control": "no-cache",
    }

    req = requests.head(flv_redirect, headers=headers)
    file_url = req.headers["location"]

    req = requests.head(flv_redirect, headers=headers)
    file_url = req.headers["location"]

    host = urlparse(file_url).netloc
    headers["Host"] = host

    html_downloader.download_file(file_url, "flv/%s.flv" % vid[2], headers)

def VerifyPath(data):
    # Insert try and catch blocks
    try:
        token_name = data["project"]["token_name"]
    except:
        token_name = data["project"]["project_name"]

    channel_names = data["channels"].keys()

    for i in range(0, len(channel_names)):
        channel_type = data["channels"][channel_names[i]]["channel_type"]
        path = data["channels"][channel_names[i]]["data_url"]

        if channel_type == "timeseries":
            timerange = data["dataset"]["timerange"]
            for j in xrange(timerange[0], timerange[1] + 1):
                # Test for tifs or such? Currently test for just not empty
                work_path = "{}{}/{}/time{}/".format(path, token_name, channel_names[i], j)
                resp = requests.head(work_path)
                assert resp.status_code == 200
        else:
            # Test for tifs or such? Currently test for just not empty
            work_path = "{}{}/{}/".format(path, token_name, channel_names[i])
            resp = requests.head(work_path)
            print (work_path)
            assert resp.status_code == 200

    def __init__(self, spider=None, *a, **kw):
        super(GetpagesfromsitemapSpider, self).__init__(*a, **kw)
        try:
            cnx = mysql.connector.connect(**config)
        except mysql.connector.Error as err:
            if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
                print("Something is wrong with your user name or password")
            elif err.errno == errorcode.ER_BAD_DB_ERROR:
                print("Database does not exists")
            else:
	            print(err)
        else:
            self.spider = spider
            cursor = cnx.cursor()
            l = []
            url = "https://channelstore.roku.com"
            resp = requests.head(url + "/sitemap.xml")
            if (resp.status_code != 404):
                l.append(resp.url)
            else:
                resp = requests.head(url + "/robots.txt")
                if (resp.status_code == 200):
                    l.append(resp.url)
        self.sitemap_urls = l
        print self.sitemap_urls

 def _check_host(self, host):
     """Check is host available"""
     try:
         requests.head(host, **self.request_params)
         return True
     except requests.ConnectionError:
         return False

def has_internet():
    """Uses www.google.com to check connectivity"""
    try:
        requests.head('http://www.google.com', timeout=1)
        return True
    except requests.ConnectionError:
        return False

def forward(resource, identifier):
    """ Redirects request for file to direct URL.

        Requires global "paths" dictionary is active. 

        resource: a given resource, like "recount2"
        identifier: relative path to file or directory

        Return value: Flask redirect response object
    """
    # Log all requests, even weird ones
    ip = str(request.headers.get('X-Forwarded-For',
                        request.remote_addr)).split(',')[0].strip()
    print >>_LOGSTREAM, '\t'.join(
        [time.strftime('%A, %b %d, %Y at %I:%M:%S %p %Z'),
             str(mmh3.hash128(ip + 'recountsalt')),
             resource,
             identifier])
    _LOGSTREAM.flush()
    if resource == 'recount':
        # Redirect to IDIES URL in order of descending version
        for i in ['2']: # add versions to precede 2 as they are released
            if identifier.startswith(' '.join(['v', i, '/'])):
                idies_url = '/'.join(
                            ['http://idies.jhu.edu/recount/data', identifier]
                        )
                idies_response = requests.head(idies_url)
                if idies_response.status_code == 200:
                    return redirect(idies_url, code=302)
        # v1 is not explicitly versioned
        idies_url = '/'.join(['http://idies.jhu.edu/recount/data', identifier])
        idies_response = requests.head(idies_url)
        if idies_response.status_code == 200:
            return redirect(idies_url, code=302)
    abort(404)