Python urllib2.urlopen方法代码示例

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def post_request(vals, url):
    """
    Build a post request.

    Args:
        vals: Dictionary of (field, values) for the POST
            request.
        url: URL to send the data to.

    Returns:
        Dictionary of JSON response or error info.
    """
    # Build the request and send to server
    data = urllib.urlencode(vals)
    
    try:
        request  = urllib2.Request(url, data)
        response = urllib2.urlopen(request)
    except urllib2.HTTPError, err:
        return {"error": err.reason, "error_code": err.code} 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def jenkins(url, port):
    try:
        cli_port = False
        ctx = ssl.create_default_context()
        ctx.check_hostname = False
        ctx.verify_mode = ssl.CERT_NONE
        try:
            output = urllib2.urlopen('https://'+url+':'+port+"/jenkins/", context=ctx, timeout=8).info()
            cli_port = int(output['X-Jenkins-CLI-Port'])
        except urllib2.HTTPError, e:
            if e.getcode() == 404:
                try:
                    output = urllib2.urlopen('https://'+url+':'+port, context=ctx, timeout=8).info()
                    cli_port = int(output['X-Jenkins-CLI-Port'])
                except:
                    pass
        except:
            pass 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def run(self):
		request = self.request
		try:
			if ((timeit.default_timer() - self.starttime) <= self.timeout and
					not SHUTDOWN_EVENT.isSet()):
				try:
					f = urlopen(request)
				except TypeError:
					# PY24 expects a string or buffer
					# This also causes issues with Ctrl-C, but we will concede
					# for the moment that Ctrl-C on PY24 isn't immediate
					request = build_request(self.request.get_full_url(),
											data=request.data.read(self.size))
					f = urlopen(request)
				f.read(11)
				f.close()
				self.result = sum(self.request.data.total)
			else:
				self.result = 0
		except (IOError, SpeedtestUploadTimeout):
			self.result = sum(self.request.data.total) 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def download_image(image_id, url, x1, y1, x2, y2, output_dir):
    """Downloads one image, crops it, resizes it and saves it locally."""
    output_filename = os.path.join(output_dir, image_id + '.png')
    if os.path.exists(output_filename):
        # Don't download image if it's already there
        return True
    try:
        # Download image
        url_file = urlopen(url)
        if url_file.getcode() != 200:
            return False
        image_buffer = url_file.read()
        # Crop, resize and save image
        image = Image.open(BytesIO(image_buffer)).convert('RGB')
        w = image.size[0]
        h = image.size[1]
        image = image.crop((int(x1 * w), int(y1 * h), int(x2 * w),
                            int(y2 * h)))
        image = image.resize((299, 299), resample=Image.ANTIALIAS)
        image.save(output_filename)
    except IOError:
        return False
    return True 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def verifyChecksumForDir(dirName, settings, type):
    """
        For each file listed in a list, verifies its checksum.
    """

    dirName = os.path.join(settings.getLocalDst(type), dirName)

    urlChecksumFile = settings.getRemoteSrc() + '/' + os.path.basename(dirName) + '.checksum'
    print("Verification of the decompressed directory '%s' started." % dirName)
    try:
        for line in urllib2.urlopen(urlChecksumFile):
            f, checksum = line.split('\t')
            f = os.path.join(dirName, f)
            # if checksum.strip() != hashlib.md5(open(f).read()).hexdigest():
            if checksum.strip() != getChecksumForFile(f):
                raise Exception("File '%s' is corrupted, it has a wrong checksum." % f)
    except Exception as e:
        print("Unable to verify directory: %s" % dirName)
        raise e

    print("Checksum verification completed successfully!") 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def download_genome(genome, out_path):
    genome_path = os.path.join(out_path,"genomes")
    out_name = genome.rstrip().split('/')[-1]
    http_address = os.path.join(genome, out_name + "_genomic.fna.gz")
    opened = urllib2.urlopen(http_address)
    out = os.path.join(genome_path, out_name + ".fa")
    tmp_out = os.path.join(genome_path, out_name + "tmp.fa")
    out_gz = out + ".gz"
    with open(out_gz,'wb') as outF:
        outF.write(opened.read())
    gf = gzip.open(out_gz)
    new_out = open(tmp_out,'wb')
    new_out.write(gf.read())
    gf.close()
    os.remove(out_gz)
    new_out.close()
    split_by_N(tmp_out, out)
    return out 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def test_connection(name, url, timeout=10):
    """Simple connection test"""
    urlinfo = urlparse(url)
    start = time.time()
    try:
        ip = socket.gethostbyname(urlinfo.netloc)
    except Exception as e:
        print('Error resolving DNS for {}: {}, {}'.format(name, url, e))
        return
    dns_elapsed = time.time() - start
    start = time.time()
    try:
        _ = urlopen(url, timeout=timeout)
    except Exception as e:
        print("Error open {}: {}, {}, DNS finished in {} sec.".format(name, url, e, dns_elapsed))
        return
    load_elapsed = time.time() - start
    print("Timing for {}: {}, DNS: {:.4f} sec, LOAD: {:.4f} sec.".format(name, url, dns_elapsed, load_elapsed)) 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def gethtml(url, lastURL=False):
    """return HTML of the given url"""

    if not (url.startswith("http://") or url.startswith("https://")):
        url = "http://" + url

    header = useragents.get()
    request = urllib2.Request(url, None, header)
    html = None

    try:
        reply = urllib2.urlopen(request, timeout=10)

    except urllib2.HTTPError, e:
        # read html content anyway for reply with HTTP500
        if e.getcode() == 500:
            html = e.read()
        #print >> sys.stderr, "[{}] HTTP error".format(e.code)
        pass 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def reverseip(url):
    """return domains from given the same server"""

    # get only domain name
    url = urlparse(url).netloc if urlparse(url).netloc != '' else urlparse(url).path.split("/")[0]

    source = "http://domains.yougetsignal.com/domains.php"
    useragent = useragents.get()
    contenttype = "application/x-www-form-urlencoded; charset=UTF-8"

    # POST method
    opener = urllib2.build_opener(
        urllib2.HTTPHandler(), urllib2.HTTPSHandler())
    data = urllib.urlencode([('remoteAddress', url), ('key', '')])

    request = urllib2.Request(source, data)
    request.add_header("Content-type", contenttype)
    request.add_header("User-Agent", useragent)

    try:
        result = urllib2.urlopen(request).read()

    except urllib2.HTTPError, e:
        print >> sys.stderr, "[{}] HTTP error".format(e.code) 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def search(self, query, per_page=10, pages=1):
        """search urls from yahoo search"""

        # store searched urls
        urls = []

        for page in range(pages):
            yahoosearch = self.yahoosearch % (query, per_page, (pages+1)*10)

            request = urllib2.Request(yahoosearch)
            request.add_header("Content-type", self.contenttype)
            request.add_header("User-Agent", self.useragent)

            result = urllib2.urlopen(request).read()
            urls += self.parse_links(result)

        return urls 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def DownloadFile(self, fileurl, dlfile):
    """Downloads a given file to a given path/filename.

    Args:
      fileurl: String with URL of file to download.
      dlfile: String with path of file to be written to.
    Raises:
      OSError: If file cannot be opened/written to, function raises OSError.
      URLError: If URL cannot be opened, fucntion raises URLError.
    """
    if not os.path.isfile(dlfile) or dlfile == TMPINDEX:
      print 'Downloading %s ...' % fileurl
      file_to_dl = urllib2.urlopen(fileurl)
      tmpfile = open(dlfile, 'wb')
      shutil.copyfileobj(file_to_dl, tmpfile)
    else:
      print '%s exists' % dlfile 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def subdomain_check(subdomains):
    # Basic query
    for subd in range(len(subdomains)):
        if subd != 0:
            try:
                #print("inside query")
                for rdata in dns.resolver.query(subdomains[subd], 'CNAME') :
                    print "Checking subdomain takeover on:  "+str(subdomains[subd])
                    try:
                        #response = urlopen("http://"+str(rdata.target))
                        response = urlopen("http://"+str(subdomains[subd]))
                        print(R+str(subdomains[subd])+"  seems Up and running fine")
                    except:
                        print(S+"Success!!! Possible sub-domain takeover on:     "+str(subdomains[subd])+E)
            except:
                print (R+"No CNAME for"+str(subdomains[subd])+"i.e. subdomain takeover not Possible") 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def query(owner, name):
    if fake:
        print '    {0}/{1}: ok'.format(owner, name)
        return (random.randint(1, 1000), random.randint(1, 300))
    else:
        try:
            req = urllib2.Request('https://api.github.com/repos/{0}/{1}'.format(owner, name))
            if user is not None and token is not None:
                b64 = base64.encodestring('{0}:{1}'.format(user, token)).replace('\n', '')
                req.add_header("Authorization", "Basic {0}".format(b64))
            u = urllib2.urlopen(req)
            j = json.load(u)
            t = datetime.datetime.strptime(j['updated_at'], "%Y-%m-%dT%H:%M:%SZ")
            days = max(int((now - t).days), 0)
            print '    {0}/{1}: ok'.format(owner, name)
            return (int(j['stargazers_count']), days)
        except urllib2.HTTPError, e:
            print '    {0}/{1}: FAILED'.format(owner, name)
            return (None, None) 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def upd(category, sort, str):
		post = urllib.urlencode({'checkbox_ftp':'on', 'checkbox_tor':'on','word':str}) 
		request = urllib2.Request('http://krasfs.ru/search.php?key=newkey')#url, post)

		request.add_header('User-Agent', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET4.0C)') 
		request.add_header('Host',    'www.krasfs.ru') 
		request.add_header('Accept', '*/*') 
		request.add_header('Accept-Language', 'ru-RU') 
		request.add_header('Referer',    'http://www.krasfs.ru') 

		try: 
			f = urllib2.urlopen(request) 
			html = f.read()
			html = html.replace(chr(10),"")
			n=html.find("<newkey>")
			k=html.find("</newkey>")
			key = html[n+8:k]
		except IOError, e: 
			if hasattr(e, 'reason'): 
				print 'We failed to reach a server. Reason: '+ e.reason
			elif hasattr(e, 'code'): 
				print 'The server couldn\'t fulfill the request. Error code: '+ e.code
			key = "59165b78-bf91-11e1-86bf-c6ab051766ba" 

# 需要导入模块: import urllib2 [as 别名]
# 或者: from urllib2 import urlopen [as 别名]
def _http_request(url, headers=None, time_out=10):
    """Perform an HTTP request and return request"""
    log(0, 'Request URL: {url}', url=url)

    try:
        if headers:
            request = Request(url, headers=headers)
        else:
            request = Request(url)
        req = urlopen(request, timeout=time_out)
        log(0, 'Response code: {code}', code=req.getcode())
        if 400 <= req.getcode() < 600:
            raise HTTPError('HTTP %s Error for url: %s' % (req.getcode(), url), response=req)
    except (HTTPError, URLError) as err:
        log(2, 'Download failed with error {}'.format(err))
        if yesno_dialog(localize(30004), '{line1}\n{line2}'.format(line1=localize(30063), line2=localize(30065))):  # Internet down, try again?
            return _http_request(url, headers, time_out)
        return None

    return req