Python net.URL类代码示例

def get(targetURL, params, username=None):

    paramStr = ""
    for aKey in params.keys():
        paramStr+=aKey+"="+URLEncoder.encode(params[aKey], "UTF-8")+"&"
    paramStr=paramStr[:-1]
    url = URL(targetURL+"?"+paramStr)
    print url
    connection = url.openConnection()

    if username!=None:    
        userpass = username
        basicAuth = "Basic " + base64.b64encode(userpass);
        #print basicAuth
        connection.setRequestProperty ("Authorization", basicAuth);
    
    connection.setRequestMethod("GET")    
    connection.setRequestProperty("Content-Language", "en-GB")
    connection.setUseCaches(0)
    connection.setDoOutput(2)
    
    inStream= connection.getInputStream()
    rd= BufferedReader(InputStreamReader(inStream))
    response = ""
    line = rd.readLine()
    while line != None:
        response +=line+"\r"
        line = rd.readLine()
    rd.close()
    return response

def getUrl(WSDLUrl, containerOSH):

    res = ObjectStateHolderVector()
    urlIP = None
    try:
        url = URL(WSDLUrl)
        hostName = url.getHost()
        urlIP = netutils.getHostAddress(hostName, None)

        if (not netutils.isValidIp(urlIP)) or netutils.isLocalIp(urlIP):
            urlIP = None
    except:
        urlIP = None
    
    urlOSH = modeling.createUrlOsh(containerOSH, WSDLUrl, 'wsdl')

    urlIpOSH = None
    if urlIP != None:
        try:
            urlIpOSH = modeling.createIpOSH(urlIP)
        except:
            urlIpOSH = None

    res.add(urlOSH)

    if urlIpOSH:
        res.add(urlIpOSH)
        urlToIpOSH = modeling.createLinkOSH('depend', urlOSH, urlIpOSH)
        res.add(urlToIpOSH)

    return res

def nukeAndPave(islandID, wipeMasters=False, wipeSlaves=False):
    if not wipeMasters and not wipeSlaves:
        return

    clusterMgr = mc.getClusterManager()
    clusters = clusterMgr.enumClusters(islandID)

    slaves = []
    masters = []

    for cluster in clusters:
        locs = clusterMgr.enumClusterLocations(cluster.getID())
        for loc in locs:
            slaveURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_SLAVE_HOST_URL_PROP))
            slaveHost = slaveURL.getHost()
            masterURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_MASTER_HOST_URL_PROP))
            masterHost = masterURL.getHost()
            slaves.append(slaveHost)
            masters.append(masterHost)

    rc = 0
    if wipeSlaves:
        print "Removing Solr from slaves to prepare for upgrade", slaves
        failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "remove-solr-install"], slaves)

        if failures.isEmpty():
            print "INFO: nuke and pave of slaves succeeded"
        else:
            print "WARN: check logs for errors before upgrading slaves. Host(s) reporting falilure:", failures
            rc = 1

    return rc

def post(targetURL, params, contentType="text/xml", username=None):
    
    if(type(params) is dict):
        paramStr = ""
        for aKey in params.keys():
            paramStr+=aKey+"="+URLEncoder.encode(params[aKey], "UTF-8")+"&"
        paramStr=paramStr[:-1]
    else:
        paramStr = params
        
    url = URL(targetURL)
    print targetURL
    print paramStr
    print contentType
    connection = url.openConnection()
    if username!=None:    
        userpass = username
        basicAuth = "Basic " + base64.b64encode(userpass);
        connection.setRequestProperty ("Authorization", basicAuth);
    connection.setRequestMethod("POST")
    if contentType != None:
        connection.setRequestProperty("Content-Type", contentType)
    connection.setRequestProperty("Content-Length", str(len(paramStr)))
    connection.setRequestProperty("Content-Language", "en-GB")
    connection.setUseCaches(0)
    connection.setDoInput(1)
    connection.setDoOutput(2)
    
    wr= DataOutputStream(connection.getOutputStream())
    wr.writeBytes(paramStr)
    wr.flush()
    wr.close()
    return getResponse(connection);

    def crawl(site, trm , depth, linksfile):
        from java.net import URL
        from org.w3c.tidy import Tidy
        pattern = re.compile('href="/wiki/(.*?)"')
        f = open(linksfile, 'a+')
        #try:
        if depth < MAX_DEPTH:
            print 'crawling [%s]...' % trm,
            print >> f, '[%s]' % trm

            td = Tidy()
            td.setXmlOut(1)

            u = URL(site + trm)

            input = BufferedInputStream(u.openStream())
            output = ByteArrayOutputStream()
            #tidy.setInputEncoding("UTF8")
            #tidy.setOutputEncoding("UTF8")

            td.parse(input, output)
            content = output.toString()
            hits = pattern.findall(content)

            for hit in hits:
                if hit.find(":") == -1:
                    print >> f, hit
            print 'done.'
            print >> f, ''
            for hit in hits:
                if hit.find(":") == -1:
                    crawl(site, hit, depth + 1, linksfile)
        #except:
        #    print "wrong"
        f.close()

def isUrlAvailable(url, acceptedStatusCodeRange, timeout=10000):
    '''
    Checks whether url is available
    str, list(str), int -> bool
    '''
    from com.hp.ucmdb.discovery.library.clients import SSLContextManager
    from com.hp.ucmdb.discovery.library.clients.http import ApacheHttpClientWrapper as HttpClientWrapper

    if not url or not acceptedStatusCodeRange:
        return 0
    with _create_http_client_wrapper() as client:
        client.setSocketTimeout(timeout)
        try:
            jurl = URL(url)
            if jurl.getProtocol() == 'https':
                port = jurl.getPort() or HttpClientWrapper.DEFAULT_HTTPS_PORT
                context = SSLContextManager.getAutoAcceptSSLContext()
                client.registerProtocol(context, port)
        except:
            logger.warn('Failed parsing url % ' % url)
        try:
            httpResult = client.get(url)
            return httpResult.statusCode in acceptedStatusCodeRange
        except:
            logger.warn('Get Failed: %s' % logger.prepareJavaStackTrace())

    return 0

	def googleSiteIndex(self, url, mCallBacks, startIndex):
		#print 'Starting Google Site: Index for URL: ' + str(url)
		data = 'Starting Google Site: Index for URL: ' + str(url) + '\n'
		self.parent.printLogTab(str(data))
		googleRequest = self.buildGoogleRequest(url, startIndex)
		googleResponse = mCallBacks.makeHttpRequest('www.google.com', int('80'), False, googleRequest)
		googleStringResponse = googleResponse.tostring()
		for urlInSearch in re.findall(r'''<a href="([^<]+)" class=l''', googleStringResponse):
			uUrl = URL(urlInSearch)
			port = 80
			if str(uUrl.getProtocol()) == "https":
				port = 443
			if mCallBacks.isInScope(uUrl):
				newRequest = self.buildGenericRequest(uUrl)
				newResponse = mCallBacks.makeHttpRequest(str(uUrl.getHost()), port, (str(uUrl.getProtocol()) == "https"), newRequest)
				newResponseString = newResponse.tostring()
				firstWord, statusCode, restResponse = newResponseString.split(" ", 2)
				requestResponse = HttpRequestResponse.HttpRequestResponse(None, uUrl.getHost(), port, uUrl.getProtocol(), newRequest, newResponse, int(statusCode), uUrl)
				mCallBacks.addToSiteMap(requestResponse)
				#print "Adding: " + urlInSearch
				data = "Adding: " + urlInSearch + "\n"
				self.parent.printLogTab(str(data))
			else:
				data = "\n" + urlInSearch + " was not in scope.\n\n"
				self.parent.printLogTab(str(data))

		data = "End of Google Site Indexing for URL " + str(url) + "\n"
		self.parent.printLogTab(str(data))

def readWebPage( webpageURL ):
    webpageURL = webpageURL.strip()
    log(VERBOSE_, "readWebpage webpageURL=" + webpageURL)
    url = URL(webpageURL)
    conn = url.openConnection()
    conn.setConnectTimeout(30000)
    conn.setReadTimeout(10000)
    conn.connect()
    responseCode = conn.getResponseCode()
    cookie = conn.getHeaderField("Set-Cookie")
    cookiePath = None
    pathDiscr = " Path="
    if cookie and cookie.find(pathDiscr) > 0:
        cookiePath = cookie[cookie.index(pathDiscr) + len(pathDiscr):]
    respLines = []
    if responseCode >= 400:
        log(ERROR_, "HTTP ERROR " + `responseCode` + ": " + `conn.getResponseMessage()`)
    else:
        log(VERBOSE_, "WebPageResponse status=" + `responseCode` + " reason=" + `conn.getResponseMessage()`)
        #log(DEBUG_,"WebPageResponse resp="+`resp` )
        reader = BufferedReader(InputStreamReader(conn.getInputStream()))
        inputLine = reader.readLine()
        while inputLine is not None:
            respLines.append(inputLine)
            inputLine = reader.readLine()

        reader.close()
    return respLines, cookiePath

def DiscoveryMain(Framework):
	OSHVResult = ObjectStateHolderVector()
	
	urlString = Framework.getParameter(PARAM_URL)
	
	reportPoweredOffVms = 0
	reportPoweredOffVmsValue = Framework.getParameter(PARAM_REPORT_POWERED_OFF_VMS)
	if reportPoweredOffVmsValue and reportPoweredOffVmsValue.lower() =='true':
		reportPoweredOffVms = 1

	
	ipAddress = None
	try:
		urlObject = URL(urlString)
		hostname = urlObject.getHost()
		
		if not hostname:
			logger.debug("Hostname is not defined in URL '%s'" % urlString)
			raise MalformedURLException()
		
		ipAddress = vcloud_discover.getIpFromUrlObject(urlObject)
		if not ipAddress or not netutils.isValidIp(ipAddress) or netutils.isLocalIp(ipAddress):
			msg = "Failed to resolve the IP address of server from specified URL"
			errormessages.resolveAndReport(msg, vcloud_discover.VcloudProtocol.DISPLAY, Framework)
			return OSHVResult
		
	except MalformedURLException:
		msg = "Specified URL '%s' is malformed" % urlString
		errormessages.resolveAndReport(msg, vcloud_discover.VcloudProtocol.DISPLAY, Framework)
	except:
		msg = logger.prepareJythonStackTrace("")
		errormessages.resolveAndReport(msg, vcloud_discover.VcloudProtocol.DISPLAY, Framework)
	else:
		
		#configure how connections should be discovered/established
		connectionDiscoverer = vcloud_discover.ConnectionDiscoverer(Framework)
		urlGenerator = vcloud_discover.ConstantUrlGenerator(urlString)
		connectionDiscoverer.setUrlGenerator(urlGenerator)
		connectionDiscoverer.addIp(ipAddress)
		
		#configure how established/failed connection should be used
		connectionHandler = vcloud_discover.BaseDiscoveryConnectionHandler(Framework)
		topologyDiscoverer = vcloud_discover.createVcloudDiscoverer(Framework)
		topologyReporter = vcloud_report.createVcloudReporter(Framework, None, reportPoweredOffVms)
		connectionHandler.setDiscoverer(topologyDiscoverer)
		connectionHandler.setReporter(topologyReporter)
		
		connectionDiscoverer.setConnectionHandler(connectionHandler)
		
		connectionDiscoverer.initConnectionConfigurations()
		
		connectionDiscoverer.discover(firstSuccessful=0)
		
		if not connectionHandler.connected:
			for errorMsg in connectionHandler.connectionErrors:
				Framework.reportError(errorMsg)
			for warningMsg in connectionHandler.connectionWarnings:
				Framework.reportWarning(warningMsg)

	return OSHVResult

def delete(targetURL, params):
    url = URL(targetURL)
    connection = url.openConnection()
    connection.setRequestMethod("DELETE")    
    connection.setRequestProperty("Content-Language", "en-GB")
    connection.setUseCaches(0)
    connection.setDoOutput(2)
    return getResponse(connection);

 def __init__(self, queryString):
     self.args = queryString
     # We would like content_type to be a class property but this is
     # not supported in Python 2.1
     self.content_type = "text/plain"
     reqURL = URL("http://myhost/mywms/")
     self.server = FakeModPythonServerObject("%s:%d" % (reqURL.getHost(), reqURL.getPort()))
     self.unparsed_uri = str(reqURL.getPath())

def createURLOSHV(urlString, framework = None):
    OSHVResult = ObjectStateHolderVector()    
    
    #urlOSH2 = modeling.createOshByCmdbIdString('uri_endpoint', urlId)       
    logger.debug("Starting URL discovery on '%s'" % urlString)
    #urlString = urlString[1:len(urlString)-1]
    if not urlString:
        return OSHVResult
    
    try:
    
        urlString = str(urlString).replace("\\", "//")
        
        urlObject = URL(urlString)
        hostname = urlObject.getHost()
    
        if not hostname:
            raise MalformedURLException("Hostname is not defined in URL '%s'" % urlString)
    
        urlObjectResolver = URLObjectResolver(urlObject)
        protocol = urlObjectResolver.getProtocolFromUrlObject()
        
        if not protocol:
            raise Exception("Failed to resolve the http/https protocol from specified URL")
    
        port = urlObjectResolver.getPortFromUrlObject()
        
        if not port:
            raise Exception("Failed to resolve the port number from specified URL")
    
        # get topology
        # create business element CI and attach the url as configuration document CI to it 
    
        ips = urlObjectResolver.getIpFromUrlObject()
        
        for ipAddress in ips:
            logger.debug('%s: Reporting ip address: %s' % (urlString, ipAddress))
            if not ipAddress or not netutils.isValidIp(ipAddress) or netutils.isLocalIp(ipAddress):
                raise Exception("Failed to resolve the IP address of server from specified URL")
    
            
            hostOSH, ipOSH, OSHVResult2 = createIPEndpointOSHV(framework, ipAddress, port, protocol, hostname)     
            OSHVResult.addAll(OSHVResult2)
            # create UriEndpoint and relations between business element and UriEndpoint
            urlOSH = modeling.createUrlOsh(hostOSH, urlString, None)
            #urlOSH.setCmdbObjectId(urlOSH2.getCmdbObjectId())            
            OSHVResult.add(urlOSH)
            OSHVResult.add(modeling.createLinkOSH('dependency', urlOSH, ipOSH)) 
           
                        
            #create Web Server
    except:
        logger.warnException("Error creating URL OSH for %s" % urlString)


    return OSHVResult

 def _get_map_file(self):
     map_param = self.getParameter('map')
     map_url = URL(self.getDocumentBase(), map_param)
     self.showStatus('Loading ' + map_url.toString())
     map_is = map_url.openStream()
     map_buf = ""
     i = map_is.read()
     while i != -1:
         map_buf = map_buf + chr(i)
         i = map_is.read()
     return StringIO(map_buf)

 def new_urlopen(req):
     if isinstance(req, str):
         full_url = req
     else:
         full_url = req.get_full_url()
     if full_url.startswith('https'):
         u = URL(full_url)
         conn = u.openConnection()
         return ConnReader(conn)
     else:
         org_urlopen(req)

 def addAuthentication(self, httpClient, method):
     BasicAuthStrategy.addAuthentication(self, httpClient, method)
     url = URL(self.__baseUrl)
     proxy = ProxySelector.getDefault().select(url.toURI()).get(0)
     httpClient.getParams().setAuthenticationPreemptive(False);
     if not proxy.type().equals(Proxy.Type.DIRECT):
         address = proxy.address()
         proxyHost = address.getHostName()
         proxyPort = address.getPort()
         httpClient.getHostConfiguration().setProxy(proxyHost, proxyPort)
         print "Using proxy '%s:%s'" % (proxyHost, proxyPort)