Python urlparse.urlparse方法代码示例

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def urlParser(target):
    log = logging.getLogger('urlparser')

    ssl = False
    o = urlparse(target)
    if o[0] not in ['http', 'https', '']:
        log.error('scheme %s not supported' % o[0])
        return
    if o[0] == 'https':
        ssl = True
    if len(o[2]) > 0:
        path = o[2]
    else:
        path = '/'
    tmp = o[1].split(':')
    if len(tmp) > 1:
        port = tmp[1]
    else:
        port = None
    hostname = tmp[0]
    query = o[4]
    return (hostname, port, path, query, ssl) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def loginredirect():
    """Initialize OAuth login."""
    app.session_interface.abandon_session(app, session)

    redirecturl, request_token = handshaker.initiate()
    session['request_token_key'], session['request_token_secret'] = \
        request_token.key, request_token.secret
    session['return_to_url'] = url_for('main')

    returnto = request.args.get('returnto')
    if returnto:
        ref_url = urlparse(request.url_root)
        test_url = urlparse(urljoin(request.host_url, returnto))
        if (
            test_url.scheme == ref_url.scheme and
            test_url.netloc == ref_url.netloc and
            test_url.path.startswith(ref_url.path)
        ):
            session['return_to_url'] = returnto

    return redirect(redirecturl) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def test_connection(name, url, timeout=10):
    """Simple connection test"""
    urlinfo = urlparse(url)
    start = time.time()
    try:
        ip = socket.gethostbyname(urlinfo.netloc)
    except Exception as e:
        print('Error resolving DNS for {}: {}, {}'.format(name, url, e))
        return
    dns_elapsed = time.time() - start
    start = time.time()
    try:
        _ = urlopen(url, timeout=timeout)
    except Exception as e:
        print("Error open {}: {}, {}, DNS finished in {} sec.".format(name, url, e, dns_elapsed))
        return
    load_elapsed = time.time() - start
    print("Timing for {}: {}, DNS: {:.4f} sec, LOAD: {:.4f} sec.".format(name, url, dns_elapsed, load_elapsed)) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def reverseip(url):
    """return domains from given the same server"""

    # get only domain name
    url = urlparse(url).netloc if urlparse(url).netloc != '' else urlparse(url).path.split("/")[0]

    source = "http://domains.yougetsignal.com/domains.php"
    useragent = useragents.get()
    contenttype = "application/x-www-form-urlencoded; charset=UTF-8"

    # POST method
    opener = urllib2.build_opener(
        urllib2.HTTPHandler(), urllib2.HTTPSHandler())
    data = urllib.urlencode([('remoteAddress', url), ('key', '')])

    request = urllib2.Request(source, data)
    request.add_header("Content-type", contenttype)
    request.add_header("User-Agent", useragent)

    try:
        result = urllib2.urlopen(request).read()

    except urllib2.HTTPError, e:
        print >> sys.stderr, "[{}] HTTP error".format(e.code) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def __sqli(url):
    """check SQL injection vulnerability"""

    std.stdout("scanning {}".format(url), end="")

    domain = url.split("?")[0]  # domain with path without queries
    queries = urlparse(url).query.split("&")
    # no queries in url
    if not any(queries):
        print "" # move cursor to new line
        return False, None

    payloads = ("'", "')", "';", '"', '")', '";', '`', '`)', '`;', '\\', "%27", "%%2727", "%25%27", "%60", "%5C")
    for payload in payloads:
        website = domain + "?" + ("&".join([param + payload for param in queries]))
        source = web.gethtml(website)
        if source:
            vulnerable, db = sqlerrors.check(source)
            if vulnerable and db != None:
                std.showsign(" vulnerable")
                return True, db

    print ""  # move cursor to new line
    return False, None 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def check_project(project_name):
    if 'token' in if_config_vars and len(if_config_vars['token']) != 0:
        logger.debug(project_name)
        try:
            # check for existing project
            check_url = urlparse.urljoin(if_config_vars['if_url'], '/api/v1/getprojectstatus')
            output_check_project = subprocess.check_output('curl "' + check_url + '?userName=' + if_config_vars['user_name'] + '&token=' + if_config_vars['token'] + '&projectList=%5B%7B%22projectName%22%3A%22' + project_name + '%22%2C%22customerName%22%3A%22' + if_config_vars['user_name'] + '%22%2C%22projectType%22%3A%22CUSTOM%22%7D%5D&tzOffset=-14400000"', shell=True)
            # create project if no existing project
            if project_name not in output_check_project:
                logger.debug('creating project')
                create_url = urlparse.urljoin(if_config_vars['if_url'], '/api/v1/add-custom-project')
                output_create_project = subprocess.check_output('no_proxy= curl -d "userName=' + if_config_vars['user_name'] + '&token=' + if_config_vars['token'] + '&projectName=' + project_name + '&instanceType=PrivateCloud&projectCloudType=PrivateCloud&dataType=' + get_data_type_from_project_type() + '&samplingInterval=' + str(if_config_vars['sampling_interval'] / 60) +  '&samplingIntervalInSeconds=' + str(if_config_vars['sampling_interval']) + '&zone=&email=&access-key=&secrete-key=&insightAgentType=' + get_insight_agent_type_from_project_type() + '" -H "Content-Type: application/x-www-form-urlencoded" -X POST ' + create_url + '?tzOffset=-18000000', shell=True)
            # set project name to proposed name
            if_config_vars['project_name'] = project_name
            # try to add new project to system
            if 'system_name' in if_config_vars and len(if_config_vars['system_name']) != 0:
                system_url = urlparse.urljoin(if_config_vars['if_url'], '/api/v1/projects/update')
                output_update_project = subprocess.check_output('no_proxy= curl -d "userName=' + if_config_vars['user_name'] + '&token=' + if_config_vars['token'] + '&operation=updateprojsettings&projectName=' + project_name + '&systemName=' + if_config_vars['system_name'] + '" -H "Content-Type: application/x-www-form-urlencoded" -X POST ' + system_url + '?tzOffset=-18000000', shell=True)
        except subprocess.CalledProcessError as e:
            logger.error('Unable to create project for ' + project_name + '. Data will be sent to ' + if_config_vars['project_name']) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def send_data_to_if(chunk_metric_data):
    send_data_time = time.time()

    # prepare data for metric streaming agent
    data_to_post = initialize_api_post_data()
    if 'DEPLOYMENT' in if_config_vars['project_type'] or 'INCIDENT' in if_config_vars['project_type']:
        for chunk in chunk_metric_data:
            chunk['data'] = json.dumps(chunk['data'])
    data_to_post[get_data_field_from_project_type()] = json.dumps(chunk_metric_data)

    logger.debug('First:\n' + str(chunk_metric_data[0]))
    logger.debug('Last:\n' + str(chunk_metric_data[-1]))
    logger.debug('Total Data (bytes): ' + str(get_json_size_bytes(data_to_post)))
    logger.debug('Total Lines: ' + str(track['line_count']))

    # do not send if only testing
    if cli_config_vars['testing']:
        return

    # send the data
    post_url = urlparse.urljoin(if_config_vars['if_url'], get_api_from_project_type())
    send_request(post_url, 'POST', 'Could not send request to IF',
                 str(get_json_size_bytes(data_to_post)) + ' bytes of data are reported.',
                 data=data_to_post, proxies=if_config_vars['if_proxies'])
    logger.debug('--- Send data time: %s seconds ---' % round(time.time() - send_data_time, 2)) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def extract_domains(self, resp):
        link_regx = re.compile('<cite.*?>(.*?)<\/cite>')
        try:
            links_list = link_regx.findall(resp)
            for link in links_list:
                link = re.sub('<span.*>', '', link)
                if not link.startswith('http'):
                    link = "http://" + link
                subdomain = urlparse.urlparse(link).netloc
                if subdomain and subdomain not in self.subdomains and subdomain != self.domain:
                    if self.verbose:
                        self.print_("%s%s: %s%s" % (R, self.engine_name, W, subdomain))
                    self.subdomains.append(subdomain.strip())
        except Exception:
            pass
        return links_list 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def _setParams(self):
        parsedURL = urlparse.urlparse(self.url)
        # setting the path
        if self.useAbsolutePath == True:
            self._path = self.url
        else:
            self._path = parsedURL.path
            self.qs = parsedURL.query

        if self._path == '':
            self._path = '/'

        # fix the body if it is in dict format
        if isinstance(self.body,dict):
            self.body = urllib.urlencode(self.body)

        # set other necessary parameters
        self.targetName = parsedURL.netloc
        self.targetPort = parsedURL.port
        self.targetProtocol = (parsedURL.scheme).lower()
        if self.targetProtocol == 'https':
            self.isSSL = True
            if self.targetPort == None: self.targetPort = 443
        elif self.targetPort == None:
            self.targetPort = 80 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def _setParams(self):
        parsedURL = urlparse.urlparse(self.url)
        # setting the path
        if self.useAbsolutePath == True:
            self._path = self.url
        else:
            self._path = parsedURL.path
            self.qs = parsedURL.query

        if self._path == '':
            self._path = '/'

        # fix the body if it is in dict format
        if isinstance(self.body,dict):
            self.body = urllib.urlencode(self.body)

        # set other necessary parameters
        self.targetName = parsedURL.hostname
        self.targetPort = parsedURL.port
        self.targetProtocol = (parsedURL.scheme).lower()
        if self.targetProtocol == 'https':
            self.isSSL = True
            if self.targetPort == None: self.targetPort = 443
        elif self.targetPort == None:
            self.targetPort = 80 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def checkIP(self, directive, directiveValues, findings):
        csp = self.csp
        for value in directiveValues:
            url = '//' + Util.getSchemeFreeUrl(value)
            host = urlparse(url).netloc
            ip = None
            validip = True
            
            try:
                ip = ipaddress.ip_address(u''+host)
            except ValueError:
                validip = False
            if validip:
                ipString = str(ip) + ''
                
                if '127.0.0.1' in ipString:
                    findings.append(Finding(csp.headerkey,FindingType.IP_SOURCE, directive.value + ' directive allows localhost as source. Please make sure to remove this in production environments.',FindingSeverity.INFO, directive, value))
                else:
                    findings.append(Finding(csp.headerkey,FindingType.IP_SOURCE, directive.value + ' directive has an IP-Address as source: ' + ipString + ' (will be ignored by browsers!). ', FindingSeverity.INFO, directive, value)) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def handle_config(root):
    for child in root.children:
        instance_name = None

        if child.key == 'Instance':
            instance_name = child.values[0]
            url = None
            for ch2 in child.children:
                if ch2.key == 'URL':
                    url = ch2.values[0]
            if not url:
                collectd.warning('No URL found in dump1090 Instance ' + instance_name)
            else:
                collectd.register_read(callback=handle_read,
                                       data=(instance_name, urlparse.urlparse(url).hostname, url),
                                       name='dump1090.' + instance_name)
                collectd.register_read(callback=handle_read_1min,
                                       data=(instance_name, urlparse.urlparse(url).hostname, url),
                                       name='dump1090.' + instance_name + '.1min',
                                       interval=60)

        else:
            collectd.warning('Ignored config entry: ' + child.key) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def convertIssue(attributes):
    convertedAttributes = {
        "number": attributes["number"],
        "url": "/repos" + urlparse.urlparse(attributes["html_url"]).path,
        "user": {"login": attributes["user"], "url": "/users/" + attributes["user"]},
    }
    if "labels" in attributes:  # pragma no branch
        convertedAttributes["labels"] = [{"name": label} for label in attributes["labels"]]
    if "title" in attributes:  # pragma no branch
        convertedAttributes["title"] = attributes["title"]
    if "created_at" in attributes:  # pragma no branch
        convertedAttributes["created_at"] = attributes["created_at"]
    if "comments" in attributes:  # pragma no branch
        convertedAttributes["comments"] = attributes["comments"]
    if "body" in attributes:  # pragma no branch
        convertedAttributes["body"] = attributes["body"]
    if "updated_at" in attributes:  # pragma no branch
        convertedAttributes["updated_at"] = attributes["updated_at"]
    if "state" in attributes:  # pragma no branch
        convertedAttributes["state"] = attributes["state"]
    return convertedAttributes 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def union(p, q):
    for url in p:
        parsed = urlparse(str(url))
        if parsed.netloc and parsed.netloc != 'www.webhostingtalk.com':
            url = 'http://%s/' % parsed.netloc
        if parsed.netloc and url not in q:
            print url
            if parsed.netloc != 'www.webhostingtalk.com':
                # Insert into Site
                try:
                    Website.objects.create(
                        url=url,
                        name=parsed.netloc,
                        last_crawled_at=datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
                    )
                except IntegrityError:
                    println('%s - already existed in Site' % url)
            else:
                # We want to deep crawl webhosting talk
                q.append(url) 

# 需要导入模块: import urlparse [as 别名]
# 或者: from urlparse import urlparse [as 别名]
def getPage( self, event ):
      """Parse URL, add addressing scheme and retrieve file"""

      # parse the URL      
      myURL = event.widget.get()
      components = urlparse.urlparse( myURL )
      self.contents.text_state = NORMAL

      # if addressing scheme not specified, use http
      if components[ 0 ] == "":
         myURL = "http://" + myURL

      # connect and retrieve the file
      try:
         tempFile = urllib.urlopen( myURL )
         self.contents.settext( tempFile.read() ) # show results 
         tempFile.close()
      except IOError:
         self.contents.settext( "Error finding file" )

      self.contents.text_state = DISABLED