Golang URL.Parse方法代码示例

func fiddler2_enable(client *http.Client, proxys string) {
	transport := &http.Transport{}
	http_proxy := url.URL{}
	proxy, _ := http_proxy.Parse(proxys)
	transport.Proxy = http.ProxyURL(proxy)
	client.Transport = transport
}

func (f *Fetcher) newHostFetcher(u *url.URL) (*hostFetcherInfiniteQ, error) {
	if u.Host == "" {
		// The URL must be rooted with a host.
		return nil, ErrEmptyHost
	}
	baseurl, err := u.Parse("/")
	if err != nil {
		return nil, err
	}

	// Create the infinite queue: the in channel to send on, and the out channel
	// to read from in the host's goroutine, and add to the hosts map
	var out chan Command
	in, out := make(chan Command, 1), make(chan Command, 1)
	chand := CmdHandlerFunc(func(cmd Command, res *http.Response, err error) {
		f.Handler.Handle(&Context{Cmd: cmd, Q: f.q}, res, err)
	})
	hf := NewHostFetcher(f.CrawlConfig, baseurl, chand, out)
	// Start the infinite queue goroutine for this host
	go sliceIQ(in, out)
	// Start the working goroutine for this host
	f.q.wg.Add(1)
	go func() {
		hf.Run()
		f.q.wg.Done()
	}()
	return &hostFetcherInfiniteQ{*hf, in}, nil
}

func (o *OpenTsdbOutput) Init(config interface{}) (err error) {
	o.OpenTsdbOutputConfig = config.(*OpenTsdbOutputConfig)
	//todo: check address validity
	// if o.url, err = url.Parse(o.Address); err != nil {
	//     return fmt.Errorf("Can't parse URL '%s': %s", o.Address, err.Error())
	// }
	//
	o.client = &http.Client{
		Transport: &timeoutTransport{Transport: new(http.Transport)},
		Timeout:   time.Minute,
	}

	var u *url.URL
	if u, err = url.Parse(o.Url); err == nil {
	}

	o.logMsgChan = make(chan []byte, o.LogMsgChSize)

	u, err = u.Parse("/api/put")
	if err != nil {
		return err
	}
	if strings.HasPrefix(u.Host, ":") {
		u.Host = "localhost" + u.Host
	}
	o.Url = u.String()

	if err != nil {
		log.Printf("initialize OpenTsdbOutput failed, %s", err.Error())
		return err
	}
	return
}

func dockerRepo(RepoEndpoint string, myProxy string) {
	repo := http.NewServeMux()
	if myProxy != "" {
		url_i := url.URL{}
		url_proxy, _ := url_i.Parse(myProxy)

		tr := &http.Transport{
			DisableCompression: false,
			DisableKeepAlives:  false,
			Proxy:              http.ProxyURL(url_proxy),
			TLSClientConfig:    &tls.Config{InsecureSkipVerify: true},
		}
		clientRepo := &http.Client{Transport: tr}

		repo.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			sendHttps(w, r, clientRepo)
		})
		log.Fatal(http.ListenAndServe(RepoEndpoint, repo))
	} else {
		tr := &http.Transport{
			DisableCompression: false,
			DisableKeepAlives:  false,
			TLSClientConfig:    &tls.Config{InsecureSkipVerify: true},
		}
		clientRepo := &http.Client{Transport: tr}

		repo.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			sendHttps(w, r, clientRepo)
		})
		log.Fatal(http.ListenAndServe(RepoEndpoint, repo))
	}
}

func (this *ReqQiushiModule) Init(global_conf *context.GlobalContext) (err error) {
	this.qiushi_url = global_conf.Qiushi.Location
	this.timeout = global_conf.Qiushi.Timeout
	/*********设置传输层参数****************/
	transport := &http.Transport{}
	transport.Dial = func(netw, addr string) (net.Conn, error) {
		c, err := net.DialTimeout(netw, addr, time.Millisecond*time.Duration(this.timeout))
		if err != nil {
			utils.WarningLog.Write("dail timeout [%s]", err.Error())
			return nil, err
		}
		return c, nil
	}
	transport.MaxIdleConnsPerHost = 10
	transport.ResponseHeaderTimeout = time.Millisecond * time.Duration(this.timeout)
	if global_conf.Proxy.Open {
		url_i := url.URL{}
		url_proxy, _ := url_i.Parse(global_conf.Proxy.Location)
		transport.Proxy = http.ProxyURL(url_proxy)
		utils.DebugLog.Write("open http proxy , proxy location [%s]", global_conf.Proxy.Location)
	}
	/**********************************/
	this.client = &http.Client{}
	this.client.Transport = transport
	utils.DebugLog.Write("req qiushi url [%s]", this.qiushi_url)

	return
}

func batchRequests(requests []*Request, endPoint *url.URL) []*Response {
	responses := make([]*Response, len(requests))
	var wg sync.WaitGroup
	// TODO: change to use go rutine
	for i, request := range requests {
		wg.Add(1)
		go func(i int, request *Request) {
			client := &http.Client{
				Timeout: 10 * time.Second,
			}
			log.Println("Resuest:", request.Method, request.RelativeURL)
			url, err := endPoint.Parse(request.RelativeURL)
			showError(err)
			req, err := http.NewRequest(request.Method, url.String(), strings.NewReader(request.Body))
			showError(err)
			resp, err := client.Do(req)
			showError(err)
			log.Println(resp)
			responses[i] = NewResponse(resp)
			wg.Done()
		}(i, request)
	}
	wg.Wait()
	return responses
}

func (disc *Discoverer) CookArticleURL(baseURL *url.URL, artLink string) (*url.URL, error) {
	// parse, extending to absolute
	u, err := baseURL.Parse(artLink)
	if err != nil {
		return nil, err
	}

	// on a host we accept?
	if !disc.isHostGood(u.Host) {
		return nil, fmt.Errorf("host rejected (%s)", u.Host)
	}

	// matches one of our url forms
	foo := u.RequestURI()
	accept := false
	for _, pat := range disc.ArtPats {
		if pat.MatchString(foo) {
			accept = true
			break
		}
	}
	if !accept {
		return nil, fmt.Errorf("url rejected")
	}

	// apply our sanitising rules for this site
	if disc.StripFragments {
		u.Fragment = ""
	}
	if disc.StripQuery {
		u.RawQuery = ""
	}
	return u, nil
}

func StartServerMultiplesBotsHostPort(uri string, pathl string, host string, port string, newrelic *RelicConfig, bots ...*TgBot) {
	var puri *url.URL
	if uri != "" {
		tmpuri, err := url.Parse(uri)
		if err != nil {
			fmt.Printf("Bad URL %s", uri)
			return
		}
		puri = tmpuri
	}

	botsmap := make(map[string]*TgBot)
	for _, bot := range bots {
		tokendiv := strings.Split(bot.Token, ":")
		if len(tokendiv) != 2 {
			return
		}

		tokenpath := fmt.Sprintf("%s%s", tokendiv[0], tokendiv[1])
		botpathl := path.Join(pathl, tokenpath)

		nuri, _ := puri.Parse(botpathl)
		remoteuri := nuri.String()
		res, error := bot.SetWebhook(remoteuri)

		if error != nil {
			ec := res.ErrorCode
			fmt.Printf("Error setting the webhook: \nError code: %d\nDescription: %s\n", &ec, res.Description)
			continue
		}
		if bot.MainListener == nil {
			bot.StartMainListener()
		}
		botsmap[tokenpath] = bot
	}

	pathtolisten := path.Join(pathl, "(?P<token>[a-zA-Z0-9-_]+)")

	m := martini.Classic()
	m.Post(pathtolisten, binding.Json(MessageWithUpdateID{}), func(params martini.Params, msg MessageWithUpdateID) {
		bot, ok := botsmap[params["token"]]

		if ok && msg.UpdateID > 0 && msg.Msg.ID > 0 {
			bot.MainListener <- msg
		} else {
			fmt.Println("Someone tried with: ", params["token"], msg)
		}
	})

	if newrelic != nil {
		gorelic.InitNewrelicAgent(newrelic.Token, newrelic.Name, false)
		m.Use(gorelic.Handler)
	}

	if host == "" || port == "" {
		m.Run()
	} else {
		m.RunOnAddr(host + ":" + port)
	}
}

func init() {
	h := backend.NewHandler()
	h.CommandName = "get"
	h.CommandPattern = "(get)( )(.*)"
	h.Usage = "get url"
	h.HandlerFunc = func(cmd *backend.Command) (string, bool) {
		//queryHost := cmd.Tokens[1]
		//queryPort := cmd.Tokens[2]
		//queryArgs := cmd.Tokens[3]

		arg := cmd.Tokens[1]
		u := new(url.URL)
		newUrl, err := u.Parse(arg)
		if err != nil {
			return "Unable to parse url: " + err.Error(), false
		}
		fmt.Println("Request: get", newUrl)
		res, err := backend.GetHttpResource(newUrl.String())
		if err != nil {
			return err.Error(), false
		}
		return res, true
	}
	backend.HandlerRegistry[h.CommandName] = h
}

func maybeResolvedLink(root *url.URL, other string) string {
	parsed, err := root.Parse(other)
	if err == nil {
		return parsed.String()
	}

	return other
}

// Init initializes the metadata send queue.
func Init(u *url.URL, debug bool) error {
	mh, err := u.Parse("/api/metadata/put")
	if err != nil {
		return err
	}
	metahost = mh.String()
	metadebug = debug
	go collectMetadata()
	return nil
}

func urlParse(u *url.URL) lua.Function {
	return func(l *lua.State) int {
		newU, err := u.Parse(lua.CheckString(l, 1))
		if err != nil {
			lua.Errorf(l, err.Error())
			panic("unreachable")
		}
		pushURL(l, newU)
		return 1
	}
}

// Add ?format=txt to the given URL
func addTXTFormatString(u *url.URL) string {
	ret, err := u.Parse("")
	if err != nil {
		fmt.Println(err)
		return u.String()
	}
	val := ret.Query()
	val.Add("format", "txt")
	ret.RawQuery = val.Encode()
	return ret.String()
}

func parse(s string, base url.URL) (ln []string, as []string) {
	z := html.NewTokenizer(strings.NewReader(s))
	lnm := make(map[string]struct{})
	asm := make(map[string]struct{})

	// anonymous func used to get attribute values
	attr := func(a string) string {
		var av string
		for {
			if k, v, ha := z.TagAttr(); string(k) == a {
				av = string(v)
				break
			} else if ha == false {
				break
			}
		}
		return av
	}

	// convert map to slice
	slc := func(m map[string]struct{}) []string {
		var v []string

		for k := range m {
			v = append(v, k)
		}
		sort.Strings(v)
		return v
	}

	for {
		tt := z.Next()
		switch tt {
		case html.ErrorToken:
			return slc(lnm), slc(asm)
		case html.StartTagToken, html.EndTagToken:
			if tn, ha := z.TagName(); ha {
				tg := string(tn)
				if av := attr(attrs[tg]); av != "" {
					switch tg {
					case atag:
						if url, err := base.Parse(av); err == nil {
							if url.Host == base.Host && url.RequestURI() != base.RequestURI() {
								lnm[url.RequestURI()] = struct{}{}
							}
						}
					case stag, itag, ltag:
						asm[av] = struct{}{}
					}
				}
			}
		}
	}
}

func sanitizeLink(u *url.URL, v string) string {
	p, err := u.Parse(v)
	if err != nil {
		return ""
	}
	if !acceptableUriSchemes[p.Scheme] {
		return ""
	}

	return p.String()
}