Golang http.Get函数代码示例

func main() {
	page_count := 1
	download_count := 0

	for {
		offset := page_count * 50
		param := map[string]string{
			"AppId":        appid,
			"Version":      "2.2",
			"Market":       "ja-JP",
			"Sources":      "Image",
			"Image.Count":  strconv.Itoa(50),
			"Image.Offset": strconv.Itoa(offset),
			"Adult":        "off",
			"Query":        "おっぱい",
		}

		var sr *json_root
		res, err := http.Get(get_request_uri(param))
		if err != nil {
			break
		}
		reader := bufio.NewReader(res.Body)
		line, err := reader.ReadBytes('\n')
		if err == nil {
			break
		}
		json.Unmarshal(line, &sr)

		for i := 0; i < len(sr.SearchResponse.Image.Results); i++ {
			result := sr.SearchResponse.Image.Results[i]
			if regexp.MustCompile(".jpg$").FindString(result.MediaUrl) == "" {
				continue
			}
			download_count++

			filename := md5hex(encode_utf8(result.MediaUrl)) + ".jpg"
			filepath := dir + filename

			if _, err := os.Stat(filepath); err == nil {
				continue
			}
			fmt.Printf("%d : Download... %s\n", download_count, result.MediaUrl)

			res, err := http.Get(result.MediaUrl)
			if err != nil {
				runtime.Goexit()
			}
			data, err := ioutil.ReadAll(res.Body)
			if err != nil {
				runtime.Goexit()
			}
			if regexp.MustCompile("^image").FindString(http.DetectContentType(data)) != "" {
				ioutil.WriteFile(filepath, data, 0666)
			}
		}
		page_count++
	}
}

func main() {
	// This works
	r, _, err := http.Get("https://www.google.com")
	if err != nil {
		log.Exit(err)
	}
	log.Println(r)

	// This doesn't
	r, _, err = http.Get("https://streaming.campfirenow.com")
	if err != nil {
		log.Exit(err)
	}
	log.Println(r)
}

func testQuestions() {
	r, _, err := http.Get("http://api.stackoverflow.com/1.0/questions?key=change_me&answers=true")
	if err != nil {
		fmt.Println(err.String())
		return
	}

	d, err := gzip.NewReader(r.Body)
	if err != nil {
		fmt.Println(err.String())
		return
	}

	var qr gostack.QuestionsResult
	b, _ := ioutil.ReadAll(d)

	err = json.Unmarshal(b, &qr)
	if err != nil {
		fmt.Println(err.String())
		return
	}

	fmt.Println(qr)

}

func (req *Googl) get(url string, params ...map[string]string) (string, os.Error) {
	if !strings.Contains(url, "://goo.gl/") {
		return "", os.EINVAL
	}

	req_url := GOOGL_V1 + "?shortUrl=" + url
	if 0 < len(params) {
		for i := 0; i < len(params); i++ {
			req_url += "&" + toQuery(params[i])
		}
	}

	if "" != req.Key {
		req_url += "&key=" + req.Key
	}

	res, _, err := http.Get(req_url)
	defer res.Body.Close()
	if err != nil {
		return "", os.Error(err)
	}

	body, _ := ioutil.ReadAll(res.Body)

	return string(body), nil
}

func (s *TestHTTPServer) Start() {
	if s.started {
		return
	}
	s.started = true

	s.request = make(chan *http.Request, 64)
	s.response = make(chan *testResponse, 64)
	s.pending = make(chan bool, 64)

	url, _ := url.Parse(s.URL)
	go http.ListenAndServe(url.Host, s)

	s.PrepareResponse(202, nil, "Nothing.")
	fmt.Fprintf(os.Stderr, "\nWaiting for the fake server to be up...")
	for {
		resp, err := http.Get(s.URL)
		if err == nil && resp.StatusCode == 202 {
			break
		}

		time.Sleep(1e8)
	}

	fmt.Fprintf(os.Stderr, "Done\n")
	s.WaitRequest()
}

func DiscoverXml(id string) (*string, os.Error) {
	resp, _, err := http.Get(id)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	parser := xml.NewParser(resp.Body)
	inURI := false
	for {
		t, err := parser.Token()
		if err != nil {
			return nil, err
		}
		switch tt := t.(type) {
		case xml.StartElement:
			if tt.Name.Local == "URI" {
				inURI = true
			}
		case xml.CharData:
			if inURI {
				s := string([]byte(tt))
				return &s, nil
			}
		}
	}
	return nil, &DiscoveryError{str: "URI not found"}
}

func (c *Client) StatusesRetweets(id uint64, count int) (t []Tweet) {
	var params string
	var tweets []Tweet

	if id == 0 {
		return nil
	}

	if count != 0 {
		params = addParam(params, "count", fmt.Sprintf("%d", count))
	}

	url := c.makeAuthURL(statusesRetweets+fmt.Sprintf("/%d", id), params)
	res, _, err := http.Get(url)
	if err != nil {
		return nil
	}

	if res.Status != "200 OK" {
		return nil
	}

	reader := bufio.NewReader(res.Body)
	line, _ := reader.ReadString(0)

	print(line + "\n")
	json.Unmarshal(line, &tweets)

	return tweets
}

func (sns *SNS) query(topic *Topic, message *Message, params map[string]string, resp interface{}) os.Error {
	params["Timestamp"] = time.UTC().Format(time.RFC3339)
	url_, err := url.Parse(sns.Region.SNSEndpoint)
	if err != nil {
		return err
	}

	sign(sns.Auth, "GET", "/", params, url_.Host)
	url_.RawQuery = multimap(params).Encode()
	r, err := http.Get(url_.String())
	if err != nil {
		return err
	}
	defer r.Body.Close()

	//dump, _ := http.DumpResponse(r, true)
	//println("DUMP:\n", string(dump))
	//return nil

	if r.StatusCode != 200 {
		return buildError(r)
	}
	err = xml.Unmarshal(r.Body, resp)
	return err
}

// scrape scrapes the given URL and saves it to disk
func scrape(url string, done chan bool) {
	// Notify main when we're done right after we return
	defer func() { done <- true }() // Anonymous functions ftw!

	fmt.Printf("Scraping %s...\n", url)
	defer fmt.Printf("Finished scraping %s\n", url)

	// Don't make the user type "http://" for every freaking URL!
	if !strings.Contains(url, "://") {
		url = "http://" + url
	}

	// Download website contents
	req, err := http.Get(url)
	if somethingBroke(err) {
		return
	}

	// Save contents to variable
	contents, err := ioutil.ReadAll(req.Body)
	defer req.Body.Close()
	if somethingBroke(err) {
		return
	}

	// Write contents to disk. TODO: Store URL, text data in a DB
	url = strings.Replace(url, "/", "___", -1)
	filename := fmt.Sprintf("%v-%v", url, time.Seconds())
	err = ioutil.WriteFile(SCRAPES_DIR+filename, contents, 0644)
	if somethingBroke(err) {
		return
	}

	return
}

func fetchUrlToFile(url, filename string, expectedSize int64) bool {
	fi, statErr := os.Stat(filename)
	if statErr == nil &&
		(expectedSize == -1 && fi.Size > 0 ||
			expectedSize == fi.Size) {
		// TODO: re-fetch mode?
		return true
	}

	netop := NewNetworkOperation()
	defer netop.Done()

	res, _, err := http.Get(url)
	if err != nil {
		addError(fmt.Sprintf("Error fetching %s: %v", url, err))
		return false
	}
	defer res.Body.Close()

	fileBytes, err := ioutil.ReadAll(res.Body)
	if err != nil {
		addError(fmt.Sprintf("Error reading XML from %s: %v", url, err))
		return false
	}

	err = ioutil.WriteFile(filename, fileBytes, 0600)
	if err != nil {
		addError(fmt.Sprintf("Error writing file %s: %v", filename, err))
		return false
	}
	return true
}

func randomQuote() string {
	// Simulates slow connection; shows off `defer`, goroutine asynchronicity
	defer func() { time.Sleep(2e9) }()

	// Scrape site
	url := "http://subfusion.net/cgi-bin/quote.pl?quote=cookie&number=1"
	req, err := http.Get(url)
	checkError(err)

	// Create regex
	re, err := regexp.Compile("<body><br><br><b><hr><br>(.*)<br><br><hr><br>")
	checkError(err)

	// Read HTML
	html, err := ioutil.ReadAll(req.Body)
	req.Body.Close()
	checkError(err)

	// Parse out quote
	quote := re.FindString(string(html))
	quote = strings.Replace(quote, "<body><br><br><b><hr><br>", "", -1)
	quote = strings.Replace(quote, "<br><br><hr><br>", "", -1)

	return quote
}

func tf(c *http.Conn, b string, e string, n int, y int, s string) int {

	var qs string
	var ntf int = 0

	if len(s) < 1 {
		return ntf
	}

	if s[0] == '#' && len(s) > 1 {
		qs = s
	} else {
		qs = "from:" + s
	}
	r, _, err := http.Get(fmt.Sprintf(queryURI,
		qformat, http.URLEscape(qs), b, e, n))
	if err == nil {
		if r.StatusCode == http.StatusOK {
			ntf = readjson(c, r.Body, b, e, y)
		} else {
			fmt.Printf("Twitter is unable to search for %s (%s)\n", s, r.Status)
		}
		r.Body.Close()
	} else {
		fmt.Printf("%v\n", err)
	}
	return ntf
}

func Crawl(url string) (result string, finalUrl string, err os.Error) {
	resp, finalUrl, err := http.Get(url)
	if err == nil {
		fmt.Println("Status: " + resp.Status)
		fmt.Printf("Close: %b\n", resp.Close)
		body := resp.Body
		defer body.Close()

		buf := make([]byte, BufSize)
		var n int
		for {
			n, err = body.Read(buf)
			if n != 0 {
				result += string(buf[0:n])
			}
			if err != nil {
				break
			}
		}
	}

	if err == os.EOF {
		err = nil
	}
	return result, finalUrl, err
}

func parseChat(msg string, m *ircbot.Message) (reply string) {
	if config.Ignores[m.GetSender()] {
		return ""
	}

	if msg == "!help acm-bot" {
		return help(nil)
	}

	if matches := urlRegex.FindAllStringSubmatch(msg, -1); matches != nil {
		for _, m := range matches {

			response, finalURL, err := http.Get(m[0])

			if err != nil {
				errors.Printf("%s - Fetch failed: %s\n", m[0], err.String())
			} else if finalURL != m[0] || config.TitleWhitelist[m[1]] {
				if t := getTitle(response.Body); t != "" {
					info.Println("Fetched: " + m[0])
					reply += fmt.Sprintf("Title:%s\n", t)
				}
			}
		}
	}

	recordSighting(m)

	return
}

func testBadges() {
	r, _, err := http.Get("http://api.stackoverflow.com/1.0/badges?key=change_me")
	if err != nil {
		fmt.Println(err.String())
		return
	}

	d, err := gzip.NewReader(r.Body)
	if err != nil {
		fmt.Println(err.String())
		return
	}

	var br gostack.BadgesResult
	b, _ := ioutil.ReadAll(d)

	err = json.Unmarshal(b, &br)
	if err != nil {
		fmt.Println(err.String())
		return
	}

	fmt.Println(br)

}