Golang com.HttpGetBytes函数代码示例

func main() {
	log.Println("201402191")
	client := &http.Client{}
	p, err := com.HttpGetBytes(client, "http://godoc.org/-/index", nil)
	if err != nil {
		log.Fatalf("Fail to load page: %v", err)
	}
	content := string(p)
	start := strings.Index(content, "<tbody>") + 7
	end := strings.Index(content, "</tbody>")
	content = content[start:end]

	pkgs := strings.Split(content, "<tr>")[1:]

	skipUntilIndex := 9052
	endWhenIndex := 12000
	for i, name := range pkgs {
		if i < skipUntilIndex {
			continue
		} else if i == endWhenIndex {
			break
		}
		name = strings.TrimSpace(name)[14:]
		end := strings.Index(name, "\">")
		name = name[:end]
		log.Printf("#%d %s", i, name)
		_, err = com.HttpGet(client, "https://gowalker.org/"+name, nil)
		if err != nil {
			log.Fatalf("Fail to load page: %v", err)
		}
		time.Sleep(0 * time.Second)
	}
}

func TestJobsRegionsPaginate(t *testing.T) {
	pg := []struct {
		from, to int
	}{
		{0, 1},
		{0, 2},
		{1, 4},
	}
	for _, page := range pg {
		for _, reg := range regionsSample {
			paginate := fmt.Sprintf("%s/jobs/regions/%s/%d/%d", ts.URL, reg.short, page.from, page.to)
			b, err := com.HttpGetBytes(client, paginate, nil)
			if err != nil {
				t.Errorf("getting regions home page %v", err)
			}
			doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(b)))
			if err != nil {
				t.Errorf("loading document %v", err)
			}
			s := doc.Find(".job-item")
			d := page.to - page.from
			if s.Length() != d {
				t.Errorf("expected %d got %d", d, s.Length())
			}
		}

	}
}

//
//
//		DOCS
//
//
func TestDocsHome(t *testing.T) {
	l := fmt.Sprintf("%s/docs", ts.URL)
	b, err := com.HttpGetBytes(client, l, nil)
	if err != nil {
		t.Errorf("getting docs home %v", err)
	}
	if !bytes.Contains(b, []byte("home.md")) {
		t.Errorf("ecpectd docs home got %s", b)
	}
}

func TestJobsRegionsHome(t *testing.T) {
	home := fmt.Sprintf("%s/jobs/regions", ts.URL)
	b, err := com.HttpGetBytes(client, home, nil)
	if err != nil {
		t.Errorf("getting regions home page %v", err)
	}
	if !bytes.Contains(b, []byte(regionsSample[1].name)) {
		t.Errorf("expected %s to contain %s", b, regionsSample[1].name)
	}
}

func getGitTree(url string) *gitTree {
	p, err := com.HttpGetBytes(&http.Client{}, url, nil)
	if err != nil {
		log.Fatal(err)
	}
	var t gitTree
	if err = json.Unmarshal(p, &t); err != nil {
		log.Fatal(err)
	}
	return &t
}

func TestJobsNewGet(t *testing.T) {
	l := fmt.Sprintf("%s/jobs/new", dashPath)
	b, err := com.HttpGetBytes(client, l, nil)
	if err != nil {
		t.Errorf("getting dashboard home %v", err)
	}
	title := "<title>new job</title>"
	if !bytes.Contains(b, []byte(title)) {
		t.Errorf(" expected new job  page got %s", b)
	}
}

//
//
//		DASHBOARD
//
//
func TestHome(t *testing.T) {
	l := fmt.Sprintf("%s/", dashPath)
	b, err := com.HttpGetBytes(client, l, nil)
	if err != nil {
		t.Errorf("getting dashboard home %v", err)
	}
	title := "<title>dashboard</title>"
	if !bytes.Contains(b, []byte(title)) {
		t.Errorf(" expected login page got %s", b)
	}
}

func TestSetLang(t *testing.T) {
	sw := fmt.Sprintf("%s/language/sw", ts.URL)
	b, err := com.HttpGetBytes(client, sw, nil)
	if err != nil {
		t.Errorf("getting home page %v", err)
	}

	// Yes a home page should contain swahili words
	if !bytes.Contains(b, []byte("nyumbani")) {
		t.Errorf("expected home page to be in swahili got %s", b)
	}
}

func getGoogleVCS(client *http.Client, match map[string]string) error {
	// Scrape the HTML project page to find the VCS.
	p, err := com.HttpGetBytes(client, com.Expand("http://code.google.com/p/{repo}/source/checkout", match), nil)
	if err != nil {
		return errors.New("doc.getGoogleVCS(" + match["importPath"] + ") -> " + err.Error())
	}
	m := googleRepoRe.FindSubmatch(p)
	if m == nil {
		return com.NotFoundError{"Could not VCS on Google Code project page."}
	}
	match["vcs"] = string(m[1])
	return nil
}

func TestGetRegister(t *testing.T) {
	l := fmt.Sprintf("%s%s", ts.URL, registerPath)
	b, err := com.HttpGetBytes(client, l, nil)
	if err != nil {
		t.Errorf("getting login page %v", err)
	}

	// The title of the page should be set to register
	title := "<title>register</title>"
	if !bytes.Contains(b, []byte(title)) {
		t.Errorf(" expected login page got %s", b)
	}
}

func getGolangRevision(client *http.Client, n *Node) error {
	match := map[string]string{}
	{
		m := golangPattern.FindStringSubmatch(n.ImportPath)
		for i, n := range golangPattern.SubexpNames() {
			if n != "" {
				match[n] = m[i]
			}
		}
		setupGoogleMatch(match)
	}
	match["repo"] = "go"

	if len(n.Value) == 0 {
		// Scrape the HTML project page to find the VCS.
		p, err := com.HttpGetBytes(client, com.Expand("http://code.google.com/p/{repo}/source/checkout", match), nil)
		if err != nil {
			return fmt.Errorf("fail to fetch page: %v", err)
		}
		m := googleRepoRe.FindSubmatch(p)
		if m == nil {
			return fmt.Errorf("cannot find VCS on Google Code project page")
		}
		match["vcs"] = string(m[1])
		n.Value = defaultTags[match["vcs"]]
	}
	match["tag"] = n.Value
	data, err := com.HttpGetBytes(client, com.Expand("http://code.google.com/p/{repo}/source/browse/?repo={subrepo}&r={tag}", match), nil)
	if err != nil {
		return fmt.Errorf("fail to get revision(%s): %v", n.ImportPath, err)
	}
	m := googleRevisionPattern.FindSubmatch(data)
	if m == nil {
		return fmt.Errorf("cannot find revision in page: %s", n.ImportPath)
	}
	n.Revision = strings.TrimPrefix(string(m[0]), `_setViewedRevision('`)
	n.ArchivePath = path.Join(setting.ArchivePath, n.ImportPath, n.Revision+".zip")
	return nil
}

func getGoogleVCS(client *http.Client, match map[string]string) error {
	// Scrape the HTML project page to find the VCS.
	p, err := com.HttpGetBytes(client, com.Expand("http://code.google.com/p/{repo}/source/checkout", match), nil)
	if err != nil {
		return fmt.Errorf("fail to fetch page: %v", err)
	}
	m := googleRepoRe.FindSubmatch(p)
	if m == nil {
		return com.NotFoundError{"Could not VCS on Google Code project page."}
	}
	match["vcs"] = string(m[1])
	return nil
}

func TestDocs(t *testing.T) {

	// with .md  extension
	l := fmt.Sprintf("%s/docs/home.md", ts.URL)
	b, err := com.HttpGetBytes(client, l, nil)
	if err != nil {
		t.Errorf("getting docs home %v", err)
	}
	if !bytes.Contains(b, []byte("home.md")) {
		t.Errorf("ecpectd docs home got %s", b)
	}

	// without .md extsnison
	l = fmt.Sprintf("%s/docs/home", ts.URL)
	b, err = com.HttpGetBytes(client, l, nil)
	if err != nil {
		t.Errorf("getting docs home %v", err)
	}
	if !bytes.Contains(b, []byte("home.md")) {
		t.Errorf("ecpectd docs home got %s", b)
	}
}

// getValidTLDs gets and returns list of valid TLDs.
func getValidTLDs() (validTLDs []string) {
	p, err := com.HttpGetBytes(&http.Client{}, "http://data.iana.org/TLD/tlds-alpha-by-domain.txt", nil)
	if err != nil {
		log.Fatal(err)
	}

	for _, line := range strings.Split(string(p), "\n") {
		line = strings.TrimSpace(line)
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		validTLDs = append(validTLDs, "."+strings.ToLower(line))
	}
	return validTLDs
}

func TestJobsHome(t *testing.T) {
	home := fmt.Sprintf("%s/jobs/", ts.URL)
	b, err := com.HttpGetBytes(client, home, nil)
	if err != nil {
		t.Errorf("getting home page %v", err)
	}
	doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(b)))
	if err != nil {
		t.Errorf("loading document %v", err)
	}
	s := doc.Find(".job-item")
	if s.Length() < 8 {
		t.Errorf("expected 8 jobs got %d", s.Length())
	}
}