本文整理汇总了Golang中github.com/PuerkitoBio/goquery.NewDocumentFromResponse函数的典型用法代码示例。如果您正苦于以下问题:Golang NewDocumentFromResponse函数的具体用法?Golang NewDocumentFromResponse怎么用?Golang NewDocumentFromResponse使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了NewDocumentFromResponse函数的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Golang代码示例。
示例1: TestSubmittingAComment
func TestSubmittingAComment(t *testing.T) {
// Given a server
_, server, _, err := LoginUser(t)
if err != nil {
t.Fatal(err)
}
// When an authenticated client requests /item?id=1
id := 1
res, err := http.Get(fmt.Sprintf(server.URL+"/item?id=%d", id))
if err != nil {
t.Fatal(err)
}
defer res.Body.Close()
// It should respond with status 200
if want, got := 200, res.StatusCode; want != got {
t.Fatalf("status : want '%v' got '%v' ", want, got)
}
doc, err := goquery.NewDocumentFromResponse(res)
csrf, ok := doc.Find("input[name='comment_csrf']").First().Attr("value")
if !ok {
t.Fatalf("csrf value not found for comment form")
}
initialCommentNumber := doc.Find(".comment").Length()
formValues := url.Values{
"comment_content": {"this is a new comment"},
"comment_csrf": {csrf},
"comment_submit": {"submit"},
"comment_parent_id": {"0"},
"comment_goto": {fmt.Sprintf("/item?id=%d", id)},
"comment_thread_id": {fmt.Sprintf("%d", id)},
}
// when an authenicated client submits a valid comment
res, err = http.Post(server.URL+gonews.Route{}.Reply(), "application/x-www-form-urlencoded", strings.NewReader(formValues.Encode()))
if err != nil {
t.Fatal(err)
}
defer res.Body.Close()
// It should respond with status 200
if want, got := 200, res.StatusCode; want != got {
t.Fatalf("Status : want '%v' got '%v' ", want, got)
}
// The number of comments on the story page should have increased by one
doc, err = goquery.NewDocumentFromResponse(res)
if err != nil {
t.Fatal(err)
}
if want, got := initialCommentNumber+1, doc.Find(".comment").Length(); want != got {
t.Fatalf(".comment length : want '%v' got '%v' ", want, got)
}
}
示例2: main
func main() {
login()
for _, v := range cookies {
fmt.Println(v)
}
var bash_str string = "#!/bin/bash \n"
baseMap := map[string]string{}
doc, _ := goquery.NewDocumentFromResponse(getResultHtml(spider_base_url))
doc.Find(".lesson-info-h2").Each(func(i int, contentSelection *goquery.Selection) {
selection := contentSelection.Find("a")
base_href, _ := selection.Attr("href")
dir_name := selection.Text()
bash_str += "mkdir \"" + dir_name + "\"\n"
baseMap[dir_name] = base_href
fmt.Println(dir_name, "-->", base_href)
})
downloadList := []DownloadBean{}
for k, v := range baseMap {
doc, _ := goquery.NewDocumentFromResponse(getResultHtml(v))
doc.Find(".lessonvideo-list dd h2").Each(func(i int, contentSelection *goquery.Selection) {
selection := contentSelection.Find("a")
href, _ := selection.Attr("href")
filename := selection.Text()
fmt.Println(k, "-->", filename, "-->", href)
bean := DownloadBean{dirname: k, href: href, filename: filename}
downloadList = append(downloadList, bean)
})
}
for _, v := range downloadList {
doc, _ := goquery.NewDocumentFromResponse(getResultHtml(v.href))
doc.Find("source").Each(func(i int, contentSelection *goquery.Selection) {
download_url, _ := contentSelection.Attr("src")
one_file := "wget " + download_url + " -O \"./" + v.dirname + "/" + v.filename + ".mp4\"\n"
bash_str += one_file
fmt.Println(one_file)
})
}
err := ioutil.WriteFile("./download.sh", []byte(bash_str), 0x777)
if err != nil {
panic(err.Error())
}
fmt.Println("写入脚本完成")
}
示例3: GetResources
// GetResources retrieves resource info
func GetResources(resp *http.Response) Resources {
doc, _ := goquery.NewDocumentFromResponse(resp)
resWrapperSelector := doc.Find("div#resWrap")
lumber := resWrapperSelector.Find("td#l4")
clay := resWrapperSelector.Find("td#l3")
iron := resWrapperSelector.Find("td#l2")
crop := resWrapperSelector.Find("td#l1")
lumberProd, lumberStored, lumberCapa := parseResource(lumber)
clayProd, clayStored, _ := parseResource(clay)
ironProd, ironStored, _ := parseResource(iron)
cropProd, cropStored, cropCapa := parseResource(crop)
return Resources{
Granary{
Capacity: cropCapa,
Crop: cropStored,
},
Warehouse{
Capacity: lumberCapa,
Lumber: lumberStored,
Clay: clayStored,
Iron: ironStored,
},
Production{
lumberProd,
clayProd,
ironProd,
cropProd,
},
}
}
示例4: Parser
//解析函数定义,声明必须相同,下同
func Parser(httpRes *http.Response) ([]string, []basic.Item) {
//两个需要返回的列表
linklist := make([]string, 0) //下一步需要请求的链接
itemlist := make([]basic.Item, 0) //保存的数据类型为 map[string]interface{}
//自定义部分
//抓取所有链接
doc, _ := goquery.NewDocumentFromResponse(httpRes)
doc.Find("a").Each(func(i int, s *goquery.Selection) {
link, exits := s.Attr("href")
if exits {
link = basic.CheckLink(link)
if link != "" {
linklist = append(linklist, link)
}
}
})
//保存每个页面的标题
title := strings.TrimSpace(doc.Find("head title").Text())
if title != "" {
item := make(map[string]interface{})
item["标题"] = title
itemlist = append(itemlist, item)
}
return linklist, itemlist
}
示例5: DoRequest
func (api *ExternalAPIEngine) DoRequest(
arg APIArg, req *http.Request, restype int) (
ar *ExternalAPIRes, hr *ExternalHTMLRes, tr *ExternalTextRes, err error) {
var resp *http.Response
var jw *jsonw.Wrapper
resp, jw, err = doRequestShared(api, arg, req, (restype == XAPIResJSON))
if err != nil {
return
}
switch restype {
case XAPIResJSON:
ar = &ExternalAPIRes{resp.StatusCode, jw}
case XAPIResHTML:
var goq *goquery.Document
goq, err = goquery.NewDocumentFromResponse(resp)
if err == nil {
hr = &ExternalHTMLRes{resp.StatusCode, goq}
}
case XAPIResText:
var buf bytes.Buffer
_, err := buf.ReadFrom(resp.Body)
defer resp.Body.Close()
if err == nil {
tr = &ExternalTextRes{resp.StatusCode, string(buf.Bytes())}
}
default:
err = fmt.Errorf("unknown restype to DoRequest")
}
return
}
示例6: harvest_board_indices
func harvest_board_indices(board_url string, board_name string) []BoardIndexPage {
var ret []BoardIndexPage
doc, err := goquery.NewDocumentFromResponse(ptt_get(board_url))
if err != nil {
log.Fatal(err)
}
re := regexp.MustCompile("/bbs/" + board_name + "/index([0-9]+)\\.html")
doc.Find("a[href^='/bbs/" + board_name + "/index']").Each(func(_ int, s *goquery.Selection) {
href, exists := s.Attr("href")
if !exists {
return
}
matched := re.FindStringSubmatch(href)
if len(matched) == 0 {
return
}
pn, err := strconv.Atoi(matched[1])
if err != nil {
log.Fatal(err)
}
ret = append(ret, BoardIndexPage{pn, href})
})
if ret[0].page_number > ret[1].page_number {
ret[0], ret[1] = ret[1], ret[0]
}
for i := ret[0].page_number + 1; i < ret[1].page_number; i++ {
ret = append(ret, BoardIndexPage{i, "/bbs/" + board_name + "/index" + strconv.Itoa(i) + ".html"})
}
return ret
}
示例7: main
func main() {
flag.Parse()
if flag.NArg() != 1 {
fmt.Fprintln(os.Stderr, "yomikata [word]")
os.Exit(1)
}
word := flag.Arg(0)
resp, err := http.Get("http://yomikata.org/word/" + word)
if err != nil {
fmt.Fprintln(os.Stderr, os.Args[0]+":", err)
os.Exit(1)
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
fmt.Fprintln(os.Stderr, os.Args[0]+":", err)
os.Exit(1)
}
if doc.Find("#word").Text() == "" {
fmt.Fprintln(os.Stderr, "わかりません")
os.Exit(1)
}
if *exact {
fmt.Println(doc.Find(".spAns .psAns").First().Text())
} else {
doc.Find(".spAns").Each(func(_ int, s *goquery.Selection) {
fmt.Printf("%s (%s)\n", s.Find(".psAns").Text(), s.Find(".psPt").Text())
})
}
}
示例8: GetStatistics
// GetStatistics retrieves statistics
func GetStatistics(resp *http.Response) Statistics {
doc, _ := goquery.NewDocumentFromResponse(resp)
s := doc.Find("tr.hl").First()
positionStr := strings.TrimRight(s.Find(".ra").Text(), ".")
position, _ := strconv.Atoi(positionStr)
username := strings.Trim(s.Find(".pla").Find("a").Text(), " ")
alliance := s.Find("al").Text()
popStr := s.Find(".pop").Text()
pop, _ := strconv.Atoi(popStr)
villageCountStr := s.Find(".vil").Text()
villageCount, _ := strconv.Atoi(villageCountStr)
stats := Statistics{
position,
username,
alliance,
pop,
villageCount,
}
return stats
}
示例9: FetchCwzb
// FetchRzrqSumData 抓取数据
func FetchCwzb(code, ctype string) ([]*Cwzb, error) {
ctypeStr := "K"
if ctype == "sz" {
ctypeStr = "J"
}
formt := "http://quotes.cnfol.com/new/f10/cwzb/%s%s.html"
resp, err := wget.Get(fmt.Sprintf(formt, code, ctypeStr))
if err != nil {
return nil, err
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, err
}
doc.Find("table tr").Each(func(i int, s *goquery.Selection) {
td := s.Find("td")
if td.Length() < 5 {
return
}
td.Each(func(i int, s *goquery.Selection) {
if i == 0 {
return
}
fmt.Println(s.Text())
})
})
return nil, nil
}
示例10: mustParseResponse
func mustParseResponse(resp *http.Response) *goquery.Document {
gq, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
log.Fatal(err)
}
return gq
}
示例11: Search
func (a PlayStore) Search(sr SearchRequest) ([]App, error) {
playurl, post := a.searchUrl(sr)
body := strings.NewReader(post)
req, err := http.NewRequest("POST", playurl, body)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value")
req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36")
client := sr.Client
resp, err := client.Do(req)
if err != nil {
return nil, err
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return nil, err
}
cards := doc.Find(".card")
apps := make([]App, cards.Length())
cards.Each(func(i int, s *goquery.Selection) {
if val, ok := s.Attr("data-docid"); ok {
apps[i].Package = val
}
if val, ok := s.Find(".title").Attr("title"); ok {
apps[i].Name = val
}
if val, ok := s.Find(".subtitle").Attr("title"); ok {
apps[i].PublisherName = val
}
if val, ok := s.Find(".card-click-target").Attr("href"); ok {
apps[i].StoreUrl = "https://play.google.com" + val
}
if val, ok := s.Find(".cover-image").Attr("src"); ok {
apps[i].IconUrl = val
}
if val, ok := s.Find(".current-rating").Attr("style"); ok {
rating := strings.Split(val, " ")
if len(rating) >= 2 {
if rating_perc, err := strconv.ParseFloat(strings.TrimRight(rating[1], "%;"), 32); err == nil {
apps[i].Rating = float32(rating_perc * 5.0 / 100.0)
}
}
}
})
return apps, nil
}
示例12: Result
// Get the collection of the ProxyList by parse the http response
func (this *XiCiDaiLi) Result(response *http.Response) ([]contract.ProxyList, error) {
doc, err := goquery.NewDocumentFromResponse(response)
if err != nil {
return nil, err
}
var proxyList []contract.ProxyList
proxyTable := doc.Find("table#ip_list")
proxyRows := proxyTable.Find("tr:nth-child(n+2)")
proxyRows.Each(func(i int, s *goquery.Selection) {
country := s.Find("td:nth-child(2) img").AttrOr("alt", "n/a")
ip := s.Find("td:nth-child(3)").Text()
port := s.Find("td:nth-child(4)").Text()
protocol := s.Find("td:nth-child(7)").Text()
if ip != "" && port != "" {
proxyList = append(proxyList, contract.ProxyList{
Ip: ip,
Port: port,
Protocol: protocol,
Country: strings.ToLower(country),
})
}
})
return proxyList, nil
}
示例13: GetMajorList
func GetMajorList(client *http.Client) []string {
banlist := map[string]string{"110431": "", "110432": "", "110511": "", "110521": "", "113110": "", "113111": ""}
getmajorlist := []string{}
res, err := client.Get(DHUHostUrl + DHUGetMajorURL)
if err != nil {
fmt.Println(err)
fmt.Println("Something Wrong In GetMajorList")
} else {
doc, _ := goquery.NewDocumentFromResponse(res)
doc.Find("select").Eq(1).Find("option").Each(func(i int, s *goquery.Selection) {
sss, ok := s.Attr("value")
if ok {
_, testmap := banlist[sss]
if testmap {
return
} else {
getmajorlist = append(getmajorlist, sss)
}
} else {
fmt.Println("Something Wrong")
}
})
return getmajorlist
}
return nil
}
示例14: GetAllLessons
func GetAllLessons(res *http.Response) []CourseList {
doc, err := goquery.NewDocumentFromResponse(res)
if err != nil {
fmt.Println(err)
return nil
} else {
newlist := []CourseList{}
dec := mahonia.NewDecoder("GB18030")
doc.Find("tr").Each(func(i int, s *goquery.Selection) {
selectid := s.Find("td").Eq(0).Text()
_, err := strconv.Atoi(selectid)
if err == nil {
_, teachername, _ := dec.Translate([]byte(s.Find("td").Eq(6).Text()), true)
newinfo := []CourseInfo{}
s.Find("td").Eq(7).Find("tr").Each(func(i int, s *goquery.Selection) {
_, weektime, _ := dec.Translate([]byte(s.Find("td").Eq(0).Text()), true)
_, daytime, _ := dec.Translate([]byte(s.Find("td").Eq(1).Text()), true)
info := CourseInfo{string(weektime), string(daytime)}
newinfo = append(newinfo, info)
})
courselist := CourseList{selectid, string(teachername), newinfo}
newlist = append(newlist, courselist)
} else {
return
}
})
return newlist
}
}
示例15: getHtml
func (this *Object) getHtml() (string, error) {
res, err := request("GET", this.TargetUrl, "")
if err != nil {
return "", err
}
if res.StatusCode != 200 {
return "", errors.New(fmt.Sprintf("http status not 200, it is %d", res.StatusCode))
}
doc, err := goquery.NewDocumentFromResponse(res)
if err != nil {
return "", err
}
if err != nil {
return "", err
}
html, err := this.parse(doc)
if err != nil {
return "", err
}
this.setArgs(doc)
if err := this.setPageLimit(doc); err != nil {
return "", err
}
return html, nil
}