Golang goquery.NewDocumentFromNode函數代碼示例

// Parse 獲取url對應的資源並根據規則進行解析
func (this *RedditLogic) Parse(redditUrl string) error {
	redditUrl = strings.TrimSpace(redditUrl)
	if redditUrl == "" {
		redditUrl = this.domain + this.golang
	} else if !strings.HasPrefix(redditUrl, "https") {
		redditUrl = "https://" + redditUrl
	}

	var (
		doc *goquery.Document
		err error
	)

	// if doc, err = goquery.NewDocument(redditUrl); err != nil {
	if doc, err = this.newDocumentFromResp(redditUrl); err != nil {
		logger.Errorln("goquery reddit newdocument error:", err)
		return err
	}

	// 最後麵的先入庫處理
	resourcesSelection := doc.Find("#siteTable .link")

	for i := resourcesSelection.Length() - 1; i >= 0; i-- {
		err = this.dealRedditOneResource(goquery.NewDocumentFromNode(resourcesSelection.Get(i)).Selection)

		if err != nil {
			logger.Errorln(err)
		}
	}

	return err
}

func GenerateDocument(rawData []byte) *goquery.Document {
	utf8String := toUtf8(rawData)
	utf8byteArray := []byte(utf8String)
	node, err := html.Parse(bytes.NewReader(utf8byteArray))
	helper.HandleFatalError("document generation failed:", err)
	return goquery.NewDocumentFromNode(node)
}

// Process the response for a URL.
func (this *worker) visitUrl(res *http.Response) []*url.URL {
	var doc *goquery.Document
	var harvested []*url.URL
	var doLinks bool

	// Load a goquery document and call the visitor function
	if node, e := html.Parse(res.Body); e != nil {
		this.logFunc(LogError, "ERROR parsing %s: %s\n", res.Request.URL.String(), e.Error())
	} else {
		doc = goquery.NewDocumentFromNode(node)
		doc.Url = res.Request.URL
	}

	// Visit the document (with nil goquery doc if failed to load)
	if this.visitor != nil {
		if harvested, doLinks = this.visitor(res, doc); doLinks && doc != nil {
			// Links were not processed by the visitor, so process links
			harvested = this.processLinks(doc)
		}
	} else {
		this.logFunc(LogInfo, "missing visitor function: %s\n", res.Request.URL.String())
	}

	return harvested
}

func (md *MangaDownloader) HttpGetHtmlDoc(u *url.URL) (*goquery.Document, error) {
	node, err := md.HttpGetHtml(u)
	if err != nil {
		return nil, err
	}
	return goquery.NewDocumentFromNode(node), err
}

func parseChapter() {
	defer chapterWg.Done()

	for chapter := range chaptersIn {
		filename := chapter.path
		b, err := ioutil.ReadFile(filename)

		var m map[string]*json.RawMessage
		err = json.Unmarshal(b, &m)
		if err != nil {
			panic(err)
		}

		var s string
		json.Unmarshal(*m["content"], &s)

		node, err := html.Parse(strings.NewReader(s))
		if err != nil {
			panic(err)
		}

		doc := gq.NewDocumentFromNode(node)
		verses := doc.Find(".verse")
		verses.Each(func(i int, s *gq.Selection) {
			chaptersOut <- Verse{book: chapter.book, chapter: chapter.chapter, html: s}
		})
	}
}

//style="display:none"
func clean_display_none(node *html.Node) *html.Node {
	doc := gq.NewDocumentFromNode(node)
	doc.Find("div").Each(func(i int, s *gq.Selection) {
		if style_display_none(s.AttrOr("style", "")) {
			s.Remove()
		}
	})
	return node
}

func parseHtml(h string) (*goquery.Document, error) {
	node, err := html.Parse(strings.NewReader(h))
	if err != nil {
		log.Println("parsing failed - %s %s", err.Error())
		return nil, err
	}
	doc := goquery.NewDocumentFromNode(node)
	return doc, nil
}

func clean_iframe_none(node *html.Node) *html.Node {
	gq.NewDocumentFromNode(node).Find("iframe").Each(func(i int, s *gq.Selection) {
		if style := s.AttrOr("style", ""); style_display_none(style) || style_dim_is_small(style) {
			s.Remove()
			return
		}
	})

	return node
}

func flattenTable(table *html.Node) (selections []*goquery.Selection) {
	doc := goquery.NewDocumentFromNode(table)
	selections = make([]*goquery.Selection, 0)
	doc.Find("tr").Each(func(i int, row *goquery.Selection) {
		row.RemoveFiltered("tr")
		if row.Text() != "" {
			selections = append(selections, row)
		}
	})
	return
}

func loadDoc(page string) *goquery.Document {
	if f, e := os.Open(fmt.Sprintf("testdata/%s", page)); e != nil {
		panic(e.Error())
	} else {
		defer f.Close()
		if node, e := html.Parse(f); e != nil {
			panic(e.Error())
		} else {
			return goquery.NewDocumentFromNode(node)
		}
	}
	return nil
}

func Print(dao *models.Dao, page models.PageHtml, c *elastigo.Conn) {

	doc, err := html.Parse(strings.NewReader(page.Html))
	if err != nil {
		log.Fatal(err)
	}
	document := goquery.NewDocumentFromNode(doc)
	if document == nil {
	}

	StartParser(dao, document, c)

	fmt.Println("url:%s", page.Url)
}

func generateDoc(htmlStr string, articleUrl string) *goquery.Document {
	node, _ := html.Parse(strings.NewReader(htmlStr))
	doc := goquery.NewDocumentFromNode(node)

	aUrl, err := url.Parse(articleUrl)
	if err == nil {
		doc.Url = aUrl
	} else {

		log.Println(err)
	}

	return doc
}

func LoadDoc(page string) *goquery.Document {
	base, _ := os.Getwd()
	if file, e := os.Open(base + "/" + page); e != nil {
		panic(e.Error())
	} else {
		defer file.Close()
		if node, e := html.Parse(file); e != nil {
			panic(e.Error())
		} else {
			return goquery.NewDocumentFromNode(node)
		}
	}
	return nil
}

//parse the http resp from Townclient
func (g *Ghostparser) ParseReleases() error {
	log.Info("%s parsing %v", TAG, g.Url)

	resp, err := g.Gc.Get(g.Url)
	if err != nil {
		log.Error("%s %s", TAG, err.Error())
		return err
	}
	defer resp.Body.Close()

	respbody, err := html.Parse(resp.Body)
	doc := goquery.NewDocumentFromNode(respbody)

	var rel Release
	doc.Find("table").Each(func(a int, sa *goquery.Selection) {
		if a == 10 { //get the right table
			sa.Find("tr").Each(func(b int, sb *goquery.Selection) {
				sb.Find("td").Each(func(c int, sc *goquery.Selection) {
					if c == 2 {
						rel = Release{}
						g.getUrlAndTagAndName(&rel, sc)

						if rel.Name != "" {
							rel.Time = time.Now().Unix()
							rel.Checksum = g.encodeName(rel.Url)
							rel.checkQual()
							if rel.Name != "" {
								rel.Hits = 0
								rel.Rating = 0
								g.downloadImage(rel.Url, rel.Checksum)
								g.addRelease(rel)
							}
						}
					}
				})
			})
		}
		if g.Count == 0 { //get page count
			if a == 51 {
				sa.Find("a").Each(func(d int, sd *goquery.Selection) {
					if d == 3 {
						g.Count, err = strconv.Atoi(sd.Text())
					}
				})
			}
		}
	})

	return nil
}

func RunCodeTests(tests ConversionTests, fn func(*goquery.Document, string)) error {
	for _, test := range tests {

		node, _ := html.Parse(strings.NewReader(test.In))
		doc := goquery.NewDocumentFromNode(node)

		fn(doc, "")
		got := doc.Text()

		if err := MustBeEqual(got, test.Want); err != nil {
			return err
		}
	}
	return nil
}