Golang html.Parse函数代码示例

func (y *Youtube) fetchMoreHistory(moreHref string) (more, content *html.Node, err error) {
	moreURL := "https://www.youtube.com" + moreHref
	moreReq, err := http.NewRequest("GET", moreURL, nil)
	if err != nil {
		return nil, nil, err
	}
	resp, err := y.s.Do(moreReq)
	if err != nil {
		return nil, nil, err
	}
	defer resp.Body.Close()

	var jsonDoc struct {
		Content string `json:"content_html"`
		More    string `json:"load_more_widget_html"`
	}

	if err := json.NewDecoder(resp.Body).Decode(&jsonDoc); err != nil {
		return nil, nil, err
	}

	content, err = html.Parse(bytes.NewBufferString(jsonDoc.Content))
	if err != nil {
		return nil, nil, err
	}
	more, _ = html.Parse(bytes.NewBufferString(jsonDoc.More))

	return
}

func scrape(cliaoke_dir string) error {
	base_uri := "http://www.albinoblacksheep.com/audio/midi/"
	response, err := http.Get(base_uri)
	if err != nil {
		return errors.New("Couldn't fetch base content: " + err.Error())
	}
	defer response.Body.Close()

	doc, err := html.Parse(response.Body)
	if err != nil {
		return errors.New("Couldn't understand document body: " + err.Error())
	}

	for _, option := range get_options(doc) {
		slug := get_value(option)
		// log.Println(option)
		if slug != "" {
			var embed *html.Node
			page := base_uri + "/" + slug
			// log.Println(page)
			err := func() error {
				response, err := http.Get(page)
				if err != nil {
					return errors.New("Error fetching page " + page + ": " + err.Error())
				}
				defer response.Body.Close()
				slugdoc, err := html.Parse(response.Body)
				if err != nil {
					return errors.New("Error parsing page " + page + ": " + err.Error())
				}
				embed = get_embed(slugdoc)
				return nil
			}()
			if err != nil {
				return err
			}
			var file_url string
			for _, r := range embed.Attr {
				if r.Key == "src" {
					err = nil
					file_url = r.Val
					goto SKIP_ERR
				}
			}
			return errors.New("No src attribute found in embed in page " + page)

		SKIP_ERR:
			if err = download_file(cliaoke_dir, file_url); err != nil {
				return err
			}
		}
	}
	return nil
}

func getNodeWithCharset(source []byte) (*html.Node, error) {
	n, err := html.Parse(bytes.NewReader(source))
	if err != nil {
		return nil, err
	}

	charset := getCharset(n)
	if charset == CS_UTF8 {
		return n, nil
	}

	data := convertString(string(source), charset, CS_UTF8)
	n, err = html.Parse(bytes.NewReader([]byte(data)))
	return n, err
}

// Parse a tree from r.
func Parse(r io.Reader) (*Node, error) {
	dom, err := html.Parse(r)
	if err != nil {
		return nil, err
	}
	return Convert(dom), nil
}

func urlToNode(url string, client *http.Client, redirect int) (*html.Node, error) {
	if redirect >= MAX_REDIRECT {
		return nil, fmt.Errorf("Too many canonical redirection at %s", url)
	}
	resp, err := client.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		if resp.StatusCode < 500 {
			return nil, nil
		} else {
			return nil, fmt.Errorf("Server returns %d code (url: %s).", resp.StatusCode, url)
		}
	}

	root, err := html.Parse(resp.Body)
	if err != nil {
		return nil, err
	}
	s := getCanonicalUrl(root)
	if s != "" && s != url {
		return urlToNode(s, client, redirect+1)
	}
	return root, err
}

func Extract(url string) ([]string, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		resp.Body.Close()
		return nil, fmt.Errorf("getting %s: %s", url, resp.Status)
	}

	doc, err := html.Parse(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, fmt.Errorf("parsing %s as HTML: %v", url, err)
	}

	var links []string
	visitNode := func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "a" {
			for _, a := range n.Attr {
				if a.Key != "href" {
					continue
				}
				link, err := resp.Request.URL.Parse(a.Val)
				if err != nil {
					continue
				}
				links = append(links, link.String())
			}
		}
	}
	forEachNode(doc, visitNode, nil)
	return links, nil
}

func title(url string) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}

	// Check Content-Type is HTML (e.g., "text/html; charset=utf-8").
	ct := resp.Header.Get("Content-Type")
	if ct != "text/html" && !strings.HasPrefix(ct, "text/html;") {
		resp.Body.Close()
		return fmt.Errorf("%s has type %s, not text/html", url, ct)
	}

	doc, err := html.Parse(resp.Body)
	resp.Body.Close()
	if err != nil {
		return fmt.Errorf("parsing %s as HTML: %v", url, err)
	}
	title, err := soleTitle(doc)
	if err != nil {
		return err
	}
	fmt.Println(title)
	return nil
}

func TestWhitelist(t *testing.T) {

	originalHTML := `<html><head></head><body>0&lt;1<p id="A" foo="abc&#34;def">` +
		`2<b empty="">3</b><i backslash="\">&amp;4</i></p>` +
		`5<blockquote></blockquote><br/>6</body></html>`
	originalHTMLAsBuffer := bytes.NewBufferString(originalHTML)

	rootNode, err := html.Parse(originalHTMLAsBuffer)

	if err != nil {
		t.Fatal(err)
	}

	dummyAttributeArray := []htmlrender.MinimalHtmlNode{
		{
			Data: "blockquote",
		},
	}

	w := new(bytes.Buffer)
	want := `<blockquote></blockquote>`

	if err := Whitelist(w, rootNode, dummyAttributeArray); err != nil {
		t.Fatal(err)
	}
	if got := w.String(); got != want {
		t.Errorf("got vs want:\n%s\n%s\n", got, want)
	}
}

func outline(url string) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	doc, err := html.Parse(resp.Body)
	if err != nil {
		return err
	}

	var depth int
	forEachNode(doc,
		func(n *html.Node) {
			if n.Type == html.ElementNode {
				fmt.Printf("%*s<%s>\n", depth*2, "", n.Data)
				depth++
			}
		},
		func(n *html.Node) {
			if n.Type == html.ElementNode {
				depth--
				fmt.Printf("%*s</%s>\n", depth*2, "", n.Data)
			}
		})

	return nil
}

func Parse(r io.Reader) (*Node, error) {
	n, err := html.Parse(r)
	if err != nil {
		return nil, err
	}
	return NewNode(n), nil
}

func getRootNode(urlToOpen string) *xmlpath.Node {
	req, err := http.NewRequest("GET", urlToOpen, nil)
	if err != nil {
		log("error1: " + err.Error())
	} else {
		response, err := http.DefaultTransport.RoundTrip(req)
		if err != nil {
			log("Error 3: " + err.Error())
		} else {
			defer response.Body.Close()
			content, err := ioutil.ReadAll(response.Body)
			if err != nil {
				log("Error 2: " + err.Error())
			} else {
				root, err := html.Parse(bytes.NewReader(content))

				if err != nil {
					log("Parse error: " + err.Error())
				}

				var b bytes.Buffer
				html.Render(&b, root)
				fixedHtml := b.String()
				reader := strings.NewReader(fixedHtml)
				rootNode, err := xmlpath.ParseHTML(reader)
				if err != nil {
					log("Error 4: " + err.Error())
				} else {
					return rootNode
				}
			}
		}
	}
	return nil
}

func TestHtmlTagCount(t *testing.T) {
	var tests = []struct {
		args string
		want map[string]int
	}{
		{`<html><head></head><body><a href="foo">Foo</a></body></html>`, map[string]int{"html": 1, "head": 1, "body": 1, "a": 1}},
		{`<html><head></head><body><ul><li><a href="/foo">Foo</a></li><li><a href="/bar">Bar</a></li></ul></body></html>`, map[string]int{"html": 1, "head": 1, "body": 1, "a": 2, "ul": 1, "li": 2}},
		{`<html><head></head><body><ul><li><a href="/foo">Foo</a></li><li><a href="/bar">Bar</a></li></ul><ul><li><a href="/hoge">Hoge</a></li><li><a href="/piyo">Piyo</a></li></ul></body></html>`, map[string]int{"html": 1, "head": 1, "body": 1, "a": 4, "ul": 2, "li": 4}},
	}

	for _, test := range tests {
		descr := fmt.Sprintf("htmlTagCount(%q)", test.args)
		doc, err := html.Parse(strings.NewReader(test.args))
		if err != nil {
			log.Fatal(err)
		}
		var counts map[string]int
		counts = map[string]int{}
		htmlTagCount(counts, doc)
		if !reflect.DeepEqual(counts, test.want) {
			t.Errorf("%s", descr)
			t.Errorf("got")
			for tagName, tagCount := range counts {
				t.Errorf("tagName = %s, tagCount = %d", tagName, tagCount)
			}
			t.Errorf("expect")
			for tagName, tagCount := range test.want {
				t.Errorf("tagName = %s, tagCount = %d", tagName, tagCount)
			}
		}
	}
}

func GetParameters(client *http.Client, site string) error {
	url, err := url.ParseRequestURI(site)
	if err != nil {
		return err
	}

	url.Path = "/parameters/profile/all"

	respBody, err := DoRequest(client, url, "GET", nil, nil)
	if err != nil {
		return err
	}

	defer respBody.Close()

	doc, err := html.Parse(respBody)
	if err != nil {
		return err
	}

	if verbose {
		fmt.Println("HTML doc parsed ok", "type:", doc.Type, "data:", doc.Data)
	}

	err = CheckHtml(doc, PARAMETERS_PAGE_TITLE)
	if err != nil {
		return err
	}

	return nil
}

func (u *URLMod) ParseMessage(msg *gochat.Message, c *gochat.Channel) string {
	url := u.Re.FindString(msg.Text)
	response, err := http.Get(url)
	if err != nil {
		return "Error, could not get URL!"
	} else {
		defer response.Body.Close()
		doc, _ := html.Parse(response.Body)
		var f func(*html.Node, bool)
		var re string
		f = func(n *html.Node, pt bool) {
			if pt && n.Type == html.TextNode {
				re = string("Link Title: " + n.Data)
				return
			}
			pt = pt || (n.Type == html.ElementNode && n.Data == "title")
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				f(c, pt)
			}
		}
		f(doc, false)
		return re
	}
	//return ""
}

func main() {
	doc, err := html.Parse(os.Stdin)
	if err != nil {
		log.Fatal(err)
	}
	printTextNodes(doc)
}