Golang html.HtmlDocument類代碼示例

func (record *Record) processOffJoins(doc *html.HtmlDocument) {
	xPath := xpath.NewXPath(doc.DocPtr())
	// join the offence with the defendants and verdict
	joinPtrs := xPath.Evaluate(doc.Root().NodePtr(),
		xpath.Compile("//join[@result='criminalCharge']"))

	for _, nodePtr := range joinPtrs {
		node := xml.NewNode(nodePtr, doc)
		targets := strings.Split(node.Attr("targets"), " ")
		var personId, offId, verdictId string
		for _, targetId := range targets {
			if strings.Contains(targetId, "defend") {
				personId = targetId
			}
			if strings.Contains(targetId, "off") {
				offId = targetId
			}
			if strings.Contains(targetId, "verdict") {
				verdictId = targetId
			}
		}
		offence := record.findOffence(offId)
		if offence == nil {
			panic("couldn't find offence " + offId)
		}
		person := record.findPerson(personId)
		if person != nil {
			offence.Defendants = append(offence.Defendants, person)
		}
		verdict := record.findVerdict(verdictId)
		if verdict != nil {
			offence.Verdict = verdict
		}
	}
}

/*
 * Invokes a MIME type plugin based on current node's type attribute, passing src attribute's value
 * as argument. Subcommand's output is piped to Gokogiri through a buffer.
 */
func (gen *Generator) handleMIMETypePlugin(e xml.Node, doc *html.HtmlDocument) (err error) {
	src := e.Attribute("src").Value()
	typ := e.Attribute("type").Value()
	cmd := exec.Command(fmt.Sprintf("m%s%s", ZAS_PREFIX, gen.resolveMIMETypePlugin(typ)), src)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return
	}
	cmd.Stderr = os.Stderr
	c := make(chan bufErr)
	go func() {
		data, err := ioutil.ReadAll(stdout)
		c <- bufErr{data, err}
	}()
	if err = cmd.Start(); err != nil {
		return
	}
	be := <-c
	if err = cmd.Wait(); err != nil {
		return
	}
	if be.err != nil {
		return be.err
	}
	parent := e.Parent()
	child, err := doc.Coerce(be.buffer)
	if err != nil {
		return
	}
	parent.AddChild(child)
	e.Remove()
	return
}

func parseMandiriHtml(document *html.HtmlDocument) (map[string]Currency, error) {
	nilai := "1.00"
	var matauang string
	var kursjual string
	var kursbeli string

	var kurs = make(map[string]Currency)
	doc, err := document.Search("//table[@class='tbl-view']/tr")
	for i, tr := range doc {
		t := 0
		for td := tr.FirstChild(); td != nil; td = td.NextSibling() {
			teks := strings.TrimSpace(td.Content())
			if i > 0 && i <= 15 && len(teks) > 0 {
				if t == 1 {
					matauang = teks
				} else if t == 2 {
					kursbeli = teks
				} else if t == 3 {
					kursjual = teks
				}
				t += 1

				if kursjual != "" && kursbeli != "" {
					kurs[matauang] = Currency{Nilai: nilai, KursJual: kursjual, KursBeli: kursbeli}
				}
			}
		}
	}

	return kurs, err
}

/*
 * Extracts first HTML commend as map. It expects it as a valid YAML map.
 */
func (gen *Generator) extractPageConfig(doc *html.HtmlDocument) (config map[interface{}]interface{}, err error) {
	result, _ := doc.Search("//comment()")
	if len(result) > 0 {
		err = yaml.Unmarshal([]byte(result[0].Content()), &config)
	}
	return
}

/*
 * Handles <embed> tags.
 *
 * They can be handled with MIME type plugins or internal exported methods like Markdown.
 */
func (gen *Generator) handleEmbedTags(doc *html.HtmlDocument) (err error) {
	result, err := doc.Search("//embed")
	if err != nil {
		return
	}
	for _, e := range result {
		plugin := gen.resolveMIMETypePlugin(e.Attribute("type").Value())
		method := reflect.ValueOf(gen).MethodByName(strings.Title(plugin))
		if method == reflect.ValueOf(nil) {
			err = gen.handleMIMETypePlugin(e, doc)
		} else {
			args := make([]reflect.Value, 2)
			args[0] = reflect.ValueOf(e)
			args[1] = reflect.ValueOf(doc)
			r := method.Call(args)
			rerr := r[0].Interface()
			if ierr, ok := rerr.(error); ok {
				err = ierr
			}
		}
		if err != nil {
			return
		}
	}
	return
}

/*
 * Removes unnecessary paragraph HTML tags generated during Markdown processing by
 * deleting any <p> without child text nodes (just to avoid deletion if semantic tags
 * are inside).
 */
func (gen *Generator) cleanUnnecessaryPTags(doc *html.HtmlDocument) (err error) {
	ps, err := doc.Search("//p")
	if err != nil {
		return
	}
	for _, p := range ps {
		hasText := false
		child := p.FirstChild()
		for child != nil {
			typ := child.NodeType()
			if typ == xml.XML_TEXT_NODE {
				// Little heuristic to remove nodes with visually empty content.
				content := strings.TrimSpace(child.Content())
				if content != "" {
					hasText = true
					break
				}
			}
			child = child.NextSibling()
		}
		// If current <p> tag doesn't have any child text node, extract children and add to its parent.
		if !hasText {
			parent := p.Parent()
			child = p.FirstChild()
			for child != nil {
				parent.AddChild(child)
				child = child.NextSibling()
			}
			p.Remove()
		}
	}
	return
}

/*
 * Returns first H1 tag as page title.
 */
func (gen *Generator) getTitle(doc *html.HtmlDocument) (title string) {
	result, _ := doc.Search("//h1")
	if len(result) > 0 {
		title = result[0].FirstChild().Content()
	}
	return
}

func parseBiHtml(document *html.HtmlDocument) (map[string]Currency, error) {
	var matauang string
	var nilai string
	var kursjual string
	var kursbeli string

	kurs := make(map[string]Currency)

	doc, err := document.Search("//table[@id='ctl00_PlaceHolderMain_biWebKursTransaksiBI_GridView1']/tr")
	for i, tr := range doc {
		t := 0
		for td := tr.FirstChild(); td != nil; td = td.NextSibling() {
			teks := strings.TrimSpace(td.Content())
			if i > 0 && len(teks) > 0 {
				if t == 0 {
					matauang = teks
				} else if t == 1 {
					nilai = teks
				} else if t == 2 {
					kursjual = teks
				} else if t == 3 {
					kursbeli = teks
				}
				t += 1
				kurs[matauang] = Currency{Nilai: nilai, KursJual: kursjual, KursBeli: kursbeli}
			}
		}
	}

	return kurs, err
}

func getLastUpdatedMandiri(document *html.HtmlDocument) string {
	str, _ := document.Search("//p[@class='catatan']")
	firstData := strings.Split(strings.Split(str[0].InnerHtml(), "<br>")[0], " ")
	lastUpdated := firstData[2] + " " + firstData[3] + " " + firstData[4] + " " + firstData[5] + " " + firstData[6]

	return lastUpdated
}

func getNewsInfo(doc *html.HtmlDocument) ([]xml.Node, error) {
	xp := "//body/div/div/div/div/div/div/div/div/div/div/div/div/div/div/ul/li"
	xps := xpath.Compile(xp)
	newDatas, err := doc.Root().Search(xps)
	if err != nil {
		return nil, err
	}
	return newDatas, nil
}

// get the value out of an <interp> tag
func getInterp(basePtr unsafe.Pointer, interpType string, doc *html.HtmlDocument) (value string) {
	xPath := xpath.NewXPath(doc.DocPtr())
	nodePtrs := xPath.Evaluate(basePtr, xpath.Compile(".//interp[@type='"+
		interpType+"']"))
	if len(nodePtrs) == 1 {
		node := xml.NewNode(nodePtrs[0], doc)
		value = node.Attr("value")
	}
	return
}

func (record *Record) processVerdicts(doc *html.HtmlDocument) {
	xPath := xpath.NewXPath(doc.DocPtr())
	verdictPtrs := xPath.Evaluate(doc.Root().NodePtr(),
		xpath.Compile("//rs[@type='verdictDescription']"))
	verdicts := make([]Verdict, len(verdictPtrs))

	for i, nodePtr := range verdictPtrs {
		node := xml.NewNode(nodePtr, doc)
		verdict := Verdict{}
		verdict.Id = node.Attr("id")
		verdict.Desc = cleanUpContent(node.Content())
		verdict.SetType(getInterp(nodePtr, "verdictCategory", doc))
		verdicts[i] = verdict
	}
	record.Verdicts = verdicts
}

func (record *Record) processOffences(doc *html.HtmlDocument) {
	xPath := xpath.NewXPath(doc.DocPtr())
	offencePtrs := xPath.Evaluate(doc.Root().NodePtr(),
		xpath.Compile("//rs[@type='offenceDescription']"))
	offences := make([]Offence, len(offencePtrs))

	for i, nodePtr := range offencePtrs {
		node := xml.NewNode(nodePtr, doc)
		offence := Offence{}
		offence.Id = node.Attr("id")
		offence.Category = getInterp(nodePtr, "offenceCategory", doc)
		offence.SubCategory = getInterp(nodePtr, "offenceSubcategory", doc)
		offence.Desc = cleanUpContent(node.Content())
		offences[i] = offence
	}
	record.Offences = offences
}

func docSearch(doc *ghtml.HtmlDocument, elementName string, pageName string, xpath string, mustFind bool) []gxml.Node {
	elementArray, err := doc.Root().Search(xpath)
	if (err != nil || len(elementArray) == 0) && mustFind == false {
		return nil
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error locating element \"%s\" in page %s (incorrect xpath?): %v\n", elementName, pageName, err)
		//fmt.Fprintf(os.Stderr, " doc=%+v\n", doc)
		os.Exit(1)
	}
	if len(elementArray) == 0 {
		fmt.Fprintf(os.Stderr, "Error locating element \"%s\" in page %s (incorrect xpath?): len() == 0\n", elementName, pageName)
		//fmt.Fprintf(os.Stderr, " doc=%+v\n", doc)
		os.Exit(1)
	}
	return elementArray
}

func (record *Record) processPersons(doc *html.HtmlDocument) {
	xPath := xpath.NewXPath(doc.DocPtr())
	personPtrs := xPath.Evaluate(doc.Root().NodePtr(),
		xpath.Compile("//persname"))
	persons := make([]Person, len(personPtrs))

	for i, nodePtr := range personPtrs {
		node := xml.NewNode(nodePtr, doc)
		person := Person{}
		person.Id = node.Attr("id")
		person.GivenName = getInterp(nodePtr, "given", doc)
		person.Surname = getInterp(nodePtr, "surname", doc)
		person.SetType(node.Attr("type"))
		person.SetGender(getInterp(nodePtr, "gender", doc))
		persons[i] = person
	}
	record.Persons = persons
}