Golang Selection.Find方法代码示例

// Serialize converts the form fields into a url.Values type.
// Returns two url.Value types. The first is the form field values, and the
// second is the form button values.
func serializeForm(sel *goquery.Selection) (url.Values, url.Values) {
	input := sel.Find("input,button,textarea")
	if input.Length() == 0 {
		return url.Values{}, url.Values{}
	}

	fields := make(url.Values)
	buttons := make(url.Values)
	input.Each(func(_ int, s *goquery.Selection) {
		name, ok := s.Attr("name")
		if ok {
			typ, ok := s.Attr("type")
			if ok || s.Is("textarea") {
				if typ == "submit" {
					val, ok := s.Attr("value")
					if ok {
						buttons.Add(name, val)
					} else {
						buttons.Add(name, "")
					}
				} else {
					val, ok := s.Attr("value")
					if !ok {
						val = ""
					}
					fields.Add(name, val)
				}
			}
		}
	})

	return fields, buttons
}

func encuentraHorarios(tabla *goquery.Selection) (horarios []Horario) {
	trs := tabla.Find("tr")
	trs.Each(func(i int, tr *goquery.Selection) {
		tds := tr.Find("td")
		var puesto, nombre, diasStr, horaStr, salonStr string
		if tds.Length() > 0 {
			puesto = strings.TrimSpace(tds.Eq(0).Text())
		}
		if tds.Length() > 1 {
			nombre = strings.TrimSpace(tds.Eq(1).Text())
		}
		if tds.Length() > 2 {
			diasStr = strings.TrimSpace(tds.Eq(2).Text())
		}
		if tds.Length() > 3 {
			horaStr = strings.TrimSpace(tds.Eq(3).Text())
		}
		if tds.Length() > 4 {
			salonStr = strings.TrimSpace(tds.Eq(4).Text())
		}
		dia := generaDia(diasStr, horaStr, salonStr)
		horario := Horario{puesto: puesto, nombre: nombre, dias: dia}
		horarios = append(horarios, horario)
	})
	return
}

func parsePhrase(selection *goquery.Selection) (result []string) {
	selection.Find(".translation_item").Each(func(index int, meaning *goquery.Selection) {
		result = append(result, strings.TrimSpace(meaning.Text()))
	})

	return
}

func testList(t *testing.T, list *goquery.Selection) {
	list.Find("ul").Each(func(_ int, items *goquery.Selection) {
		testList(t, items)
		items.RemoveFiltered("ul")
	})
	checkAlphabeticOrder(t, list)
}

// Parse from div.tweet
func (tweet *Tweet) Parse(s *goquery.Selection) (err error) {
	success := false
	attrs := []string{
		"data-item-id",
		"data-screen-name",
		"data-name",
	}
	data := map[string]string{}

	for _, attr := range attrs {
		var value string
		if value, success = s.Attr(attr); !success {
			tweet.Success = 0
			err = fmt.Errorf("not having %s attribute", attr)
			return
		}
		data[attr] = value
	}

	tweet.ItemID = data["data-item-id"]
	tweet.ScreenName = data["data-screen-name"]
	tweet.Name = data["data-name"]
	tweet.Success = 1

	// if could get the above attribues, allow the following values to be blank.
	tweet.Time, _ = s.Find("._timestamp").Attr("data-time")
	tweet.Text = s.Find(".tweet-text").Text()
	return
}

func ScrapeExamples(s *goquery.Selection) []string {
	examples := []string{}
	s.Find("span.h").Each(func(i int, s *goquery.Selection) {
		examples = append(examples, s.Text())
	})
	return examples
}

func scrapPayload(s *goquery.Selection, n int) string {
	url, ok := s.Find("a").Attr("href")
	if !ok {
		die("unable to find URL for scrapping")
	}
	return scrapPayloadURL("https://developer.github.com"+url, n)
}

func extractCredits(selection *goquery.Selection) string {
	if result := trim(selection.Find(".credits").Text()); strings.Contains(result, "#") {
		return "0"
	} else {
		return result
	}
}

func ParseCourse(s *goquery.Selection) Course {
	subject := strings.TrimSpace(s.Find("td").Eq(0).Text())
	catalog := strings.TrimSpace(s.Find("td").Eq(1).Text())
	termStr := strings.TrimSpace(s.Find("td").Eq(2).Text())
	class := strings.TrimSpace(s.Find("td").Eq(3).Text())
	title := strings.TrimSpace(s.Find("td").Eq(4).Text())
	instructor := strings.TrimSpace(s.Find("td").Eq(5).Text())
	credits := strings.TrimSpace(s.Find("td").Eq(6).Text())
	catalogNum, _ := strconv.Atoi(catalog)
	classNum, _ := strconv.Atoi(strings.TrimSpace(class))

	// Damn you unicode NBSP!!!
	filter := strings.Replace(termStr, "\u0020", "", -1)
	termCleaned := strings.Split(filter, "\u00A0")[0]

	course := Course{
		Subject:       subject,
		CatalogNumber: catalogNum,
		ClassNumber:   classNum,
		Title:         title,
		Instructor:    instructor,
		Credits:       credits,
		Term:          NewTerm(termCleaned),
	}
	return course
}

//checks the density of links within a node, is there not much text and most of it contains bad links?
//if so it's no good
func (this *contentExtractor) isHighLinkDensity(node *goquery.Selection) bool {
	links := node.Find("a")
	if links == nil || links.Size() == 0 {
		return false
	}
	text := node.Text()
	words := strings.Split(text, " ")
	nwords := len(words)
	sb := make([]string, 0)
	links.Each(func(i int, s *goquery.Selection) {
		linkText := s.Text()
		sb = append(sb, linkText)
	})
	linkText := strings.Join(sb, "")
	linkWords := strings.Split(linkText, " ")
	nlinkWords := len(linkWords)
	nlinks := links.Size()
	linkDivisor := float64(nlinkWords) / float64(nwords)
	score := linkDivisor * float64(nlinks)

	if this.config.debug {
		logText := ""
		if len(node.Text()) >= 51 {
			logText = node.Text()[0:50]
		} else {
			logText = node.Text()
		}
		log.Printf("Calculated link density score as %1.5f for node %s\n", score, logText)
	}
	if score > 1.0 {
		return true
	}
	return false
}

func extractCourseDescription(selection *goquery.Selection) string {
	url := trim(fmt.Sprintln(selection.Find(".catalogdescription a").AttrOr("href", "")))
	fmt.Println("LOGGING URL", url)
	client := http.Client{}
	req, _ := http.NewRequest("GET", "http://catalog.njit.edu/ribbit/index.cgi?format=html&page=fsinjector.rjs&fullpage=true", nil)
	req.Header.Add("Referer", url)
	resp, err := client.Do(req)
	if err != nil {

		return ""
	}
	if resp != nil {
		defer resp.Body.Close()
	}

	body, _ := ioutil.ReadAll(resp.Body)
	//checkError(err)
	result := substringAfter(string(body), "courseblockdesc")
	if len(result) < 4 {
		return ""
	}
	result = substringBefore(result[3:], "<b")
	if string(result[0]) == "<" || strings.Contains(result, "at SISConnxService") {
		return ""
	}
	result = strings.Replace(result, "\\\"", "\"", -1)
	doc, _ := goquery.NewDocumentFromReader(strings.NewReader(result))

	return trim(doc.Text())
}

func (this *contentExtractor) getSiblingsContent(currentSibling *goquery.Selection, baselinescoreSiblingsPara float64) []*goquery.Selection {
	ps := make([]*goquery.Selection, 0)
	if currentSibling.Get(0).DataAtom.String() == "p" && len(currentSibling.Text()) > 0 {
		ps = append(ps, currentSibling)
		return ps
	} else {
		potentialParagraphs := currentSibling.Find("p")
		potentialParagraphs.Each(func(i int, s *goquery.Selection) {
			text := s.Text()
			if len(text) > 0 {
				ws := this.config.stopWords.stopWordsCount(this.config.targetLanguage, text)
				paragraphScore := ws.stopWordCount
				siblingBaselineScore := 0.30
				highLinkDensity := this.isHighLinkDensity(s)
				score := siblingBaselineScore * baselinescoreSiblingsPara
				if score < float64(paragraphScore) && !highLinkDensity {
					node := new(html.Node)
					node.Type = html.TextNode
					node.Data = text
					node.DataAtom = atom.P
					nodes := make([]*html.Node, 1)
					nodes[0] = node
					newSelection := new(goquery.Selection)
					newSelection.Nodes = nodes
					ps = append(ps, newSelection)
				}
			}

		})
	}
	return ps
}

func GetText(s *goquery.Selection) string {
	texts, _ := s.Find("td").Html()
	texts = TrimLinefeed(texts)
	texts = strings.Replace(texts, "<br/>", ",", -1)
	texts = strings.Replace(texts, "デッキレベル０再録", "", -1)
	return ReplaceIcon(texts)
}

func ScrapeNotes(s *goquery.Selection) []string {
	notes := []string{}
	s.Find("abbr").Not("abbr:first-of-type").Not("abbr.c").Each(func(i int, s *goquery.Selection) {
		note, _ := s.Attr("title")
		notes = append(notes, note)
	})
	return notes
}

func ScrapeOrigins(s *goquery.Selection) []string {
	origins := []string{}
	s.Find("abbr.c").Each(func(i int, s *goquery.Selection) {
		origin, _ := s.Attr("title")
		origins = append(origins, origin)
	})
	return origins
}