本文整理匯總了Golang中github.com/yahoo/gryffin.Scan.Spawn方法的典型用法代碼示例。如果您正苦於以下問題：Golang Scan.Spawn方法的具體用法？Golang Scan.Spawn怎麽用？Golang Scan.Spawn使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類github.com/yahoo/gryffin.Scan的用法示例。
在下文中一共展示了Scan.Spawn方法的3個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Golang代碼示例。
示例1: toScan
func (l *link) toScan(parent *gryffin.Scan) *gryffin.Scan {
if req, err := http.NewRequest("GET", l.Url, nil); err == nil {
s := parent.Spawn()
s.MergeRequest(req)
return s
}
// invalid url
return nil
}示例2: toScan
func (f *form) toScan(parent *gryffin.Scan) *gryffin.Scan {
m := strings.ToUpper(f.Method)
u := f.Url
var r io.Reader
if m == "POST" {
r = ioutil.NopCloser(strings.NewReader(f.Data))
} else {
u += "&" + f.Data
}
if req, err := http.NewRequest(m, u, r); err == nil {
s := parent.Spawn()
s.MergeRequest(req)
return s
}
// invalid url
return nil
}示例3: Do
func (r *NoScriptRenderer) Do(s *gryffin.Scan) {
r.chanResponse = make(chan *gryffin.Scan, 10)
r.chanLinks = make(chan *gryffin.Scan, 10)
crawl := func() {
defer close(r.chanResponse)
defer close(r.chanLinks)
client := &http.Client{}
client.Timeout = time.Duration(3) * time.Second
if response, err := client.Do(s.Request); err == nil {
s.Response = response
} else {
s.Logm("NoScriptRenderer", fmt.Sprintf("error in building request: %s", err))
return
}
s.ReadResponseBody()
if s.IsDuplicatedPage() {
return
}
tokenizer := html.NewTokenizer(strings.NewReader(s.ResponseBody))
r.chanResponse <- s
for {
t := tokenizer.Next()
switch t {
case html.ErrorToken:
return
case html.StartTagToken:
token := tokenizer.Token()
if token.DataAtom.String() == "a" {
for _, attr := range token.Attr {
if attr.Key == "href" {
link := s.Spawn()
// TODO - we drop relative URL as it would drop "#".
// Yet, how about real relative URLs?
if req, err := http.NewRequest("GET", attr.Val, nil); err == nil {
if true {
// || req.URL.IsAbs() {
link.MergeRequest(req)
if link.IsScanAllowed() {
r.chanLinks <- link
}
} else {
// ignore relative URL. TOFIX.
}
} else {
log.Printf("error in building request: %s", err)
}
}
}
}
}
}
// parse and find links.
}
go crawl()
return
}