本文整理汇总了Golang中github.com/yahoo/gryffin.Scan.Spawn方法的典型用法代码示例。如果您正苦于以下问题：Golang Scan.Spawn方法的具体用法？Golang Scan.Spawn怎么用？Golang Scan.Spawn使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类github.com/yahoo/gryffin.Scan的用法示例。
在下文中一共展示了Scan.Spawn方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Golang代码示例。
示例1: toScan
func (l *link) toScan(parent *gryffin.Scan) *gryffin.Scan {
if req, err := http.NewRequest("GET", l.Url, nil); err == nil {
s := parent.Spawn()
s.MergeRequest(req)
return s
}
// invalid url
return nil
}示例2: toScan
func (f *form) toScan(parent *gryffin.Scan) *gryffin.Scan {
m := strings.ToUpper(f.Method)
u := f.Url
var r io.Reader
if m == "POST" {
r = ioutil.NopCloser(strings.NewReader(f.Data))
} else {
u += "&" + f.Data
}
if req, err := http.NewRequest(m, u, r); err == nil {
s := parent.Spawn()
s.MergeRequest(req)
return s
}
// invalid url
return nil
}示例3: Do
func (r *NoScriptRenderer) Do(s *gryffin.Scan) {
r.chanResponse = make(chan *gryffin.Scan, 10)
r.chanLinks = make(chan *gryffin.Scan, 10)
crawl := func() {
defer close(r.chanResponse)
defer close(r.chanLinks)
client := &http.Client{}
client.Timeout = time.Duration(3) * time.Second
if response, err := client.Do(s.Request); err == nil {
s.Response = response
} else {
s.Logm("NoScriptRenderer", fmt.Sprintf("error in building request: %s", err))
return
}
s.ReadResponseBody()
if s.IsDuplicatedPage() {
return
}
tokenizer := html.NewTokenizer(strings.NewReader(s.ResponseBody))
r.chanResponse <- s
for {
t := tokenizer.Next()
switch t {
case html.ErrorToken:
return
case html.StartTagToken:
token := tokenizer.Token()
if token.DataAtom.String() == "a" {
for _, attr := range token.Attr {
if attr.Key == "href" {
link := s.Spawn()
// TODO - we drop relative URL as it would drop "#".
// Yet, how about real relative URLs?
if req, err := http.NewRequest("GET", attr.Val, nil); err == nil {
if true {
// || req.URL.IsAbs() {
link.MergeRequest(req)
if link.IsScanAllowed() {
r.chanLinks <- link
}
} else {
// ignore relative URL. TOFIX.
}
} else {
log.Printf("error in building request: %s", err)
}
}
}
}
}
}
// parse and find links.
}
go crawl()
return
}