本文整理汇总了Golang中github.com/yahoo/gryffin.Scan.Logmf方法的典型用法代码示例。如果您正苦于以下问题：Golang Scan.Logmf方法的具体用法？Golang Scan.Logmf怎么用？Golang Scan.Logmf使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类github.com/yahoo/gryffin.Scan的用法示例。
在下文中一共展示了Scan.Logmf方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Golang代码示例。
示例1: Do
func (r *PhantomJSRenderer) Do(s *gryffin.Scan) {
r.chanResponse = make(chan *gryffin.Scan, 10)
r.chanLinks = make(chan *gryffin.Scan, 10)
r.done = make(chan string)
// Construct the command.
// render.js http(s)://<host>[:port][/path] [{"method":"post", "data":"a=1&b=2"}]
url := s.Request.URL.String()
cookies := make([]string, 0)
// ua := s.Request.UserAgent()
ua := "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36"
for _, c := range s.Cookies {
cookies = append(cookies, c.String())
}
arg := input{
Method: s.Request.Method,
Headers: inputHeaders{
UserAgent: ua,
Cookie: strings.Join(cookies, ";"),
},
}
opt, err := json.Marshal(arg)
if err != nil {
s.Error("PhantomjsRenderer.Do", err)
return
}
// s.Logmf("PhantomjsRenderer.Do", "Running: render.js %s '%s'", url, string(opt))
s.Logmf("PhantomjsRenderer.Do", "Running: render.js")
cmd := exec.Command(
os.Getenv("GOPATH")+"/src/github.com/yahoo/gryffin/renderer/resource/render.js",
url,
string(opt))
stdout, err := cmd.StdoutPipe()
if err != nil {
s.Error("PhantomjsRenderer.Do", err)
return
}
if err := cmd.Start(); err != nil {
s.Error("PhantomjsRenderer.Do", err)
return
}
r.process = cmd.Process
// wait until done or timeout.
go r.extract(stdout, s)
go r.wait(s)
cmd.Wait()
}示例2: kill
func (r *PhantomJSRenderer) kill(reason string, s *gryffin.Scan) {
if err := r.process.Kill(); err == nil {
s.Logmf("PhantomjsRenderer.Do", "[%s] Terminating the crawl process.", reason)
}
}示例3: Do
func (r *PhantomJSRenderer) Do(s *gryffin.Scan) {
r.chanResponse = make(chan *gryffin.Scan, 10)
r.chanLinks = make(chan *gryffin.Scan, 10)
// Construct the command.
// render.js http(s)://<host>[:port][/path] [{"method":"post", "data":"a=1&b=2"}]
url := s.Request.URL.String()
cookies := make([]string, 0)
// ua := s.Request.UserAgent()
ua := "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36"
for _, c := range s.Cookies {
cookies = append(cookies, c.String())
}
arg := input{
Method: s.Request.Method,
Headers: inputHeaders{
UserAgent: ua,
Cookie: strings.Join(cookies, ";"),
},
}
opt, err := json.Marshal(arg)
if err != nil {
s.Error("PhantomjsRenderer.Do", err)
return
}
// s.Logmf("PhantomjsRenderer.Do", "Running: render.js %s '%s'", url, string(opt))
s.Logmf("PhantomjsRenderer.Do", "Running: render.js")
cmd := exec.Command(
os.Getenv("GOPATH")+"/src/github.com/yahoo/gryffin/renderer/resource/render.js",
url,
string(opt))
stdout, err := cmd.StdoutPipe()
if err != nil {
s.Error("PhantomjsRenderer.Do", err)
return
}
if err := cmd.Start(); err != nil {
s.Error("PhantomjsRenderer.Do", err)
return
}
kill := func(reason string) {
if err := cmd.Process.Kill(); err != nil {
// TODO - forgive "os: process already finished"
s.Error("PhantomjsRenderer.Do", err)
// log.Printf("error: %s", err)
} else {
s.Logmf("PhantomjsRenderer.Do", "[%s] Terminating the crawl process.", reason)
}
}
// Kill when timeout
_ = time.Second
if r.Timeout != 0 {
timeout := func() {
<-time.After(time.Duration(r.Timeout) * time.Second)
kill("Timeout")
}
go timeout()
}
crawl := func() {
defer close(r.chanResponse)
defer close(r.chanLinks)
dec := json.NewDecoder(stdout)
for {
var m message
err := dec.Decode(&m)
if err == io.EOF {
return
break
} else {
if m.responseMessage != nil {
m.Response.fill(s)
if s.IsDuplicatedPage() {
kill("Duplicated")
return
}
s.Logm("PhantomjsRenderer.Do.UniqueCrawl", m.MsgType)
r.chanResponse <- s
for _, link := range m.Response.Details.Links {
if newScan := link.toScan(s); newScan != nil && newScan.IsScanAllowed() {
r.chanLinks <- newScan
}
}
} else if m.domMessage != nil {
for _, link := range m.domMessage.Links {
if newScan := link.toScan(s); newScan != nil && newScan.IsScanAllowed() {
r.chanLinks <- newScan
}
}
//.........这里部分代码省略.........