當前位置: 首頁>>代碼示例>>Golang>>正文


Golang Scan.Error方法代碼示例

本文整理匯總了Golang中github.com/yahoo/gryffin.Scan.Error方法的典型用法代碼示例。如果您正苦於以下問題:Golang Scan.Error方法的具體用法?Golang Scan.Error怎麽用?Golang Scan.Error使用的例子?那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在github.com/yahoo/gryffin.Scan的用法示例。


在下文中一共展示了Scan.Error方法的2個代碼示例,這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚,您的評價將有助於係統推薦出更棒的Golang代碼示例。

示例1: Do

func (r *PhantomJSRenderer) Do(s *gryffin.Scan) {

    r.chanResponse = make(chan *gryffin.Scan, 10)
    r.chanLinks = make(chan *gryffin.Scan, 10)
    r.done = make(chan string)

    // Construct the command.
    // render.js http(s)://<host>[:port][/path] [{"method":"post", "data":"a=1&b=2"}]
    url := s.Request.URL.String()
    cookies := make([]string, 0)
    // ua := s.Request.UserAgent()
    ua := "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36"

    for _, c := range s.Cookies {
        cookies = append(cookies, c.String())
    }

    arg := input{
        Method: s.Request.Method,
        Headers: inputHeaders{
            UserAgent: ua,
            Cookie:    strings.Join(cookies, ";"),
        },
    }

    opt, err := json.Marshal(arg)
    if err != nil {
        s.Error("PhantomjsRenderer.Do", err)
        return
    }

    // s.Logmf("PhantomjsRenderer.Do", "Running: render.js %s '%s'", url, string(opt))
    s.Logmf("PhantomjsRenderer.Do", "Running: render.js")

    cmd := exec.Command(
        os.Getenv("GOPATH")+"/src/github.com/yahoo/gryffin/renderer/resource/render.js",
        url,
        string(opt))

    stdout, err := cmd.StdoutPipe()
    if err != nil {
        s.Error("PhantomjsRenderer.Do", err)
        return
    }

    if err := cmd.Start(); err != nil {
        s.Error("PhantomjsRenderer.Do", err)
        return
    }

    r.process = cmd.Process

    // wait until done or timeout.
    go r.extract(stdout, s)
    go r.wait(s)

    cmd.Wait()

}
開發者ID:binarydreams,項目名稱:gryffin,代碼行數:59,代碼來源:phantomjs.go

示例2: Do

func (r *PhantomJSRenderer) Do(s *gryffin.Scan) {

    r.chanResponse = make(chan *gryffin.Scan, 10)
    r.chanLinks = make(chan *gryffin.Scan, 10)

    // Construct the command.
    // render.js http(s)://<host>[:port][/path] [{"method":"post", "data":"a=1&b=2"}]
    url := s.Request.URL.String()
    cookies := make([]string, 0)
    // ua := s.Request.UserAgent()
    ua := "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36"

    for _, c := range s.Cookies {
        cookies = append(cookies, c.String())
    }

    arg := input{
        Method: s.Request.Method,
        Headers: inputHeaders{
            UserAgent: ua,
            Cookie:    strings.Join(cookies, ";"),
        },
    }

    opt, err := json.Marshal(arg)
    if err != nil {
        s.Error("PhantomjsRenderer.Do", err)
        return
    }

    // s.Logmf("PhantomjsRenderer.Do", "Running: render.js %s '%s'", url, string(opt))
    s.Logmf("PhantomjsRenderer.Do", "Running: render.js")

    cmd := exec.Command(
        os.Getenv("GOPATH")+"/src/github.com/yahoo/gryffin/renderer/resource/render.js",
        url,
        string(opt))

    stdout, err := cmd.StdoutPipe()
    if err != nil {
        s.Error("PhantomjsRenderer.Do", err)
        return
    }

    if err := cmd.Start(); err != nil {
        s.Error("PhantomjsRenderer.Do", err)
        return
    }

    kill := func(reason string) {
        if err := cmd.Process.Kill(); err != nil {
            // TODO - forgive "os: process already finished"
            s.Error("PhantomjsRenderer.Do", err)
            // log.Printf("error: %s", err)
        } else {
            s.Logmf("PhantomjsRenderer.Do", "[%s] Terminating the crawl process.", reason)
        }
    }
    // Kill when timeout
    _ = time.Second
    if r.Timeout != 0 {
        timeout := func() {
            <-time.After(time.Duration(r.Timeout) * time.Second)
            kill("Timeout")
        }
        go timeout()
    }

    crawl := func() {
        defer close(r.chanResponse)
        defer close(r.chanLinks)

        dec := json.NewDecoder(stdout)

        for {
            var m message
            err := dec.Decode(&m)
            if err == io.EOF {
                return
                break
            } else {
                if m.responseMessage != nil {
                    m.Response.fill(s)
                    if s.IsDuplicatedPage() {
                        kill("Duplicated")
                        return
                    }
                    s.Logm("PhantomjsRenderer.Do.UniqueCrawl", m.MsgType)
                    r.chanResponse <- s
                    for _, link := range m.Response.Details.Links {
                        if newScan := link.toScan(s); newScan != nil && newScan.IsScanAllowed() {
                            r.chanLinks <- newScan
                        }
                    }
                } else if m.domMessage != nil {
                    for _, link := range m.domMessage.Links {
                        if newScan := link.toScan(s); newScan != nil && newScan.IsScanAllowed() {
                            r.chanLinks <- newScan
                        }
                    }
//.........這裏部分代碼省略.........
開發者ID:0xwindows,項目名稱:gryffin,代碼行數:101,代碼來源:phantomjs.go


注:本文中的github.com/yahoo/gryffin.Scan.Error方法示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台,相關代碼片段篩選自各路編程大神貢獻的開源項目,源碼版權歸原作者所有,傳播和使用請參考對應項目的License;未經允許,請勿轉載。