本文整理匯總了Golang中github.com/henrylee2cn/pholcus/crawl/downloader/context.Request.GetUrl方法的典型用法代碼示例。如果您正苦於以下問題：Golang Request.GetUrl方法的具體用法？Golang Request.GetUrl怎麽用？Golang Request.GetUrl使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類github.com/henrylee2cn/pholcus/crawl/downloader/context.Request的用法示例。
在下文中一共展示了Request.GetUrl方法的3個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Golang代碼示例。
示例1: Download
func (self *Surfer) Download(cReq *context.Request) *context.Response {
cResp := context.NewResponse(nil)
resp, err := self.download.Download(cReq.GetMethod(), cReq.GetUrl(), cReq.GetReferer(), cReq.GetPostData(), cReq.GetHeader(), cReq.GetCookies())
cResp.SetRequest(cReq)
cResp.SetResponse(resp)
if err != nil {
cResp.SetStatus(true, err.Error())
return cResp
}
cResp.SetStatus(false, "")
return cResp
}示例2: Download
func (self *Surfer) Download(cReq *context.Request) *context.Response {
cResp := context.NewResponse(nil)
resp, err := self.download.Download(cReq.GetMethod(), cReq.GetUrl(), cReq.GetReferer(), cReq.GetPostData(), cReq.GetHeader(), cReq.GetCookies())
cResp.SetRequest(cReq)
if err != nil {
cResp.SetStatus(true, err.Error())
return cResp
}
// get converter to utf-8
body := self.changeCharsetEncodingAuto(resp.Body, resp.Header.Get("Content-Type"))
//fmt.Printf("utf-8 body %v \r\n", bodyStr)
defer resp.Body.Close()
cResp.SetText(body)
cResp.SetStatus(false, "")
return cResp
}示例3: Push
// 添加請求到隊列
func (self *scheduler) Push(req *context.Request) {
pushMutex.Lock()
defer func() {
pushMutex.Unlock()
}()
if self.status == status.STOP {
return
}
is := self.Compare(req.GetUrl() + req.GetMethod())
// 有重複則返回
if is {
return
}
// 留作未來分發請求用
// if pholcus.Self.GetRunMode() == config.SERVER || req.CanOutsource() {
// return
// }
self.SrcManage.Push(req)
}