本文整理匯總了Golang中github.com/apoydence/hydra/examples/wordCount/types.SetupFunction.Cancelled方法的典型用法代碼示例。如果您正苦於以下問題：Golang SetupFunction.Cancelled方法的具體用法？Golang SetupFunction.Cancelled怎麽用？Golang SetupFunction.Cancelled使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類github.com/apoydence/hydra/examples/wordCount/types.SetupFunction的用法示例。
在下文中一共展示了SetupFunction.Cancelled方法的4個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Golang代碼示例。
示例1: UrlParser
func UrlParser(sf types.SetupFunction) {
in, out := sf.SetName("UrlParser").AsFilter("UrlProducer").Build()
defer close(out)
for urlBM := range in {
if sf.Cancelled() {
continue
}
urlStr := ToString(urlBM)
if path.Ext(urlStr) == "" && urlStr[len(urlStr)-1] != '/' {
urlStr += "/"
}
url, err := urlParser.Parse(urlStr)
if err != nil {
continue
}
body := Download(url.String())
for link := range fetchLinks(body) {
joinedLink, err := url.Parse(link)
if err != nil {
continue
}
out <- NewStringMarshaler(joinedLink.String())
}
}
}示例2: MimeDetector
func MimeDetector(sf types.SetupFunction) {
in, out := sf.SetName("MimeDetector").AsFilter("UrlParser").Build()
defer close(out)
buffer := make([]byte, 512)
for urlBM := range in {
if sf.Cancelled() {
continue
}
url := ToString(urlBM)
body := Download(url)
if body == nil {
continue
}
func(body io.ReadCloser) {
defer body.Close()
n, err := body.Read(buffer)
if err == nil || err == io.EOF {
encoded := http.DetectContentType(buffer[:n]) + "->" + url
out <- NewStringMarshaler(encoded)
}
}(body)
}
}示例3: MimeSplitterHtml
func MimeSplitterHtml(sf types.SetupFunction) {
in, out := sf.SetName("MimeSplitterHtml").AsFilter("MimeDetector").Build()
defer close(out)
for urlBM := range in {
if sf.Cancelled() {
continue
}
url := ToString(urlBM)
mime, u := decodeMimeUrl(url)
if strings.Contains(mime, "html") {
out <- NewStringMarshaler(u)
}
}
}示例4: UrlProducer
func UrlProducer(sf types.SetupFunction, parent string) {
in, out := sf.SetName("UrlProducer").AsFilter(parent).Build()
defer close(out)
visitedUrls := make(map[string]interface{})
for urlBm := range in {
if sf.Cancelled() {
continue
}
url := ToString(urlBm)
if _, visited := visitedUrls[url]; !visited {
visitedUrls[url] = nil
out <- NewStringMarshaler(url)
}
}
}