本文整理汇总了Golang中github.com/apoydence/hydra/examples/wordCount/types.SetupFunction.Cancelled方法的典型用法代码示例。如果您正苦于以下问题：Golang SetupFunction.Cancelled方法的具体用法？Golang SetupFunction.Cancelled怎么用？Golang SetupFunction.Cancelled使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类github.com/apoydence/hydra/examples/wordCount/types.SetupFunction的用法示例。
在下文中一共展示了SetupFunction.Cancelled方法的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Golang代码示例。
示例1: UrlParser
func UrlParser(sf types.SetupFunction) {
in, out := sf.SetName("UrlParser").AsFilter("UrlProducer").Build()
defer close(out)
for urlBM := range in {
if sf.Cancelled() {
continue
}
urlStr := ToString(urlBM)
if path.Ext(urlStr) == "" && urlStr[len(urlStr)-1] != '/' {
urlStr += "/"
}
url, err := urlParser.Parse(urlStr)
if err != nil {
continue
}
body := Download(url.String())
for link := range fetchLinks(body) {
joinedLink, err := url.Parse(link)
if err != nil {
continue
}
out <- NewStringMarshaler(joinedLink.String())
}
}
}示例2: MimeDetector
func MimeDetector(sf types.SetupFunction) {
in, out := sf.SetName("MimeDetector").AsFilter("UrlParser").Build()
defer close(out)
buffer := make([]byte, 512)
for urlBM := range in {
if sf.Cancelled() {
continue
}
url := ToString(urlBM)
body := Download(url)
if body == nil {
continue
}
func(body io.ReadCloser) {
defer body.Close()
n, err := body.Read(buffer)
if err == nil || err == io.EOF {
encoded := http.DetectContentType(buffer[:n]) + "->" + url
out <- NewStringMarshaler(encoded)
}
}(body)
}
}示例3: MimeSplitterHtml
func MimeSplitterHtml(sf types.SetupFunction) {
in, out := sf.SetName("MimeSplitterHtml").AsFilter("MimeDetector").Build()
defer close(out)
for urlBM := range in {
if sf.Cancelled() {
continue
}
url := ToString(urlBM)
mime, u := decodeMimeUrl(url)
if strings.Contains(mime, "html") {
out <- NewStringMarshaler(u)
}
}
}示例4: UrlProducer
func UrlProducer(sf types.SetupFunction, parent string) {
in, out := sf.SetName("UrlProducer").AsFilter(parent).Build()
defer close(out)
visitedUrls := make(map[string]interface{})
for urlBm := range in {
if sf.Cancelled() {
continue
}
url := ToString(urlBm)
if _, visited := visitedUrls[url]; !visited {
visitedUrls[url] = nil
out <- NewStringMarshaler(url)
}
}
}