当前位置: 首页>>代码示例>>Golang>>正文


Golang types.SetupFunction类代码示例

本文整理汇总了Golang中github.com/apoydence/hydra/examples/wordCount/types.SetupFunction的典型用法代码示例。如果您正苦于以下问题:Golang SetupFunction类的具体用法?Golang SetupFunction怎么用?Golang SetupFunction使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。


在下文中一共展示了SetupFunction类的14个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Golang代码示例。

示例1: MimeDetector

func MimeDetector(sf types.SetupFunction) {
	in, out := sf.SetName("MimeDetector").AsFilter("UrlParser").Build()
	defer close(out)

	buffer := make([]byte, 512)

	for urlBM := range in {
		if sf.Cancelled() {
			continue
		}
		url := ToString(urlBM)
		body := Download(url)
		if body == nil {
			continue
		}
		func(body io.ReadCloser) {
			defer body.Close()
			n, err := body.Read(buffer)

			if err == nil || err == io.EOF {
				encoded := http.DetectContentType(buffer[:n]) + "->" + url
				out <- NewStringMarshaler(encoded)
			}
		}(body)
	}
}
开发者ID:jmptrader,项目名称:hydra,代码行数:26,代码来源:webCrawler.go

示例2: UrlParser

func UrlParser(sf types.SetupFunction) {
	in, out := sf.SetName("UrlParser").AsFilter("UrlProducer").Build()
	defer close(out)

	for urlBM := range in {
		if sf.Cancelled() {
			continue
		}
		urlStr := ToString(urlBM)
		if path.Ext(urlStr) == "" && urlStr[len(urlStr)-1] != '/' {
			urlStr += "/"
		}
		url, err := urlParser.Parse(urlStr)
		if err != nil {
			continue
		}

		body := Download(url.String())
		for link := range fetchLinks(body) {
			joinedLink, err := url.Parse(link)
			if err != nil {
				continue
			}

			out <- NewStringMarshaler(joinedLink.String())
		}
	}
}
开发者ID:jmptrader,项目名称:hydra,代码行数:28,代码来源:webCrawler.go

示例3: pathProducer

func pathProducer(sf types.SetupFunction, argv []string) {
	out := sf.SetName("PathProducer").AsProducer().Build()
	defer close(out)

	for _, path := range argv {
		out <- NewStringMarshaler(path)
	}
}
开发者ID:jmptrader,项目名称:hydra,代码行数:8,代码来源:fileContentCounter.go

示例4: PathValidator

func PathValidator(sf types.SetupFunction, parent string) {
	in, out := sf.SetName("PathValidator").AsFilter(parent).Build()
	defer close(out)

	for path := range in {
		if _, err := os.Stat(ToString(path)); err == nil {
			out <- path
		}
	}
}
开发者ID:jmptrader,项目名称:hydra,代码行数:10,代码来源:wordCount.go

示例5: urlLooper

func urlLooper(sf types.SetupFunction, feeder chan<- interface{}, done types.AtomicBool) {
	in := sf.AsConsumer("MimeSplitterHtml").Build()

	for url := range in {
		if !done.Get() {
			u := ToString(url)
			feeder <- u
		}
	}
}
开发者ID:jmptrader,项目名称:hydra,代码行数:10,代码来源:textDownloaderMain.go

示例6: urlFeeder

func urlFeeder(sf types.SetupFunction, feeder <-chan interface{}, done types.AtomicBool) {
	out := sf.SetName("UrlFeeder").AsProducer().Build()
	defer close(out)

	for url := range feeder {
		if !done.Get() {
			out <- NewStringMarshaler(url.(string))
		}
	}
}
开发者ID:jmptrader,项目名称:hydra,代码行数:10,代码来源:textDownloaderMain.go

示例7: wordPrinter

func wordPrinter(sf types.SetupFunction, done chan struct{}) {
	defer close(done)
	in := sf.AsConsumer("github.com/apoydence/hydra/examples/wordCount.FinalWordCounter").Build()

	for wordMap := range in {
		for k, v := range ToMap(wordMap) {
			println(k, v)
		}
	}
}
开发者ID:jmptrader,项目名称:hydra,代码行数:10,代码来源:fileContentCounter.go

示例8: WordCounter

func WordCounter(sf types.SetupFunction) {
	in, out := sf.AsFilter("github.com/apoydence/hydra/examples/wordCount.SymbolRemover").Build()
	defer close(out)
	m := make(map[string]uint32)

	for word := range in {
		incMap(ToString(word), 1, m)
	}

	out <- NewWordCountMarshaler(m)
}
开发者ID:jmptrader,项目名称:hydra,代码行数:11,代码来源:wordCount.go

示例9: FinalWordCounter

func FinalWordCounter(sf types.SetupFunction) {
	in, out := sf.AsFilter("github.com/apoydence/hydra/examples/wordCount.WordCounter").Build()
	defer close(out)

	m := make(map[string]uint32)

	for wordMap := range in {
		for k, v := range ToMap(wordMap) {
			incMap(k, v, m)
		}
	}

	out <- NewWordCountMarshaler(m)
}
开发者ID:jmptrader,项目名称:hydra,代码行数:14,代码来源:wordCount.go

示例10: MimeSplitterHtml

func MimeSplitterHtml(sf types.SetupFunction) {
	in, out := sf.SetName("MimeSplitterHtml").AsFilter("MimeDetector").Build()
	defer close(out)
	for urlBM := range in {
		if sf.Cancelled() {
			continue
		}
		url := ToString(urlBM)
		mime, u := decodeMimeUrl(url)
		if strings.Contains(mime, "html") {
			out <- NewStringMarshaler(u)
		}
	}
}
开发者ID:jmptrader,项目名称:hydra,代码行数:14,代码来源:webCrawler.go

示例11: SymbolRemover

func SymbolRemover(sf types.SetupFunction) {
	in, out := sf.AsFilter("github.com/apoydence/hydra/examples/wordCount.WordExtractor").Build()
	defer close(out)

	for word := range in {
		str := make([]byte, 0)
		bytes := []byte(strings.ToLower(ToString(word)))
		for _, x := range bytes {
			if (x >= 0x30 && x <= 0x39) || (x >= 0x61 && x <= 0x7a) {
				str = append(str, x)
			}
		}
		if len(str) > 0 {
			out <- NewStringMarshaler(string(str))
		}
	}
}
开发者ID:jmptrader,项目名称:hydra,代码行数:17,代码来源:wordCount.go

示例12: WordExtractor

func WordExtractor(sf types.SetupFunction) {
	in, out := sf.AsFilter("PathValidator").Build()
	defer close(out)

	for path := range in {
		p := ToString(path)
		f, err := os.Open(p)
		if err == nil {
			scanner := bufio.NewScanner(f)
			scanner.Split(bufio.ScanWords)
			for scanner.Scan() {
				out <- NewStringMarshaler(scanner.Text())
			}
		}

	}
}
开发者ID:jmptrader,项目名称:hydra,代码行数:17,代码来源:wordCount.go

示例13: UrlProducer

func UrlProducer(sf types.SetupFunction, parent string) {
	in, out := sf.SetName("UrlProducer").AsFilter(parent).Build()
	defer close(out)

	visitedUrls := make(map[string]interface{})

	for urlBm := range in {
		if sf.Cancelled() {
			continue
		}
		url := ToString(urlBm)
		if _, visited := visitedUrls[url]; !visited {
			visitedUrls[url] = nil
			out <- NewStringMarshaler(url)
		}
	}
}
开发者ID:jmptrader,项目名称:hydra,代码行数:17,代码来源:webCrawler.go

示例14: textDownloader

func textDownloader(sf types.SetupFunction, download string, mb int, done chan struct{}, closer func()) {
	in := sf.AsConsumer("MimeSplitterText").Build()

	var totalSize int64 = 0

	for url := range in {
		if totalSize >= int64(mb)*1024*1024 {
			closer()
			continue
		}

		u := ToString(url)
		println("download", u, path.Base(u))
		totalSize += saveToFile(Download(u), path.Join(download, path.Base(u)))
		println("Size", totalSize)
	}

	close(done)
}
开发者ID:jmptrader,项目名称:hydra,代码行数:19,代码来源:textDownloaderMain.go


注:本文中的github.com/apoydence/hydra/examples/wordCount/types.SetupFunction类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。