Golang stream.New函數代碼示例

func (pipeline *Pipeline) Partition(fn stream.PredicateFn) (*Pipeline, *Pipeline) {
	lhsIn, lhsOut := stream.New(pipeline.Stream.Capacity())
	rhsIn := dispatchers.New(pipeline.Context).If(fn).Dispatch(pipeline.Stream, lhsOut)
	lhsPipeline := &Pipeline{Context: pipeline.Context, Stream: lhsIn, parallel: pipeline.parallel}
	rhsPipeline := &Pipeline{Context: pipeline.Context, Stream: rhsIn, parallel: pipeline.parallel}
	return lhsPipeline, rhsPipeline
}

func TestBatchCacheLoader(t *testing.T) {
	Convey("Given I have an empty batch of size 2", t, func() {
		batch := &appx.MemcacheLoadBatch{Size: 2}
		So(batch.Empty(), ShouldBeTrue)
		So(batch.Full(), ShouldBeFalse)

		Convey("When I add an entity to the batch", func() {
			batch.Add(NewUserWithFakeKey(User{Name: "borges"}))

			Convey("Then the batch is no longer empty", func() {
				So(batch.Empty(), ShouldBeFalse)

				Convey("And it is not yet full", func() {
					So(batch.Full(), ShouldBeFalse)
				})
			})
		})

		Convey("When I add enough entities", func() {
			batch.Add(NewUserWithFakeKey(User{Name: "borges"}))
			batch.Add(NewUserWithFakeKey(User{Name: "diego"}))

			Convey("Then the batch is full", func() {
				So(batch.Full(), ShouldBeTrue)
			})
		})

		Convey("When I commit the batch", func() {
			in, out := stream.New(1)

			entity1 := NewUserWithFakeKey(User{
				Name: "entity1",
				SSN:  "123123",
			})

			entity2 := NewUserWithFakeKey(User{
				Name: "entity2",
				SSN:  "321321",
			})

			batch.Add(entity1)
			batch.Add(entity2)
			batch.Commit(stream.NewEmitter(rivers.NewContext(), out))
			close(out)

			Convey("Then a copy of the batch is sent to the output stream", func() {
				committedBatch := (<-in).(*appx.MemcacheLoadBatch)
				So(committedBatch.Size, ShouldEqual, 2)
				So(committedBatch.Keys[0], ShouldEqual, entity1.CacheID())
				So(committedBatch.Keys[1], ShouldEqual, entity2.CacheID())
				So(committedBatch.Items[entity1.CacheID()], ShouldResemble, &appx.CachedEntity{Entity: entity1})
				So(committedBatch.Items[entity2.CacheID()], ShouldResemble, &appx.CachedEntity{Entity: entity2})

				Convey("And the batch is now empty", func() {
					So(batch.Empty(), ShouldBeTrue)
				})
			})
		})
	})
}

func TestZipperBy(t *testing.T) {
	adder := func(a, b stream.T) stream.T {
		return a.(int) + b.(int)
	}

	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in1, out1 := stream.New(2)
			out1 <- 1
			out1 <- 2
			close(out1)

			in2, out2 := stream.New(4)
			out2 <- 3
			out2 <- 4
			out2 <- 5
			out2 <- 6
			close(out2)

			Convey("When I apply the combiner to the streams", func() {
				combiner := combiners.ZipBy(adder)
				combiner.Attach(context)
				combined := combiner.Combine(in1, in2)

				Convey("Then a transformed stream is returned", func() {
					So(combined.ReadAll(), ShouldResemble, []stream.T{4, 6, 5, 6})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					combiner := combiners.ZipBy(adder)
					combiner.Attach(context)
					combined := combiner.Combine(in1, in2)

					Convey("Then no item is sent to the next stage", func() {
						So(combined.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}

func (dispatcher *dispatcher) Dispatch(in stream.Readable, writables ...stream.Writable) stream.Readable {
	notDispatchedReadable, notDispatchedWritable := stream.New(in.Capacity())

	dispatchedCount := 0
	done := make(chan bool, len(writables))

	closeWritables := func() {
		defer func() {
			for _, writable := range writables {
				close(writable)
			}
		}()

		expectedDoneMessages := dispatchedCount * len(writables)
		for i := 0; i < expectedDoneMessages; i++ {
			select {
			case <-dispatcher.context.Failure():
				return
			case <-time.After(dispatcher.context.Deadline()):
				panic(stream.Timeout)
			case <-done:
				continue
			}
		}
	}

	go func() {
		defer dispatcher.context.Recover()
		defer close(notDispatchedWritable)
		defer closeWritables()

		for data := range in {
			select {
			case <-dispatcher.context.Failure():
				return
			case <-time.After(dispatcher.context.Deadline()):
				panic(stream.Timeout)
			default:
				if dispatcher.fn(data) {
					dispatchedCount++
					for _, writable := range writables {
						// dispatch data asynchronously so that
						// slow receivers don't block the dispatch
						// process
						go func(w stream.Writable, d stream.T) {
							w <- d
							done <- true
						}(writable, data)
					}
				} else {
					notDispatchedWritable <- data
				}
			}
		}
	}()

	return notDispatchedReadable
}

func TestFifo(t *testing.T) {
	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in1, out1 := stream.New(2)
			out1 <- 1
			out1 <- 2
			close(out1)

			in2, out2 := stream.New(2)
			out2 <- 3
			out2 <- 4
			close(out2)

			Convey("When I apply the combiner to the streams", func() {
				combiner := combiners.FIFO()
				combiner.Attach(context)
				combined := combiner.Combine(in1, in2)

				Convey("Then a transformed stream is returned", func() {
					items := combined.ReadAll()
					So(items, should.Contain, 1)
					So(items, should.Contain, 2)
					So(items, should.Contain, 3)
					So(items, should.Contain, 4)
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					combiner := combiners.Zip()
					combiner.Attach(context)
					combined := combiner.Combine(in1, in2)

					Convey("Then no item is sent to the next stage", func() {
						So(combined.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}

func (combiner *zipBy) Combine(in ...stream.Readable) stream.Readable {
	max := func(rs ...stream.Readable) int {
		max := 0
		for _, r := range rs {
			capacity := r.Capacity()
			if max < capacity {
				max = capacity
			}
		}
		return max
	}

	reader, writer := stream.New(max(in...))

	go func() {
		defer combiner.context.Recover()
		defer close(writer)

		var zipped stream.T
		doneIndexes := make(map[int]bool)

		for len(doneIndexes) < len(in) {
			select {
			case <-combiner.context.Failure():
				return
			case <-time.After(combiner.context.Deadline()):
				panic(stream.Timeout)
			default:
				for i, readable := range in {
					data, opened := <-readable

					if !opened {
						if _, registered := doneIndexes[i]; !registered {
							doneIndexes[i] = true
						}
						continue
					}

					if zipped == nil {
						zipped = data
					} else {
						zipped = combiner.fn(zipped, data)
					}
				}

				if zipped != nil {
					writer <- zipped
					zipped = nil
				}
			}
		}
	}()

	return reader
}

func TestEach(t *testing.T) {
	collect := func(items *[]stream.T) stream.EachFn {
		return func(data stream.T) {
			*items = append(*items, data)
		}
	}

	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(2)
			out <- 1
			out <- 2
			close(out)

			Convey("When I apply the transformer to the stream", func() {
				var items []stream.T
				transformer := transformers.Each(collect(&items))
				transformer.Attach(context)
				next := transformer.Transform(in)

				Convey("Then all items are sent to the next stage", func() {
					So(next.ReadAll(), ShouldResemble, []stream.T{1, 2})

					Convey("And all items are transformed", func() {
						So(items, ShouldResemble, []stream.T{1, 2})
					})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					var items []stream.T
					transformer := transformers.Each(collect(&items))
					transformer.Attach(context)
					next := transformer.Transform(in)

					Convey("Then no item is sent to the next stage", func() {
						So(next.ReadAll(), ShouldBeEmpty)

						Convey("And no item is transformed", func() {
							So(items, ShouldBeEmpty)
						})
					})
				})
			})
		})
	})
}

func (pipeline *Pipeline) SplitN(n int) []*Pipeline {
	pipelines := make([]*Pipeline, n)
	writables := make([]stream.Writable, n)
	for i := 0; i < n; i++ {
		readable, writable := stream.New(pipeline.Stream.Capacity())
		writables[i] = writable
		pipelines[i] = &Pipeline{
			Context:  pipeline.Context,
			Stream:   readable,
			parallel: pipeline.parallel,
		}
	}
	dispatchers.New(pipeline.Context).Always().Dispatch(pipeline.Stream, writables...)
	return pipelines
}

func TestBatcher(t *testing.T) {
	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(3)
			out <- 1
			out <- 2
			out <- 3
			close(out)

			Convey("When I apply the batch transformer to the stream", func() {
				transformer := transformers.Batch(2)
				transformer.Attach(context)
				next := transformer.Transform(in)

				Convey("Then a transformed stream is returned", func() {
					So(next.ReadAll(), ShouldResemble, []stream.T{[]stream.T{1, 2}, []stream.T{3}})
				})
			})

			Convey("When I apply the batch by transformer to the stream", func() {
				transformer := transformers.BatchBy(&batch{size: 1})
				transformer.Attach(context)
				next := transformer.Transform(in)

				Convey("Then a transformed stream is returned", func() {
					So(next.ReadAll(), ShouldResemble, []stream.T{[]stream.T{1}, []stream.T{2}, []stream.T{3}})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					transformer := transformers.Flatten()
					transformer.Attach(context)
					next := transformer.Transform(in)

					Convey("Then no item is sent to the next stage", func() {
						So(next.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}

func (observable *Observable) Produce() stream.Readable {
	if observable.Capacity <= 0 {
		observable.Capacity = 10
	}
	readable, writable := stream.New(observable.Capacity)

	go func() {
		defer observable.context.Recover()
		defer close(writable)

		if observable.Emit != nil {
			observable.Emit(stream.NewEmitter(observable.context, writable))
		}
	}()

	return readable
}

func (observer *Observer) Transform(in stream.Readable) stream.Readable {
	readable, writable := stream.New(in.Capacity())
	emitter := stream.NewEmitter(observer.context, writable)

	go func() {
		defer observer.context.Recover()
		defer close(writable)

		for {
			select {
			case <-observer.context.Failure():
				return
			case <-observer.context.Done():
				return
			case <-time.After(observer.context.Deadline()):
				panic(stream.Timeout)
			default:
				data, more := <-in
				if !more {
					if observer.OnCompleted != nil {
						observer.OnCompleted(emitter)
					}
					return
				}

				if observer.OnNext == nil {
					continue
				}

				if err := observer.OnNext(data, emitter); err != nil {
					if err == stream.Done {
						// Tell producer to shutdown without errors
						observer.context.Close(nil)
						return
					}
					panic(err)
				}
			}
		}
	}()

	return readable
}

func TestProcessor(t *testing.T) {
	evensFilter := func(d stream.T, emitter stream.Emitter) {
		if d.(int)%2 == 0 {
			emitter.Emit(d)
		}
	}

	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(2)
			out <- 1
			out <- 2
			close(out)

			Convey("When I apply the transformer to the stream", func() {
				transformer := transformers.OnData(evensFilter)
				transformer.Attach(context)
				transformed := transformer.Transform(in)

				Convey("Then a transformed stream is returned", func() {
					So(transformed.ReadAll(), ShouldResemble, []stream.T{2})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					transformer := transformers.OnData(evensFilter)
					transformer.Attach(context)
					next := transformer.Transform(in)

					Convey("Then no item is sent to the next stage", func() {
						So(next.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}

func (combiner *zip) Combine(in ...stream.Readable) stream.Readable {
	capacity := func(rs ...stream.Readable) int {
		capacity := 0
		for _, r := range rs {
			capacity += r.Capacity()
		}
		return capacity
	}

	reader, writer := stream.New(capacity(in...))

	go func() {
		defer combiner.context.Recover()
		defer close(writer)

		for {
			select {
			case <-combiner.context.Failure():
				return
			case <-time.After(combiner.context.Deadline()):
				panic(stream.Timeout)
			default:
				doneCount := 0
				for _, readable := range in {
					data, more := <-readable
					if !more {
						doneCount++
						continue
					}
					writer <- data
				}

				if doneCount == len(in) {
					return
				}
			}
		}
	}()

	return reader
}

func TestFindBy(t *testing.T) {
	evens := func(d stream.T) bool { return d.(int)%2 == 0 }

	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(3)
			out <- 1
			out <- 2
			out <- 4
			close(out)

			Convey("When I apply the transformer to the stream", func() {
				transformer := transformers.FindBy(evens)
				transformer.Attach(context)
				next := transformer.Transform(in)

				Convey("Then a transformed stream is returned", func() {
					So(next.ReadAll(), ShouldResemble, []stream.T{2})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					transformer := transformers.FindBy(evens)
					transformer.Attach(context)
					next := transformer.Transform(in)

					Convey("Then no item is sent to the next stage", func() {
						So(next.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}

func TestReducer(t *testing.T) {
	sum := func(acc, next stream.T) stream.T { return acc.(int) + next.(int) }

	Convey("Given I have a context", t, func() {
		context := rivers.NewContext()

		Convey("And a stream of data", func() {
			in, out := stream.New(3)
			out <- 1
			out <- 2
			out <- 3
			close(out)

			Convey("When I apply a mapper transformer to the stream", func() {
				transformer := transformers.Reduce(0, sum)
				transformer.Attach(context)
				next := transformer.Transform(in)

				Convey("Then a transformed stream is returned", func() {
					So(next.ReadAll(), ShouldResemble, []stream.T{6})
				})
			})

			Convey("When I close the context", func() {
				context.Close(stream.Done)

				Convey("And I apply the transformer to the stream", func() {
					transformer := transformers.Reduce(0, sum)
					transformer.Attach(context)
					next := transformer.Transform(in)

					Convey("Then no item is sent to the next stage", func() {
						So(next.ReadAll(), ShouldBeEmpty)
					})
				})
			})
		})
	})
}