Python Stream.Stream类代码示例

def run_templater(inputFile, outputDir, tmpF):
	templateFile = open(tmpF).read() 
	env = jinja2.Environment()
	env.globals.update(sorted=sorted, to_snake=to_snake_case)

	template = env.from_string(templateFile)


	s = Stream(inputFile, 'Letter')
	
	# Dirty counter for well-formedness
	wf = 0
	bf = 0


	for key, item in s.stream():
		templatedText = template.render(item)
		f = open(outputDir+key+".xml", 'w')
		f.write(templatedText)
		f.close()

		# A dirty checker for wellformedness
		try:
			ET.fromstring(templatedText)
			wf += 1
		except Exception as e:
			print(key, " is BAD:: ", e)
			bf += 1

	print('GOOD: ', wf)
	print('BAD: ', bf)

def main():
    def max_of_std(lst, state):
        a = np.array(lst)
        state = max(a.std(), state)
        return (state, state)
    print "example_1"
    print "example function from list to value: mean_and_sigma() "
    window_size = 10
    step_size = 10
    print "window_size = ", window_size
    print "step_size = ", step_size
    print ""

    x = Stream('x')
    # x is the in_stream.
    # sum() is the function on the window

    z = stream_func(x, f_type='window', f=max_of_std, num_outputs=1, state=0.0,
                    window_size=window_size, step_size=step_size)

    z.set_name('z')
 

    x.extend([random.random() for i in range(30)])
    x.print_recent()
    z.print_recent()

	def getValuesFromFile(self):
 		stream = Stream(self.filePath, self.column, sheet="ID")
 		
 		for k, v in stream.stream():
 			
 			#print(k)

 			if k != 'None' and v["DATE"] < self.date:
	 			yield str(k) + '.0'

def main():

    def mean_of_window(stream):

        def mean(lst, state):
            sum_window, next_value_dropped = state
            if sum_window is None:
                sum_window = sum(lst)
            else:
                sum_window = sum_window + lst[-1] - next_value_dropped
            mean = sum_window/len(lst)
            next_value_dropped = lst[0]
            state = (sum_window, next_value_dropped)
            return (mean, state)

        return stream_func(
            inputs=stream,
            f_type='window',
            f=mean,
            num_outputs=1,
            state=(None, None),
            window_size=2,
            step_size=2)


    def max_of_std(lst, state):
        a = np.array(lst).std()
        if a > state:
            state = a
            return (a, state)
        else:
            return (_no_value, state)
        

    x = Stream('x')
    # x is the input stream.


    g = partial(stream_func, f_type='window',
                f=max_of_std,
                num_outputs=1, state=0.0,
                window_size=10, step_size=10)

    y = g(x)
    z = g(y)
    means = mean_of_window(x)

    y.set_name('y')
    z.set_name('z')
    means.set_name('means')

    x.extend([random.random() for i in range(30)])
    x.print_recent()
    y.print_recent()
    z.print_recent()
    means.print_recent()

 def create_stream(self, position):
     stream = Stream(color=randint(0, 3))
     self.streams.append(stream)
     self.add_widget(stream)
     self.bind(on_ignore_touch=stream.on_ignore_touch)
     self.bind(on_acknowledge_touch=stream.on_acknowledge_touch)
     self.bind(on_mouse_state_changed=stream.on_mouse_state_changed)
     stream.head_position = (position[0], position[1])
     stream.active = True
     stream.ignore_touch = False

def main():
    # Functions: list -> list
    def square(lst):
        return [v*v for v in lst]

    def double(lst):
        return [2*v for v in lst]

    def even(lst):
        return [v for v in lst if not v%2]

    # Functions: stream -> stream.
    # Each element of the output stream is f() applied to the corresponding
    # element of the input stream.
    stream_square = partial(stream_func, f_type='list', f=square, num_outputs=1)
    stream_double = partial(stream_func, f_type='list', f=double, num_outputs=1)
    stream_even = partial(stream_func, f_type='list', f=even, num_outputs=1)
    # Create stream x, and give it name 'x'.
    x = Stream('input')

    # u is the stream returned by stream_square(x)  and
    # v is the stream returned by stream_double(x)
    # w is the stream returned by stream_square(v) and
    #   so w could have been defined as:
    #   stream_square(stream_double(x))
    # a is the stream containing only even values of x
    u = stream_square(x)
    v = stream_double(x)
    w = stream_square(v)
    a = stream_even(x)

    # Give names to streams u, v, and w. This is helpful in reading output.
    u.set_name('square of input')
    v.set_name('double of input')
    w.set_name('square of double of input')
    a.set_name('even values in input')

    print
    print 'add [3, 5] to the tail of the input stream'
    # Add values to the tail of stream x.
    x.extend([3, 5])

    # Print the N most recent values of streams x, u, v, w.
    x.print_recent()
    u.print_recent()
    v.print_recent()
    w.print_recent()
    a.print_recent()    

    print
    print 'add [2, 6] to the tail of the input stream'
    # Add more values to the tail of stream x.
    x.extend([2, 6])

    # Print the N most recent values of streams x, u, v, w.
    x.print_recent()
    u.print_recent()
    v.print_recent()
    w.print_recent()
    a.print_recent()

def test():

    x = Stream('x')

    y,z = exp_smooth_max_and_std_of_windows_in_stream(x)

    y.set_name('y')
    z.set_name('z')

    check(y, [1.6000000000000001, 4.3200000000000003])
    check(z, [0.65319726474218087, 0.78383671769061702])
    
    x.extend([1, 2, 3, 4, 5])
    x.print_recent()
    y.print_recent()
    z.print_recent()
    print

    x.extend([6, 12, 13, 14, 15])
    x.print_recent()
    y.print_recent()
    z.print_recent()
    print

    check_empty()

def test():

    x = Stream('x')

    y,z = max_min_of_windows_in_stream(x)

    y.set_name('y')
    z.set_name('z')

    check(y, [5])
    check(z, [3])
    
    x.extend([3,5])
    x.print_recent()
    y.print_recent()
    z.print_recent()
    print

    x.extend([11,15])
    x.print_recent()
    y.print_recent()
    z.print_recent()
    print

    check_empty()

def main():
    # Functions: list -> list of lists
    def even_odd(list_of_integers):
        evens_list = [n for n in list_of_integers if not n%2]
        odds_list = [n for n in list_of_integers if n%2]
        return (evens_list, odds_list)

    # Functions: stream -> stream.
    # The n-th element of the output stream is f() applied to the n-th
    # elements of each of the input streams.
    # Function mean is defined above, and functions sum and max are the
    # standard Python functions.
    stream_even_odd = partial(stream_func, f_type='list', f=even_odd, num_outputs=2)

    # Create stream x, and give it name 'x'.
    x = Stream('input_0')

    # u is the stream returned by stream_sum([x,y])  and
    # v is the stream returned by stream_max([x,y])
    # w is the stream returned by stream_mean([x,y]).
    # u[i] = sum(x[i],y[i])
    # v[i] = max(x[i],y[i])
    # w[i] = mean(x[i],y[i])    
    evens, odds = stream_even_odd(x)

    # Give names to streams u, v, and w. This is helpful in reading output.
    evens.set_name('even numbers in x')
    odds.set_name('odd numbers in x')

    print
    print 'Adding [3, 5, 8] to input stream'
    # Add values to the tail of stream x.
    x.extend([3, 5, 8])

    # Print recent values of the streams
    print 'recent values of input streams'
    x.print_recent()

    print 'recent values of output streams'
    evens.print_recent()
    odds.print_recent()


    print
    print 'Adding [4, 6, 2, 1] to the input stream'
    # Add more values to the tail of stream x.
    x.extend([4, 6, 2, 1])

    # Print recent values of the streams
    print 'recent values of input streams'
    x.print_recent()

    print 'recent values of output streams'
    evens.print_recent()
    odds.print_recent()

class Filter:
	def __init__(self, inputFilePath, outputFilePath):
		self.inclusionList = []
		self.exclusionList = []
		self.stream = Stream(inputFilePath, 'Letter')
		self.new_shelve_file = outputFilePath

	def inclusionListAdd(self, filterList):
		self.inclusionList += filterList()

	def exclusionListAdd(self, filterList):
		self.exclusionList += filterList()


	def filter(self):
		with ShelveManager(self.new_shelve_file) as new_shelf:
			for index, fields in self.stream.stream():
				#print(str(index))
				if str(index)+'.0'  in self.inclusionList and str(index)+'.0' not in self.exclusionList:
				
					new_shelf[index] = self._get_fields(fields)

	@editLogger('Filtered from Completed List', 'PythonScript')
	def _get_fields(self, fields):
		return fields

    def start(self, users):
        #users is a list of dictionary type objects
        # dict type objects from getConfig()
        print "Starting..."
        #start Twitter stream
        streamThreads  = []
        streamBuffers  = []
        
        for user in users:
            sr = Stream(user['con_key'], user['con_secret'], 
                        user['key'], user['secret'], 
                        user['name'])
            
            #insert user list into db
            list = sr.getUserList(self.parseLists(user['lists']))
            self.sql.insert_into_userList(list, user['db'])
            
            track = []
            for keyword in user['track'].split(','):
                track.append(keyword)
            
            #get buffer and run stream
            buff = sr.getTweetsBuffer()
# 	    if list is not None or track is not None:
#                 stream = sr.run(list, track)
# 	    else:
# 		  stream = sr.run(None)
            stream = sr.run(list, track)
            #add new buff and stream to list
            streamBuffers.append({'buff':buff, 'db':user['db']})
            streamThreads.append(stream)
            print "Started user: " + user['name']
            self.logger.info("Started user: " + user['name'])

        while True:
            try:
                for buffer in streamBuffers:
                    tweet = buffer['buff'].pop()
                    if not tweet:
                        time.sleep(1)
                    else:
                        self.sql.insert_into(buffer['db'], tweet)
            except KeyboardInterrupt:
                print "Exiting..."
                os._exit(0)

def main():
    b, a = butter_bandpass(lowcut=0.1, highcut=5.0, fs=50, order=5)
    y = np.zeros(len(a)-1)
    state = (b, a, y)
    filename = '20110111000000.2D.OBS34.SXZ.npy'
    data_array = np.load(filename)
    print 'len(data_array)', len(data_array)
    data_array = data_array[:1000000]
    x = Stream('x')
    print 'a', a
    print 'b', b
    y = stream_func(x, f_type='window', f=linear_filter, num_outputs=1,
                    state=state, window_size=len(b), step_size=1)
    x.extend(data_array)
    y.set_name('y')
    plt.plot(y.recent[5000:y.stop])
    plt.show()
    plt.close()

def test():

    # Create stream x, and give it name 'x'.
    x = Stream('input_0')

    # u is the stream returned by stream_sum([x,y])  and
    # v is the stream returned by stream_max([x,y])
    # w is the stream returned by stream_mean([x,y]).
    # u[i] = sum(x[i],y[i])
    # v[i] = max(x[i],y[i])
    # w[i] = mean(x[i],y[i])    
    evens, odds = stream_even_odd(x)

    # Give names to streams u, v, and w. This is helpful in reading output.
    evens.set_name('even numbers in x')
    odds.set_name('odd numbers in x')

    check(evens, [8, 4, 6, 2])
    check(odds, [3,5])

    print
    print 'Adding [3, 5, 8], [1, 7, 2], [2, 3] to 3 input streams'
    # Add values to the tail of stream x.
    x.extend([3, 5, 8])

    # Print recent values of the streams
    print 'recent values of input streams'
    x.print_recent()

    print 'recent values of output streams'
    evens.print_recent()
    odds.print_recent()


    print
    print 'Adding [4, 6, 2], [2, 3, 8], [5, 3, 0, -1] to 3 input streams'
    # Add more values to the tail of stream x.
    x.extend([4, 6, 2])

    # Print recent values of the streams
    print 'recent values of input streams'
    x.print_recent()

    print 'recent values of output streams'
    evens.print_recent()
    odds.print_recent()

    check_empty()

 def get(self):
     current_time = datetime.datetime.now()
     all_streams = Stream.query()
     for stream in all_streams:
         for date in stream.view_queue:
             minback = current_time - datetime.timedelta(hours=1)
             if date < minback:
                 stream.view_queue.remove(date)
                 stream.views = stream.views - 1
         stream.put()

def test():

    # Create stream x, and give it name 'x'.
    x = Stream('input_0')

    id_0_average, id_1_average = stream_split_by_sensor_id(x)

    # Give names to streams u, v, and w. This is helpful in reading output.
    id_0_average.set_name('average of id_0 sensors in x')
    id_1_average.set_name('average of id_1 sensors in x')

    check(id_0_average, [2.0, 3.0, 5.0, 4.0, 4.0])
    check(id_1_average, [5.0, 3.0, 3.0, 4.0, 5.0, 6.0])

    print
    print 'Adding ([(0,2), (0,4), (1,5), (1,1), (0,9)]'
    print 'to the input stream.'
    # Add values to the tail of stream x.
    x.extend([(0,2), (0,4), (1,5), (1,1), (0,9)])

    # Print recent values of the streams
    print
    print 'recent values of input streams'
    x.print_recent()

    print
    print 'recent values of output streams'
    id_0_average.print_recent()
    id_1_average.print_recent()

    print
    print
    print 'Adding ([(1,3), (1,7), (0,1), (1,9), (1,11), (0,4)])'
    print 'to the input stream.'
    # Add values to the tail of stream x.
    x.extend([(1,3), (1,7), (0,1), (1,9), (1,11), (0,4)])

    # Print recent values of the streams
    print 'recent values of input streams'
    print
    x.print_recent()

    print 'recent values of output streams'
    print
    id_0_average.print_recent()
    id_1_average.print_recent()

    check_empty()