本文整理汇总了Python中stream.Stream.close方法的典型用法代码示例。如果您正苦于以下问题:Python Stream.close方法的具体用法?Python Stream.close怎么用?Python Stream.close使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类stream.Stream
的用法示例。
在下文中一共展示了Stream.close方法的12个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: _on_accept
# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
def _on_accept(self):
_logger.debug('_on_accept')
while True:
try:
sock, addr = self._fd.accept()
_logger.debug('fd: %d accept fd: %d',
self._fd.fileno(), sock.fileno())
except socket.error as msg:
if msg.errno == errno.ECONNABORTED:
continue
if msg.errno != errno.EAGAIN and msg.errno != errno.EINPROGRESS:
_logger.error('fd: %d, accept: %s',
self._fd.fileno(), os.strerror(msg.errno))
self._fd.close()
if self._onClosed is not None:
try:
self._onClosed(self)
except Exception as ex:
_logger.error('_onClosed: %s', str(ex))
_logger.exception(traceback.format_exc())
return
else:
new_stream = Stream(sock, prefix=self._prefix)
new_stream._connected = True
try:
self._onAccepted(new_stream, addr)
except Exception as e:
_logger.error('_onAccepted: %s', e)
_logger.exception(traceback.format_exc())
new_stream.close()
示例2: Select
# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Select(MiniEngine):
def __init__(self, input, transformer):
MiniEngine.__init__(self)
self._input = input
self._input_ep = input.connect()
self._t = transformer
# make sure the transformer can handle the records in the stream
assert self._t.accepts(input.schema())
# We cannot reliably determine the sort order of the output stream
# as the transformation applied to the attributes is unkown. If
# for example the transformer inverts the value of an attribute the
# attribute still has the same time and possibly name, but the
# values in fact should now be sorted in reverse order.
output_order = SortOrder()
# Construct the output stream.
self._output_stream = Stream(
self._t.schema(),
output_order,
'SELECT'
)
def output(self):
return self._output_stream
def run(self):
closed = False
while not closed:
try:
# print 'SELECT: waiting to receive'
r = self._input_ep.receive()
if type(r) is StopWord:
# print 'SELECT: got stop word'
self._output_stream.send(r)
else:
# print 'SELECT: got record'
self._output_stream.send(self._t(r))
self._input_ep.processed()
except StreamClosedException:
closed = True
self._output_stream.close()
print 'Closing SELECT stream'
示例3: Aggregate
# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Aggregate(MiniEngine):
def __init__(self, input, aggregator):
MiniEngine.__init__(self)
self._input = input
self._input_ep = self._input.connect()
self._a = aggregator
assert self._a.accepts(self._input.schema())
self._output = Stream(
self._input.schema(),
self._input.sort_order(),
'Aggregate'
)
def output(self):
return self._output
def run(self):
closed = False
# Initialize aggregate
self._a.init()
while not closed:
try:
r = self._input_ep.receive()
if type(r) is StopWord:
# If the record is a stop word send the current
# aggregate value if anything was aggregated.
if self._a.count():
self._output.send(self._a.record())
# Additionally send the stop word
self._output.send(r)
# Reset the aggregate value
self._a.init()
else:
# Add the current record to the aggregate
self._a(r)
self._input_ep.processed()
except StreamClosedException:
closed = True
print 'Closing AGGREGATE stream'
self._output.close()
示例4: ArrayStreamer
# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class ArrayStreamer(MiniEngine):
def __init__(self, schema, data, sort_order = None):
MiniEngine.__init__(self)
self._schema = schema
self._data = data
self._output_stream = Stream(
self._schema,
sort_order or SortOrder(),
'ARRAY STREAMER'
)
def output(self):
return self._output_stream
def run(self):
for r in self._data:
self._output_stream.send(r)
print 'Closing ARRAY stream'
self._output_stream.close()
示例5: Component
# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Component():
def __init__(self, morse, name, fqn, stream = None, port = None, services = []):
self._morse = morse
self.name = name
self.fqn = fqn # fully qualified name
if stream == 'IN':
self.stream = Stream(self._morse.com, port)
self.publish = self.stream.publish
elif stream == 'OUT':
self.stream = Stream(self._morse.com, port)
self.get = self.stream.get
self.last = self.stream.last
self.subscribe = self.stream.subscribe
self.unsubscribe = self.stream.unsubscribe
else:
self.stream = None
for s in services:
pymorselogger.debug("Adding service %s to component %s" % (s, self.name))
self._add_service(s)
def _add_service(self, m):
def innermethod(*args):
pymorselogger.debug("Sending asynchronous request %s with args %s." % (m, args))
req = self._morse._make_request(self.fqn, m, *args)
future = self._morse.executor.submit(self._morse._execute_rpc, req)
#TODO: find a way to throw an execption in the main thread
# if the RPC request fails at invokation for stupid reasons
# like wrong # of params
return future
innermethod.__doc__ = "This method is a proxy for the MORSE %s service." % m
innermethod.__name__ = str(m)
setattr(self,innermethod.__name__,innermethod)
def close(self):
if self.stream:
self.stream.close()
示例6: Filter
# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Filter(MiniEngine):
def __init__(self, input, predicate):
MiniEngine.__init__(self)
self._input = input
self._predicate = predicate
assert self._predicate.accepts(self._input.schema())
self._input_ep = self._input.connect()
self._output = Stream(
self._input.schema(),
self._input.sort_order(),
'Filter'
)
def output(self):
return self._output
def run(self):
closed = False
while not closed:
try:
r = self._input_ep.receive()
if type(r) is StopWord:
# Send if the record is a stop word
self._output.send(r)
elif self._predicate(r):
# Send if the record satisfies the predicate.
self._output.send(r)
self._input_ep.processed()
except StreamClosedException:
closed = True
print 'Closing FILTER stream'
self._output.close()
示例7: Join
# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
#.........这里部分代码省略.........
def __init__(self, first, second):
MiniEngine.__init__(self)
self._first = first
self._second = second
# Construct the schema of the output stream.
self._schema = Schema()
for a in self._first.schema() + self._second.schema():
self._schema.append(a)
self._queue = Queue(100)
self._first_ep = self._first.connect()
self._first_ep.notify(self._queue)
self._second_ep = self._second.connect()
self._second_ep.notify(self._queue)
self._output = Stream(
self._schema,
SortOrder(),
'Join'
)
self._m = {
self._first_ep: self._first,
self._second_ep: self._second,
}
self._empty = 0
def output(self):
return self._output
def _merge(self, buffers, i):
assert buffers[self._first_ep].finished(i)
assert buffers[self._second_ep].finished(i)
b1 = buffers[self._first_ep].get(i)
b2 = buffers[self._second_ep].get(i)
if len(b2) == 1:
self._empty += 1
for r1 in b1[:-1]:
for r2 in b2[:-1]:
yield r1 + r2
buffers[self._first_ep].remove(i)
buffers[self._second_ep].remove(i)
def run(self):
done = False
buffers = {
self._first_ep: self.PartitionBuffer(),
self._second_ep: self.PartitionBuffer(),
}
while not done or not self._queue.empty():
e = self._queue.get()
if e not in buffers:
print 'ERROR: no buffer for endpoint'
continue
valid = True
closed = False
while valid and not closed:
try:
r = e.receive(False)
buffers[e].append(r)
if type(r) is StopWord:
current = buffers[e].current()
buffers[e].next()
# Only merge if all buffers have completed this
# partition.
merge = True
for o in buffers:
merge &= buffers[o].finished(current)
if merge:
for x in self._merge(buffers, current):
self._output.send(x)
self._output.send(StopWord())
# Advance this buffer's partition by 1
e.processed()
except StreamClosedException:
closed = True
except Empty:
valid = False
except:
raise
else:
done = True
for o in buffers:
done &= o.closed()
self._queue.task_done()
self._output.close()
print 'Join done. %d empty buffers.' % (self._empty)
示例8: Sort
# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Sort(MiniEngine):
def __init__(self, input_stream, sort_attributes, all = False):
MiniEngine.__init__(self)
self._input_stream = input_stream
self._input_ep = input_stream.connect()
self._schema = self._input_stream.schema()
self._all = all
self._indices = []
# Passed as attribues = [('name', comparison_function), ...]
for a in sort_attributes:
# Check if the given attribute exists in the schema.
i = self._schema.index(a[0])
t = self._schema[i].type()
if a[1]:
# If a comparison function is specified, use it.
self._indices.append((i, a[1]))
elif hasattr(t, '__cmp__'):
# Otherwise test if the given type has a comparator and use
# it.
self._indices.append((i, None))
else:
raise Exception('Type of attribute [%s] does not have ' + \
'a comparison operator.' % (a))
self._output_stream = Stream(
self._schema,
SortOrder(),
'SORT'
)
def output(self):
return self._output_stream
def _compare(self, a, b):
for i in self._indices:
# Defined as i = (index, comparator)
if i[1]:
x = i[1](a[i[0]], b[i[0]])
if x != 0:
return x
else:
x = cmp(a[i[0]], b[i[0]])
if x != 0:
return x
return 0
def run(self):
closed = False
last = None
set = []
while not closed:
try:
r = self._input_ep.receive()
if type(r) is StopWord:
if not self._all:
set.sort(self._compare)
for x in set:
self._output_stream.send(x)
set = []
self._output_stream.send(r)
else:
set.append(r)
self._input_ep.processed()
except StreamClosedException:
closed = True
if self._all:
set.sort(self._compare)
for x in set:
self._output_stream.send(x)
self._output_stream.send(StopWord())
print 'Closing SORT stream'
self._output_stream.close()
示例9: Group
# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Group(MiniEngine):
def __init__(self, input_stream, group_attributes):
MiniEngine.__init__(self)
self._input_stream = input_stream
self._input_ep = input_stream.connect()
self._schema = self._input_stream.schema()
print self._schema
self._indices = {}
for a in group_attributes:
i = self._schema.index(a)
t = self._schema[i].type()
if group_attributes[a]:
self._indices[i] = group_attributes[a]
elif hasattr(t, '__eq__'):
self._indices[i] = None
else:
raise Exception('Type of attribute [%s] does not have ' + \
'an equality operator.' % (a))
self._output_stream = Stream(
self._schema,
self._input_stream.sort_order(),
'GROUP'
)
def output(self):
return self._output_stream
def _compare(self, a, b):
for i in self._indices:
if self._indices[i]:
if not self._indices[i](a[i], b[i]):
return False
else:
if a[i] != b[i]:
return False
return True
def run(self):
closed = False
last = None
while not closed:
try:
r = self._input_ep.receive()
if type(r) is StopWord:
# print 'Sending: %s' % (str(StopWord()))
#self._output_stream.send(StopWord())
# print 'Sending: %s' % (str(r))
#self._output_stream.send(r)
pass
else:
if last == None or self._compare(last, r):
# print 'Sending: %s' % (str(r))
self._output_stream.send(r)
else:
# print 'Sending: %s' % (str(StopWord()))
self._output_stream.send(StopWord())
# print 'Sending: %s' % (str(r))
self._output_stream.send(r)
last = r
self._input_ep.processed()
except StreamClosedException:
closed = True
self._output_stream.send(StopWord())
print 'Closing GROUP stream'
self._output_stream.close()
示例10: DataAccessor
# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class DataAccessor(MiniEngine):
def __init__(self, query_stream, data_source, access_method):
MiniEngine.__init__(self)
self._query_stream = query_stream
# Create an accessor for the combination of access method and data
# source. This should fail if access method and data source are not
# compatible.
self._accessor = access_method(data_source)
self._data_source = data_source
self._access_method = access_method
# make sure the accessor understands the query schema
assert self._accessor.accepts(self._query_stream.schema())
self._query_stream_ep = self._query_stream.connect()
# Create an output stream for this data accessor. The schema of the
# output stream is determined by the data source.
output_schema = self._data_source.schema()
# We can only reasonably infer the sort order if all of the query
# attributes are included in the output schema.
if query_stream.sort_order() in output_schema:
# The new sort order is the same as that of the query stream.
sort_order = query_stream.sort_order()
else:
# No sort order can be inferred; using empty.
sort_order = SortOrder()
self._output_stream = Stream(
output_schema,
sort_order,
'DATA ACCESSOR'
)
def output(self):
'''
Returns the output stream for the given data accessor.
'''
return self._output_stream;
def run(self):
'''
The main loop of the data accessor mini-engine. It processes every
element from the query stream and performs a corresponding query
against the configured data source.
'''
# for each query in the query stream's end-point
i = 0
closed = False
while not closed:
try:
# print 'DA: Waiting to receive'
q = self._query_stream_ep.receive()
i += 1
if type(q) is StopWord:
continue
if q:
# process the query and write result records into the output
# stream
for r in self._accessor.query(self._query_stream.schema(), q):
self._output_stream.send(r)
# finalize the partition that belongs to one query with a
# stop word
self._output_stream.send(StopWord())
else:
# Technically stop words are not allowed in the query
# stream, but they are silently ignored here.
pass
self._query_stream_ep.processed()
except StreamClosedException:
closed = True
self._output_stream.close()
print 'Closing DA stream'
示例11: Mux
# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Mux(MiniEngine):
def __init__(self, *streams):
MiniEngine.__init__(self)
if not streams:
raise Exception('Mux: must specify at least one stream.')
self._streams = streams
self._queue = Queue(100)
self._stats = {}
self._endpoints = dict([(s.connect(), s) for s in self._streams])
for e in self._endpoints:
self._stats[e] = 0
e.notify(self._queue)
for s in self._streams:
if s.schema() != self._streams[0].schema():
raise Exception('Mux: schema of streams must match.')
self._output = Stream(
self._streams[0].schema(),
SortOrder(),
'Mux Output'
)
def output(self):
return self._output
def run(self):
while self._endpoints or not self._queue.empty():
# print '\t\twaiting for endpoint'
e = self._queue.get()
if e not in self._endpoints:
print '\t********* got non-existing endpoint'
continue
valid = True
closed = False
while valid and not closed:
# print '\t\t%s: receiving' % (self._endpoints[e])
try:
r = e.receive(False)
self._stats[e] += 1
self._output.send(r)
e.processed()
except StreamClosedException:
# print '\t\tReceive ClosedException.'
closed = True
except:
valid = False
# print '\t\tReceive failed.'
else:
if e in self._endpoints:
# print '%s: closed? %s' % (self._endpoints[e], e.closed())
if closed:
# print '%s: closed? %s' % (self._endpoints[e], e.closed())
del self._endpoints[e]
else:
# print '%s: already removed' % (e)
pass
self._queue.task_done()
#print '\t\tAll streams done.'
self._output.close()
# for e in self._stats:
# print 'Received %d records from %s' % (self._stats[e], e)
print 'Mux: done.'
示例12: Tunnel
# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Tunnel(object):
_TCP_INITIAL_DATA = 0
_TCP_FIN_DATA = 1
_TCP_CLOSED_DATA = 2
_UDP_INITIAL_DATA = 3
_UDP_CLOSED_DATA = 4
_TUN_INITIAL_DATA = 5
_PAYLOAD = 10
_HEARTBEAT = 100
_static_handlers = {
_HEARTBEAT: (lambda _, __, ___: None)
}
@staticmethod
def set_tcp_initial_handler(handler):
Tunnel._static_handlers[Tunnel._TCP_INITIAL_DATA] = handler
@staticmethod
def set_tcp_fin_received_handler(handler):
Tunnel._static_handlers[Tunnel._TCP_FIN_DATA] = handler
@staticmethod
def set_tcp_closed_handler(handler):
Tunnel._static_handlers[Tunnel._TCP_CLOSED_DATA] = handler
@staticmethod
def set_udp_initial_handler(handler):
Tunnel._static_handlers[Tunnel._UDP_INITIAL_DATA] = handler
@staticmethod
def set_udp_closed_handler(handler):
Tunnel._static_handlers[Tunnel._UDP_CLOSED_DATA] = handler
@staticmethod
def set_tun_initial_handler(handler):
Tunnel._static_handlers[Tunnel._TUN_INITIAL_DATA] = handler
def __init__(self, connection=None, connect_to=None):
self._stream = connection
self._connect_to = connect_to
self._on_initial_data = None
self._on_payload = None
self._on_stream_closed = None
self._handlers = self._static_handlers.copy()
self._handlers.update({
Tunnel._PAYLOAD: lambda _, id_, data: self._on_payload(self, id_, data)
})
self._on_ready_to_send = None
self._on_send_buffer_full = None
self._hb_event = None
self.connections = {}
def __hash__(self):
return hash(self._stream)
def __eq__(self, other):
if not isinstance(other, Tunnel):
return False
return self._stream == other._stream
def __str__(self):
return str(self._stream)
def _send_heartbeat(self):
self._send_content(Tunnel._HEARTBEAT, None, None)
self._enable_heartbeat()
def _enable_heartbeat(self):
self._hb_event = Event.add_timer(HEARTBEAT_INTERVAL)
self._hb_event.set_handler(lambda ev: self._send_heartbeat())
def _disable_heartbeat(self):
if self._hb_event is not None:
self._hb_event.del_timer()
self._hb_event = None
def _on_fin_received(self):
self._disable_heartbeat()
self._stream.close()
def initialize(self):
if self._stream is None:
self._stream = Stream(prefix='TUNNEL')
# self._stream.set_buffer_size(BUFF_SIZE)
self._stream.set_tcp_no_delay()
self._stream.append_send_handler(obscure.pack_data)
self._stream.append_send_handler(obscure.random_padding)
# self._stream.append_send_handler(obscure.gen_aes_encrypt())
self._stream.append_send_handler(obscure.gen_xor_encrypt())
# self._stream.append_send_handler(obscure.base64_encode)
self._stream.append_send_handler(obscure.gen_http_encode(self._connect_to is not None))
self._stream.append_receive_handler(obscure.gen_http_decode(self._connect_to is not None))
# self._stream.append_receive_handler(obscure.base64_decode)
self._stream.append_receive_handler(obscure.gen_xor_decrypt())
# self._stream.append_receive_handler(obscure.gen_aes_decrypt())
self._stream.append_receive_handler(obscure.unpad_random)
self._stream.append_receive_handler(obscure.unpack_data)
#.........这里部分代码省略.........