Python Stream.close方法代码示例

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
 def _on_accept(self):
     _logger.debug('_on_accept')
     while True:
         try:
             sock, addr = self._fd.accept()
             _logger.debug('fd: %d accept fd: %d',
                           self._fd.fileno(), sock.fileno())
         except socket.error as msg:
             if msg.errno == errno.ECONNABORTED:
                 continue
             if msg.errno != errno.EAGAIN and msg.errno != errno.EINPROGRESS:
                 _logger.error('fd: %d, accept: %s',
                               self._fd.fileno(), os.strerror(msg.errno))
                 self._fd.close()
                 if self._onClosed is not None:
                     try:
                         self._onClosed(self)
                     except Exception as ex:
                         _logger.error('_onClosed: %s', str(ex))
                         _logger.exception(traceback.format_exc())
             return
         else:
             new_stream = Stream(sock, prefix=self._prefix)
             new_stream._connected = True
             try:
                 self._onAccepted(new_stream, addr)
             except Exception as e:
                 _logger.error('_onAccepted: %s', e)
                 _logger.exception(traceback.format_exc())
                 new_stream.close()

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Select(MiniEngine):
    def __init__(self, input, transformer):
        MiniEngine.__init__(self)
        self._input = input
        self._input_ep = input.connect()
        self._t = transformer

        # make sure the transformer can handle the records in the stream
        assert self._t.accepts(input.schema())

        # We cannot reliably determine the sort order of the output stream
        # as the transformation applied to the attributes is unkown. If
        # for example the transformer inverts the value of an attribute the
        # attribute still has the same time and possibly name, but the
        # values in fact should now be sorted in reverse order.
        output_order = SortOrder()
        
        # Construct the output stream.
        self._output_stream = Stream(
            self._t.schema(), 
            output_order,
            'SELECT'
        )

    def output(self):
        return self._output_stream
    
    def run(self):
        closed = False
        while not closed:
            try:
                # print 'SELECT: waiting to receive'
                r = self._input_ep.receive()
                if type(r) is StopWord:
                    # print 'SELECT: got stop word'
                    self._output_stream.send(r)
                else:
                    # print 'SELECT: got record'
                    self._output_stream.send(self._t(r))
                self._input_ep.processed()
            except StreamClosedException:
                closed = True
        self._output_stream.close()
        print 'Closing SELECT stream'

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Aggregate(MiniEngine):
    def __init__(self, input, aggregator):
        MiniEngine.__init__(self)
        self._input = input
        self._input_ep = self._input.connect()
        self._a = aggregator
        assert self._a.accepts(self._input.schema())

        self._output = Stream(
            self._input.schema(),
            self._input.sort_order(),
            'Aggregate'
        )

    def output(self):
        return self._output

    def run(self):
        closed = False

        # Initialize aggregate
        self._a.init()
        while not closed:
            try:
                r = self._input_ep.receive()
                if type(r) is StopWord:
                    # If the record is a stop word send the current
                    # aggregate value if anything was aggregated.
                    if self._a.count():
                        self._output.send(self._a.record())
                    # Additionally send the stop word
                    self._output.send(r)
                    # Reset the aggregate value
                    self._a.init()
                else:
                    # Add the current record to the aggregate
                    self._a(r)
                self._input_ep.processed()
            except StreamClosedException:
                closed = True
        print 'Closing AGGREGATE stream'
        self._output.close()

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class ArrayStreamer(MiniEngine):
    def __init__(self, schema, data, sort_order = None):
        MiniEngine.__init__(self)

        self._schema = schema
        self._data = data

        self._output_stream = Stream(
            self._schema, 
            sort_order or SortOrder(),
            'ARRAY STREAMER'
        )

    def output(self):
        return self._output_stream

    def run(self):
        for r in self._data:
            self._output_stream.send(r)
        print 'Closing ARRAY stream'
        self._output_stream.close()

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Component():
    def __init__(self, morse, name, fqn, stream = None, port = None, services = []):
        self._morse = morse
        self.name = name
        self.fqn = fqn # fully qualified name

        if stream == 'IN':
            self.stream = Stream(self._morse.com, port)
            self.publish = self.stream.publish
        elif stream == 'OUT':
            self.stream = Stream(self._morse.com, port)
            self.get = self.stream.get
            self.last = self.stream.last
            self.subscribe = self.stream.subscribe
            self.unsubscribe = self.stream.unsubscribe
        else:
            self.stream = None

        for s in services:
            pymorselogger.debug("Adding service %s to component %s" % (s, self.name))
            self._add_service(s)

    def _add_service(self, m):
        def innermethod(*args):
            pymorselogger.debug("Sending asynchronous request %s with args %s." % (m, args))
            req = self._morse._make_request(self.fqn, m, *args)
            future = self._morse.executor.submit(self._morse._execute_rpc, req)
            #TODO: find a way to throw an execption in the main thread
            # if the RPC request fails at invokation for stupid reasons
            # like wrong # of params
            return future

        innermethod.__doc__ = "This method is a proxy for the MORSE %s service." % m
        innermethod.__name__ = str(m)
        setattr(self,innermethod.__name__,innermethod)

    def close(self):
        if self.stream:
            self.stream.close()

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Filter(MiniEngine):
    def __init__(self, input, predicate):
        MiniEngine.__init__(self)
        self._input = input
        self._predicate = predicate

        assert self._predicate.accepts(self._input.schema())

        self._input_ep = self._input.connect()

        self._output = Stream(
            self._input.schema(),
            self._input.sort_order(),
            'Filter'
        )

    def output(self):
        return self._output

    def run(self):
        closed = False
        while not closed:
            try:
                r = self._input_ep.receive()
                
                if type(r) is StopWord:
                    # Send if the record is a stop word
                    self._output.send(r)
                elif self._predicate(r):
                    # Send if the record satisfies the predicate.
                    self._output.send(r)
                self._input_ep.processed()
            except StreamClosedException:
                closed = True
        print 'Closing FILTER stream'
        self._output.close()

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]

#.........这里部分代码省略.........

    def __init__(self, first, second):
        MiniEngine.__init__(self)

        self._first = first
        self._second = second

        # Construct the schema of the output stream.
        self._schema = Schema()
        for a in self._first.schema() + self._second.schema():
            self._schema.append(a)

        self._queue = Queue(100)
        
        self._first_ep = self._first.connect()
        self._first_ep.notify(self._queue)

        self._second_ep = self._second.connect()
        self._second_ep.notify(self._queue)
        
        self._output = Stream(
            self._schema,
            SortOrder(),
            'Join'
        )

        self._m = {
            self._first_ep: self._first,
            self._second_ep: self._second,
        }

        self._empty = 0

    def output(self):
        return self._output

    def _merge(self, buffers, i):
        assert buffers[self._first_ep].finished(i)
        assert buffers[self._second_ep].finished(i)

        b1 = buffers[self._first_ep].get(i)
        b2 = buffers[self._second_ep].get(i)

        if len(b2) == 1:
            self._empty += 1

        for r1 in b1[:-1]:
            for r2 in b2[:-1]:
                yield r1 + r2

        buffers[self._first_ep].remove(i)
        buffers[self._second_ep].remove(i)
        
    def run(self):
        done = False
        buffers = {
            self._first_ep: self.PartitionBuffer(),
            self._second_ep: self.PartitionBuffer(),
        }
        while not done or not self._queue.empty():
            e = self._queue.get()
            
            if e not in buffers:
                print 'ERROR: no buffer for endpoint'
                continue

            valid = True
            closed = False
            while valid and not closed:
                try:
                    r = e.receive(False)
                    buffers[e].append(r)
                    if type(r) is StopWord:
                        current = buffers[e].current()
                        buffers[e].next()
                        # Only merge if all buffers have completed this
                        # partition.
                        merge = True
                        for o in buffers:
                            merge &= buffers[o].finished(current)
                        if merge:
                            for x in self._merge(buffers, current):
                                self._output.send(x)
                            self._output.send(StopWord())

                        # Advance this buffer's partition by 1
                    e.processed()
                except StreamClosedException:
                    closed = True
                except Empty:
                    valid = False
                except:
                    raise
            else:
                done = True
                for o in buffers:
                    done &= o.closed()
            self._queue.task_done()
        self._output.close()
        print 'Join done. %d empty buffers.' % (self._empty)

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Sort(MiniEngine):
    def __init__(self, input_stream, sort_attributes, all = False):
        MiniEngine.__init__(self)
        self._input_stream = input_stream
        self._input_ep = input_stream.connect()
        self._schema = self._input_stream.schema()
        self._all = all
        self._indices = []
        # Passed as attribues = [('name', comparison_function), ...]
        for a in sort_attributes:
            # Check if the given attribute exists in the schema.
            i = self._schema.index(a[0])
            t = self._schema[i].type()

            if a[1]:
                # If a comparison function is specified, use it.
                self._indices.append((i, a[1]))
            elif hasattr(t, '__cmp__'):
                # Otherwise test if the given type has a comparator and use
                # it.
                self._indices.append((i, None))
            else:
                raise Exception('Type of attribute [%s] does not have ' + \
                                'a comparison operator.' % (a))

        self._output_stream = Stream(
            self._schema,
            SortOrder(), 
            'SORT'
        )

    def output(self):
        return self._output_stream

    def _compare(self, a, b):
        for i in self._indices:
            # Defined as i = (index, comparator)
            if i[1]:
                x = i[1](a[i[0]], b[i[0]])
                if x != 0:
                    return x
            else:
                x = cmp(a[i[0]], b[i[0]])
                if x != 0:
                    return x
        return 0

    def run(self):
        closed = False
        last = None
        set = []
        while not closed:
            try:
                r = self._input_ep.receive()
                if type(r) is StopWord:
                    if not self._all:
                        set.sort(self._compare)
                        for x in set:
                            self._output_stream.send(x)
                        set = []
                        self._output_stream.send(r)
                else:
                    set.append(r)
                self._input_ep.processed()
            except StreamClosedException:
                closed = True
        if self._all:
            set.sort(self._compare)
            for x in set:
                self._output_stream.send(x)
            self._output_stream.send(StopWord())
        print 'Closing SORT stream'
        self._output_stream.close()

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Group(MiniEngine):
    def __init__(self, input_stream, group_attributes):
        MiniEngine.__init__(self)
        self._input_stream = input_stream
        self._input_ep = input_stream.connect()
        self._schema = self._input_stream.schema()
        print self._schema
        self._indices = {}
        for a in group_attributes:
            i = self._schema.index(a)
            t = self._schema[i].type()
            if group_attributes[a]:
                self._indices[i] = group_attributes[a]
            elif hasattr(t, '__eq__'):
                self._indices[i] = None
            else:
                raise Exception('Type of attribute [%s] does not have ' + \
                                'an equality operator.' % (a))

        self._output_stream = Stream(
            self._schema,
            self._input_stream.sort_order(), 
            'GROUP'
        )

    def output(self):
        return self._output_stream

    def _compare(self, a, b):
        for i in self._indices:
            if self._indices[i]:
                if not self._indices[i](a[i], b[i]):
                    return False
            else:
                if a[i] != b[i]:
                    return False
        return True

    def run(self):
        closed = False
        last = None
        while not closed:
            try:
                r = self._input_ep.receive()
                if type(r) is StopWord:
#                    print 'Sending: %s' % (str(StopWord()))
                    #self._output_stream.send(StopWord())
#                    print 'Sending: %s' % (str(r))
                    #self._output_stream.send(r)
                    pass
                else:
                    if last == None or self._compare(last, r):
#                        print 'Sending: %s' % (str(r))
                        self._output_stream.send(r)
                    else:
#                        print 'Sending: %s' % (str(StopWord()))
                        self._output_stream.send(StopWord())
#                        print 'Sending: %s' % (str(r))
                        self._output_stream.send(r)
                    last = r
                self._input_ep.processed()
            except StreamClosedException:
                closed = True
        self._output_stream.send(StopWord())
        print 'Closing GROUP stream'
        self._output_stream.close()

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class DataAccessor(MiniEngine):
    def __init__(self, query_stream, data_source, access_method):
        MiniEngine.__init__(self)
        self._query_stream = query_stream
        
        # Create an accessor for the combination of access method and data
        # source. This should fail if access method and data source are not
        # compatible.
        self._accessor = access_method(data_source)
        self._data_source = data_source
        self._access_method = access_method

        # make sure the accessor understands the query schema
        assert self._accessor.accepts(self._query_stream.schema())
        self._query_stream_ep = self._query_stream.connect()

        # Create an output stream for this data accessor. The schema of the
        # output stream is determined by the data source.
        output_schema = self._data_source.schema()
    
        # We can only reasonably infer the sort order if all of the query
        # attributes are included in the output schema.
        if query_stream.sort_order() in output_schema:
            # The new sort order is the same as that of the query stream.
            sort_order = query_stream.sort_order()
        else:
            # No sort order can be inferred; using empty.
            sort_order = SortOrder()

        self._output_stream = Stream(
            output_schema, 
            sort_order, 
            'DATA ACCESSOR'
        )

    def output(self):
        '''
        Returns the output stream for the given data accessor.
        '''
        return self._output_stream;

    def run(self):
        '''
        The main loop of the data accessor mini-engine. It processes every
        element from the query stream and performs a corresponding query
        against the configured data source.
        '''
        # for each query in the query stream's end-point
        i = 0
        closed = False
        while not closed:
            try:
                # print 'DA: Waiting to receive'
                q = self._query_stream_ep.receive()
                i += 1
                if type(q) is StopWord:
                    continue
                if q:
                    # process the query and write result records into the output
                    # stream
                    for r in self._accessor.query(self._query_stream.schema(), q):
                        self._output_stream.send(r)
                    # finalize the partition that belongs to one query with a
                    # stop word
                    self._output_stream.send(StopWord())
                else:
                    # Technically stop words are not allowed in the query
                    # stream, but they are silently ignored here.
                    pass
                self._query_stream_ep.processed()
            except StreamClosedException:
                closed = True
        self._output_stream.close()
        print 'Closing DA stream'

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Mux(MiniEngine):
    def __init__(self, *streams):
        MiniEngine.__init__(self)
        if not streams:
            raise Exception('Mux: must specify at least one stream.')
        self._streams = streams
        self._queue = Queue(100)
        self._stats = {}
        self._endpoints = dict([(s.connect(), s) for s in self._streams])
        for e in self._endpoints:
            self._stats[e] = 0
            e.notify(self._queue)

        for s in self._streams:
            if s.schema() != self._streams[0].schema():
                raise Exception('Mux: schema of streams must match.')

        self._output = Stream(
            self._streams[0].schema(),
            SortOrder(),
            'Mux Output'
        )

    def output(self):
        return self._output

    def run(self):
        while self._endpoints or not self._queue.empty():
            # print '\t\twaiting for endpoint'
            e = self._queue.get()
            
            if e not in self._endpoints:
                print '\t********* got non-existing endpoint'
                continue

            valid = True
            closed = False
            while valid and not closed:
                # print '\t\t%s: receiving' % (self._endpoints[e])
                try:
                    r = e.receive(False)
                    self._stats[e] += 1
                    self._output.send(r)
                    e.processed()
                except StreamClosedException:
                    # print '\t\tReceive ClosedException.'
                    closed = True
                except:
                    valid = False
                    # print '\t\tReceive failed.'
            else:
                if e in self._endpoints:
                    # print '%s: closed? %s' % (self._endpoints[e], e.closed())
                    if closed:
                        # print '%s: closed? %s' % (self._endpoints[e], e.closed())
                        del self._endpoints[e]
                else:
                    # print '%s: already removed' % (e)
                    pass
            self._queue.task_done()
        #print '\t\tAll streams done.'
        self._output.close()
        # for e in self._stats:
        #    print 'Received %d records from %s' % (self._stats[e], e)
        print 'Mux: done.'

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import close [as 别名]
class Tunnel(object):
    _TCP_INITIAL_DATA = 0
    _TCP_FIN_DATA = 1
    _TCP_CLOSED_DATA = 2
    _UDP_INITIAL_DATA = 3
    _UDP_CLOSED_DATA = 4
    _TUN_INITIAL_DATA = 5
    _PAYLOAD = 10
    _HEARTBEAT = 100

    _static_handlers = {
        _HEARTBEAT: (lambda _, __, ___: None)
    }

    @staticmethod
    def set_tcp_initial_handler(handler):
        Tunnel._static_handlers[Tunnel._TCP_INITIAL_DATA] = handler

    @staticmethod
    def set_tcp_fin_received_handler(handler):
        Tunnel._static_handlers[Tunnel._TCP_FIN_DATA] = handler

    @staticmethod
    def set_tcp_closed_handler(handler):
        Tunnel._static_handlers[Tunnel._TCP_CLOSED_DATA] = handler

    @staticmethod
    def set_udp_initial_handler(handler):
        Tunnel._static_handlers[Tunnel._UDP_INITIAL_DATA] = handler

    @staticmethod
    def set_udp_closed_handler(handler):
        Tunnel._static_handlers[Tunnel._UDP_CLOSED_DATA] = handler

    @staticmethod
    def set_tun_initial_handler(handler):
        Tunnel._static_handlers[Tunnel._TUN_INITIAL_DATA] = handler

    def __init__(self, connection=None, connect_to=None):
        self._stream = connection
        self._connect_to = connect_to
        self._on_initial_data = None
        self._on_payload = None
        self._on_stream_closed = None
        self._handlers = self._static_handlers.copy()
        self._handlers.update({
            Tunnel._PAYLOAD: lambda _, id_, data: self._on_payload(self, id_, data)
        })
        self._on_ready_to_send = None
        self._on_send_buffer_full = None
        self._hb_event = None
        self.connections = {}

    def __hash__(self):
        return hash(self._stream)

    def __eq__(self, other):
        if not isinstance(other, Tunnel):
            return False
        return self._stream == other._stream

    def __str__(self):
        return str(self._stream)

    def _send_heartbeat(self):
        self._send_content(Tunnel._HEARTBEAT, None, None)
        self._enable_heartbeat()

    def _enable_heartbeat(self):
        self._hb_event = Event.add_timer(HEARTBEAT_INTERVAL)
        self._hb_event.set_handler(lambda ev: self._send_heartbeat())

    def _disable_heartbeat(self):
        if self._hb_event is not None:
            self._hb_event.del_timer()
            self._hb_event = None

    def _on_fin_received(self):
        self._disable_heartbeat()
        self._stream.close()

    def initialize(self):
        if self._stream is None:
            self._stream = Stream(prefix='TUNNEL')

        # self._stream.set_buffer_size(BUFF_SIZE)
        self._stream.set_tcp_no_delay()

        self._stream.append_send_handler(obscure.pack_data)
        self._stream.append_send_handler(obscure.random_padding)
        # self._stream.append_send_handler(obscure.gen_aes_encrypt())
        self._stream.append_send_handler(obscure.gen_xor_encrypt())
        # self._stream.append_send_handler(obscure.base64_encode)
        self._stream.append_send_handler(obscure.gen_http_encode(self._connect_to is not None))
        self._stream.append_receive_handler(obscure.gen_http_decode(self._connect_to is not None))
        # self._stream.append_receive_handler(obscure.base64_decode)
        self._stream.append_receive_handler(obscure.gen_xor_decrypt())
        # self._stream.append_receive_handler(obscure.gen_aes_decrypt())
        self._stream.append_receive_handler(obscure.unpad_random)
        self._stream.append_receive_handler(obscure.unpack_data)
#.........这里部分代码省略.........