Python Stream.send方法代码示例

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import send [as 别名]
    def _on_decode_error(self, received):
        self._disable_heartbeat()
        self._stream._encoders = []
        backend = Stream(prefix="SIMPLE")

        def tunnel_ready_to_send(_):
            backend.start_receiving()

        def tunnel_send_buffer_full(_):
            backend.stop_receiving()

        def tunnel_received(_, data, _addr):
            backend.send(data)
            return backend.is_ready_to_send()

        def tunnel_closed(_):
            backend.close()

        def backend_received(_, data, _addr):
            self._stream.send(data)
            return self._stream.is_ready_to_send()

        def backend_closed(_self):
            self._stream.close()

        self._stream.set_on_ready_to_send(tunnel_ready_to_send)
        self._stream.set_on_send_buffer_full(tunnel_send_buffer_full)
        self._stream.set_on_received(tunnel_received)
        self._stream.set_on_closed(tunnel_closed)
        backend.set_on_received(backend_received)
        backend.set_on_closed(backend_closed)
        if received is not None and len(received) > 0:
            backend.send(received)
        backend.connect(UNKNOWN_CONN_ADDR, UNKNOWN_CONN_PORT)

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import send [as 别名]
class Select(MiniEngine):
    def __init__(self, input, transformer):
        MiniEngine.__init__(self)
        self._input = input
        self._input_ep = input.connect()
        self._t = transformer

        # make sure the transformer can handle the records in the stream
        assert self._t.accepts(input.schema())

        # We cannot reliably determine the sort order of the output stream
        # as the transformation applied to the attributes is unkown. If
        # for example the transformer inverts the value of an attribute the
        # attribute still has the same time and possibly name, but the
        # values in fact should now be sorted in reverse order.
        output_order = SortOrder()
        
        # Construct the output stream.
        self._output_stream = Stream(
            self._t.schema(), 
            output_order,
            'SELECT'
        )

    def output(self):
        return self._output_stream
    
    def run(self):
        closed = False
        while not closed:
            try:
                # print 'SELECT: waiting to receive'
                r = self._input_ep.receive()
                if type(r) is StopWord:
                    # print 'SELECT: got stop word'
                    self._output_stream.send(r)
                else:
                    # print 'SELECT: got record'
                    self._output_stream.send(self._t(r))
                self._input_ep.processed()
            except StreamClosedException:
                closed = True
        self._output_stream.close()
        print 'Closing SELECT stream'

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import send [as 别名]
class Aggregate(MiniEngine):
    def __init__(self, input, aggregator):
        MiniEngine.__init__(self)
        self._input = input
        self._input_ep = self._input.connect()
        self._a = aggregator
        assert self._a.accepts(self._input.schema())

        self._output = Stream(
            self._input.schema(),
            self._input.sort_order(),
            'Aggregate'
        )

    def output(self):
        return self._output

    def run(self):
        closed = False

        # Initialize aggregate
        self._a.init()
        while not closed:
            try:
                r = self._input_ep.receive()
                if type(r) is StopWord:
                    # If the record is a stop word send the current
                    # aggregate value if anything was aggregated.
                    if self._a.count():
                        self._output.send(self._a.record())
                    # Additionally send the stop word
                    self._output.send(r)
                    # Reset the aggregate value
                    self._a.init()
                else:
                    # Add the current record to the aggregate
                    self._a(r)
                self._input_ep.processed()
            except StreamClosedException:
                closed = True
        print 'Closing AGGREGATE stream'
        self._output.close()

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import send [as 别名]
class ArrayStreamer(MiniEngine):
    def __init__(self, schema, data, sort_order = None):
        MiniEngine.__init__(self)

        self._schema = schema
        self._data = data

        self._output_stream = Stream(
            self._schema, 
            sort_order or SortOrder(),
            'ARRAY STREAMER'
        )

    def output(self):
        return self._output_stream

    def run(self):
        for r in self._data:
            self._output_stream.send(r)
        print 'Closing ARRAY stream'
        self._output_stream.close()

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import send [as 别名]
class Filter(MiniEngine):
    def __init__(self, input, predicate):
        MiniEngine.__init__(self)
        self._input = input
        self._predicate = predicate

        assert self._predicate.accepts(self._input.schema())

        self._input_ep = self._input.connect()

        self._output = Stream(
            self._input.schema(),
            self._input.sort_order(),
            'Filter'
        )

    def output(self):
        return self._output

    def run(self):
        closed = False
        while not closed:
            try:
                r = self._input_ep.receive()
                
                if type(r) is StopWord:
                    # Send if the record is a stop word
                    self._output.send(r)
                elif self._predicate(r):
                    # Send if the record satisfies the predicate.
                    self._output.send(r)
                self._input_ep.processed()
            except StreamClosedException:
                closed = True
        print 'Closing FILTER stream'
        self._output.close()

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import send [as 别名]
    def run(self):
        """Starts the loop"""
        print (NICK)
        print ("Bot.run()")

        Stream.connect((NETWORK, PORT))
        # receive buffer, and connect setup
        Stream.recv(4096)  # rcv buffer
        Stream.send("NICK " + NICK + "\r\n")
        Stream.send("USER magikarp magikarp magikarp :magikarp\r\n")

        # main loop
        while True:
            data = Stream.recv(4096)  # get lines
            print (data)  # print lines

            # Basic init commands after server connection
            if data.find("MODE " + NICK + " +i") != -1:
                Stream.send("JOIN " + CHAN + "\r\n")
                # Stream.send('PRIVMSG ' + CHAN + ' :Morning, ' + CHAN + '\r\n')

            # Constant ping lookout
            if data.find("PING") != -1:
                Stream.send("PONG " + data.split()[1] + "\r\n")

            elif data.find("PRIVMSG") != -1:  # if there is a PRIVMSG in data then parse it
                message = ":".join(data.split(":")[2:])  # split the command from the message
                print (message)

                function = message.split()[0]  # split the massage to get function name

                if (
                    message.lower().find("awesome") != -1 and not function.find("^") != -1
                ):  # split the massage to get function name:
                    nick = data.split("!")[0].replace(":", "")  # snatch the nick issuing the command
                    destination = "".join(data.split(":")[:2]).split(" ")[-2]
                    # Stream.send('PRIVMSG ' + destination + ' :Yeah ' + nick + '! Awesome!\r\n')

                if Parser().ContainsAny(message, ["http", "http", "www", ".com", ".org", ".eu"]) == 1:
                    nick = data.split("!")[0].replace(":", "")  # snatch the nick issuing the command
                    destination = "".join(data.split(":")[:2]).split(" ")[-2]
                    arg = data.split()
                    args = []
                    for index, item in enumerate(arg):  # for every index and item in arg
                        if (
                            index > 2
                            and Parser().ContainsAny(item, ["http", "http", "www", ".com", ".org", ".eu"]) == 1
                        ):
                            n = 1
                            if args == []:
                                # item = (item.split(':', 1)[1])
                                args.append(item)
                            else:
                                args.append(" " + item)
                                n += 1

                    args.append("\n")
                    print args

                    if args != "":
                        fileObj = open(FILEDIR + "/botlinks", "a")
                        fileObj.write("[" + destination + "] " + CurrentTimeString() + " " + nick + ": ")
                        for i in args:
                            fileObj.write(i)
                        fileObj.close()

                if message.lower().find("^") != -1:  # if the message contains the chan name
                    nick = data.split("!")[0].replace(":", "")  # snatch the nick issuing the command
                    print ("nick: " + nick)
                    destination = "".join(data.split(":")[:2]).split(" ")[-2]
                    print ("dest: " + destination)
                    function = message.split()[0]  # split the massage to get function name
                    print ("function: " + function)
                    print ("The function called is " + function + " from " + nick)  # command and the caller
                    arg = data.split()  # arg[0] is the actual comand

                    args = ""
                    for index, item in enumerate(arg):  # for every index and item in arg
                        if index > 3:
                            if args == "":
                                args = item
                            else:
                                args += " " + item
                    print (args)

                    if function == "^credits":  # if function is equal to ^credits
                        Stream.send(
                            "PRIVMSG "
                            + destination
                            + " :"
                            + nick
                            + ": I'm developed by magikmw - http://github.com/magikmw/magikarp \r\n"
                        )

                    elif function == "^say":
                        if args != "":
                            # Stream.send('PRIVMSG ' + destination + ' :' + args + '\r\n')
                            Stream.send(
                                "PRIVMSG "
                                + destination
#.........这里部分代码省略.........

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import send [as 别名]

#.........这里部分代码省略.........

    def __init__(self, first, second):
        MiniEngine.__init__(self)

        self._first = first
        self._second = second

        # Construct the schema of the output stream.
        self._schema = Schema()
        for a in self._first.schema() + self._second.schema():
            self._schema.append(a)

        self._queue = Queue(100)
        
        self._first_ep = self._first.connect()
        self._first_ep.notify(self._queue)

        self._second_ep = self._second.connect()
        self._second_ep.notify(self._queue)
        
        self._output = Stream(
            self._schema,
            SortOrder(),
            'Join'
        )

        self._m = {
            self._first_ep: self._first,
            self._second_ep: self._second,
        }

        self._empty = 0

    def output(self):
        return self._output

    def _merge(self, buffers, i):
        assert buffers[self._first_ep].finished(i)
        assert buffers[self._second_ep].finished(i)

        b1 = buffers[self._first_ep].get(i)
        b2 = buffers[self._second_ep].get(i)

        if len(b2) == 1:
            self._empty += 1

        for r1 in b1[:-1]:
            for r2 in b2[:-1]:
                yield r1 + r2

        buffers[self._first_ep].remove(i)
        buffers[self._second_ep].remove(i)
        
    def run(self):
        done = False
        buffers = {
            self._first_ep: self.PartitionBuffer(),
            self._second_ep: self.PartitionBuffer(),
        }
        while not done or not self._queue.empty():
            e = self._queue.get()
            
            if e not in buffers:
                print 'ERROR: no buffer for endpoint'
                continue

            valid = True
            closed = False
            while valid and not closed:
                try:
                    r = e.receive(False)
                    buffers[e].append(r)
                    if type(r) is StopWord:
                        current = buffers[e].current()
                        buffers[e].next()
                        # Only merge if all buffers have completed this
                        # partition.
                        merge = True
                        for o in buffers:
                            merge &= buffers[o].finished(current)
                        if merge:
                            for x in self._merge(buffers, current):
                                self._output.send(x)
                            self._output.send(StopWord())

                        # Advance this buffer's partition by 1
                    e.processed()
                except StreamClosedException:
                    closed = True
                except Empty:
                    valid = False
                except:
                    raise
            else:
                done = True
                for o in buffers:
                    done &= o.closed()
            self._queue.task_done()
        self._output.close()
        print 'Join done. %d empty buffers.' % (self._empty)

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import send [as 别名]
class Sort(MiniEngine):
    def __init__(self, input_stream, sort_attributes, all = False):
        MiniEngine.__init__(self)
        self._input_stream = input_stream
        self._input_ep = input_stream.connect()
        self._schema = self._input_stream.schema()
        self._all = all
        self._indices = []
        # Passed as attribues = [('name', comparison_function), ...]
        for a in sort_attributes:
            # Check if the given attribute exists in the schema.
            i = self._schema.index(a[0])
            t = self._schema[i].type()

            if a[1]:
                # If a comparison function is specified, use it.
                self._indices.append((i, a[1]))
            elif hasattr(t, '__cmp__'):
                # Otherwise test if the given type has a comparator and use
                # it.
                self._indices.append((i, None))
            else:
                raise Exception('Type of attribute [%s] does not have ' + \
                                'a comparison operator.' % (a))

        self._output_stream = Stream(
            self._schema,
            SortOrder(), 
            'SORT'
        )

    def output(self):
        return self._output_stream

    def _compare(self, a, b):
        for i in self._indices:
            # Defined as i = (index, comparator)
            if i[1]:
                x = i[1](a[i[0]], b[i[0]])
                if x != 0:
                    return x
            else:
                x = cmp(a[i[0]], b[i[0]])
                if x != 0:
                    return x
        return 0

    def run(self):
        closed = False
        last = None
        set = []
        while not closed:
            try:
                r = self._input_ep.receive()
                if type(r) is StopWord:
                    if not self._all:
                        set.sort(self._compare)
                        for x in set:
                            self._output_stream.send(x)
                        set = []
                        self._output_stream.send(r)
                else:
                    set.append(r)
                self._input_ep.processed()
            except StreamClosedException:
                closed = True
        if self._all:
            set.sort(self._compare)
            for x in set:
                self._output_stream.send(x)
            self._output_stream.send(StopWord())
        print 'Closing SORT stream'
        self._output_stream.close()

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import send [as 别名]
class Group(MiniEngine):
    def __init__(self, input_stream, group_attributes):
        MiniEngine.__init__(self)
        self._input_stream = input_stream
        self._input_ep = input_stream.connect()
        self._schema = self._input_stream.schema()
        print self._schema
        self._indices = {}
        for a in group_attributes:
            i = self._schema.index(a)
            t = self._schema[i].type()
            if group_attributes[a]:
                self._indices[i] = group_attributes[a]
            elif hasattr(t, '__eq__'):
                self._indices[i] = None
            else:
                raise Exception('Type of attribute [%s] does not have ' + \
                                'an equality operator.' % (a))

        self._output_stream = Stream(
            self._schema,
            self._input_stream.sort_order(), 
            'GROUP'
        )

    def output(self):
        return self._output_stream

    def _compare(self, a, b):
        for i in self._indices:
            if self._indices[i]:
                if not self._indices[i](a[i], b[i]):
                    return False
            else:
                if a[i] != b[i]:
                    return False
        return True

    def run(self):
        closed = False
        last = None
        while not closed:
            try:
                r = self._input_ep.receive()
                if type(r) is StopWord:
#                    print 'Sending: %s' % (str(StopWord()))
                    #self._output_stream.send(StopWord())
#                    print 'Sending: %s' % (str(r))
                    #self._output_stream.send(r)
                    pass
                else:
                    if last == None or self._compare(last, r):
#                        print 'Sending: %s' % (str(r))
                        self._output_stream.send(r)
                    else:
#                        print 'Sending: %s' % (str(StopWord()))
                        self._output_stream.send(StopWord())
#                        print 'Sending: %s' % (str(r))
                        self._output_stream.send(r)
                    last = r
                self._input_ep.processed()
            except StreamClosedException:
                closed = True
        self._output_stream.send(StopWord())
        print 'Closing GROUP stream'
        self._output_stream.close()

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import send [as 别名]
class DataAccessor(MiniEngine):
    def __init__(self, query_stream, data_source, access_method):
        MiniEngine.__init__(self)
        self._query_stream = query_stream
        
        # Create an accessor for the combination of access method and data
        # source. This should fail if access method and data source are not
        # compatible.
        self._accessor = access_method(data_source)
        self._data_source = data_source
        self._access_method = access_method

        # make sure the accessor understands the query schema
        assert self._accessor.accepts(self._query_stream.schema())
        self._query_stream_ep = self._query_stream.connect()

        # Create an output stream for this data accessor. The schema of the
        # output stream is determined by the data source.
        output_schema = self._data_source.schema()
    
        # We can only reasonably infer the sort order if all of the query
        # attributes are included in the output schema.
        if query_stream.sort_order() in output_schema:
            # The new sort order is the same as that of the query stream.
            sort_order = query_stream.sort_order()
        else:
            # No sort order can be inferred; using empty.
            sort_order = SortOrder()

        self._output_stream = Stream(
            output_schema, 
            sort_order, 
            'DATA ACCESSOR'
        )

    def output(self):
        '''
        Returns the output stream for the given data accessor.
        '''
        return self._output_stream;

    def run(self):
        '''
        The main loop of the data accessor mini-engine. It processes every
        element from the query stream and performs a corresponding query
        against the configured data source.
        '''
        # for each query in the query stream's end-point
        i = 0
        closed = False
        while not closed:
            try:
                # print 'DA: Waiting to receive'
                q = self._query_stream_ep.receive()
                i += 1
                if type(q) is StopWord:
                    continue
                if q:
                    # process the query and write result records into the output
                    # stream
                    for r in self._accessor.query(self._query_stream.schema(), q):
                        self._output_stream.send(r)
                    # finalize the partition that belongs to one query with a
                    # stop word
                    self._output_stream.send(StopWord())
                else:
                    # Technically stop words are not allowed in the query
                    # stream, but they are silently ignored here.
                    pass
                self._query_stream_ep.processed()
            except StreamClosedException:
                closed = True
        self._output_stream.close()
        print 'Closing DA stream'

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import send [as 别名]
class Mux(MiniEngine):
    def __init__(self, *streams):
        MiniEngine.__init__(self)
        if not streams:
            raise Exception('Mux: must specify at least one stream.')
        self._streams = streams
        self._queue = Queue(100)
        self._stats = {}
        self._endpoints = dict([(s.connect(), s) for s in self._streams])
        for e in self._endpoints:
            self._stats[e] = 0
            e.notify(self._queue)

        for s in self._streams:
            if s.schema() != self._streams[0].schema():
                raise Exception('Mux: schema of streams must match.')

        self._output = Stream(
            self._streams[0].schema(),
            SortOrder(),
            'Mux Output'
        )

    def output(self):
        return self._output

    def run(self):
        while self._endpoints or not self._queue.empty():
            # print '\t\twaiting for endpoint'
            e = self._queue.get()
            
            if e not in self._endpoints:
                print '\t********* got non-existing endpoint'
                continue

            valid = True
            closed = False
            while valid and not closed:
                # print '\t\t%s: receiving' % (self._endpoints[e])
                try:
                    r = e.receive(False)
                    self._stats[e] += 1
                    self._output.send(r)
                    e.processed()
                except StreamClosedException:
                    # print '\t\tReceive ClosedException.'
                    closed = True
                except:
                    valid = False
                    # print '\t\tReceive failed.'
            else:
                if e in self._endpoints:
                    # print '%s: closed? %s' % (self._endpoints[e], e.closed())
                    if closed:
                        # print '%s: closed? %s' % (self._endpoints[e], e.closed())
                        del self._endpoints[e]
                else:
                    # print '%s: already removed' % (e)
                    pass
            self._queue.task_done()
        #print '\t\tAll streams done.'
        self._output.close()
        # for e in self._stats:
        #    print 'Received %d records from %s' % (self._stats[e], e)
        print 'Mux: done.'

# 需要导入模块: from stream import Stream [as 别名]
# 或者: from stream.Stream import send [as 别名]
class Tunnel(object):
    _TCP_INITIAL_DATA = 0
    _TCP_FIN_DATA = 1
    _TCP_CLOSED_DATA = 2
    _UDP_INITIAL_DATA = 3
    _UDP_CLOSED_DATA = 4
    _TUN_INITIAL_DATA = 5
    _PAYLOAD = 10
    _HEARTBEAT = 100

    _static_handlers = {
        _HEARTBEAT: (lambda _, __, ___: None)
    }

    @staticmethod
    def set_tcp_initial_handler(handler):
        Tunnel._static_handlers[Tunnel._TCP_INITIAL_DATA] = handler

    @staticmethod
    def set_tcp_fin_received_handler(handler):
        Tunnel._static_handlers[Tunnel._TCP_FIN_DATA] = handler

    @staticmethod
    def set_tcp_closed_handler(handler):
        Tunnel._static_handlers[Tunnel._TCP_CLOSED_DATA] = handler

    @staticmethod
    def set_udp_initial_handler(handler):
        Tunnel._static_handlers[Tunnel._UDP_INITIAL_DATA] = handler

    @staticmethod
    def set_udp_closed_handler(handler):
        Tunnel._static_handlers[Tunnel._UDP_CLOSED_DATA] = handler

    @staticmethod
    def set_tun_initial_handler(handler):
        Tunnel._static_handlers[Tunnel._TUN_INITIAL_DATA] = handler

    def __init__(self, connection=None, connect_to=None):
        self._stream = connection
        self._connect_to = connect_to
        self._on_initial_data = None
        self._on_payload = None
        self._on_stream_closed = None
        self._handlers = self._static_handlers.copy()
        self._handlers.update({
            Tunnel._PAYLOAD: lambda _, id_, data: self._on_payload(self, id_, data)
        })
        self._on_ready_to_send = None
        self._on_send_buffer_full = None
        self._hb_event = None
        self.connections = {}

    def __hash__(self):
        return hash(self._stream)

    def __eq__(self, other):
        if not isinstance(other, Tunnel):
            return False
        return self._stream == other._stream

    def __str__(self):
        return str(self._stream)

    def _send_heartbeat(self):
        self._send_content(Tunnel._HEARTBEAT, None, None)
        self._enable_heartbeat()

    def _enable_heartbeat(self):
        self._hb_event = Event.add_timer(HEARTBEAT_INTERVAL)
        self._hb_event.set_handler(lambda ev: self._send_heartbeat())

    def _disable_heartbeat(self):
        if self._hb_event is not None:
            self._hb_event.del_timer()
            self._hb_event = None

    def _on_fin_received(self):
        self._disable_heartbeat()
        self._stream.close()

    def initialize(self):
        if self._stream is None:
            self._stream = Stream(prefix='TUNNEL')

        # self._stream.set_buffer_size(BUFF_SIZE)
        self._stream.set_tcp_no_delay()

        self._stream.append_send_handler(obscure.pack_data)
        self._stream.append_send_handler(obscure.random_padding)
        # self._stream.append_send_handler(obscure.gen_aes_encrypt())
        self._stream.append_send_handler(obscure.gen_xor_encrypt())
        # self._stream.append_send_handler(obscure.base64_encode)
        self._stream.append_send_handler(obscure.gen_http_encode(self._connect_to is not None))
        self._stream.append_receive_handler(obscure.gen_http_decode(self._connect_to is not None))
        # self._stream.append_receive_handler(obscure.base64_decode)
        self._stream.append_receive_handler(obscure.gen_xor_decrypt())
        # self._stream.append_receive_handler(obscure.gen_aes_decrypt())
        self._stream.append_receive_handler(obscure.unpad_random)
        self._stream.append_receive_handler(obscure.unpack_data)
#.........这里部分代码省略.........