Python bintrees.FastRBTree类代码示例

def find_top_k_with_FastRBTree(filename = TDATA, k = 10):
    """
    Profile result:
       
       5 million strings:
       memory consuming: 259 MB
       time consuming: 39.9689998627
       [(753, 'bf'),
        (753, 'qj'),
        (753, 'zb'),
        (753, 'vz'),
        (763, 'ma'),
        (755, 'lx'),
        (779, 'qp'),
        (768, 'bg'),
        (758, 'eq'),
        (767, 'tf')]
    """
    result = []
    t = FastRBTree()
    with open(filename) as f:
        for line in f:
            key = line.strip()
            t[key] = t.setdefault(key, 0) + 1

    # heapq
    for key, val in t.iter_items():
        if len(result) < k:
            heapq.heappush(result, (val, key))
        else:
            heapq.heappushpop(result, (val, key))
    
    return result

    def __init__(self, cache_size_limit, trace, csv_suffix=".csv"):
        self.cache_size_limit = cache_size_limit
        self.cache = {}
        self.hits = 0.0
        self.requests = 0.0
        self.ts_order = ['row', 'hit']
        self.ts_datapoint = {key: None for key in self.ts_order}
        self.ts_datapoint['row'] = 0
        self.ts_file = open("csv/min" + csv_suffix, "w")
        self.ts_writer = csv.writer(self.ts_file)
        self.ts_writer.writerow(self.ts_order)

        self.clairvoyance = FastRBTree()

        self.precog = FastRBTree()
        last_time = time.time()
        for i, page_opcode in enumerate(trace):
            if time.time() > last_time + 0.1:
                last_time = time.time()
                print '1', i, '\r',
            sys.stdout.flush()
            page, _ = page_opcode
            try:
                self.precog[page].append(i)
            except KeyError:
                self.precog[page] = collections.deque()
                self.precog[page].append(i)

            known_max = i
        known_max += 2
        for times in self.precog.values():
            times.append(known_max)
            known_max += 1
        print
        print 'Done loading.'

class Tree(object):
	def __init__(self):
		self.priceTree = FastRBTree()
		self.volume = 0
		self.priceMap = {} # Map from price -> orderList object
		self.orderMap = {} # Order ID to Order object
	def __len__(self):
		return len(self.orderMap)
	def getPrice(self, price):
		return self.priceMap[price]
	def getOrder(self, idNum):
		return self.orderMap[idNum]
	def createPrice(self, price):
		newList = OrderList()
		self.priceTree.insert(price, newList)
		self.priceMap[price] = newList
	def removePrice(self, price):
		self.priceTree.remove(price)
		del self.priceMap[price]
	def priceExists(self, price):
		return price in self.priceMap
	def orderExists(self, idNum):
		return idNum in self.orderMap
	def insertTick(self, tick):
		if tick.price not in self.priceMap:
			self.createPrice(tick.price)
		order = Order(tick, self.priceMap[tick.price])
		self.priceMap[order.price].appendOrder(order)
		self.orderMap[order.idNum] = order
		self.volume += order.qty
	def updateOrder(self, tick):
		order = self.orderMap[tick.idNum]
		originalVolume = order.qty
		if tick.price != order.price:
			# Price changed
			orderList = self.priceMap[order.price]
			orderList.removeOrder(order)
			if len(orderList) == 0:
				removePrice(order.price)
			self.insertTick(tick)
		else:
			# Quantity changed
			order.updateQty(tick.qty,tick.price)
		self.volume += order.qty - originalVolume
	def removeOrderById(self, idNum):
		order = self.orderMap[idNum]
		self.volume -= order.qty
		order.orderList.removeOrder(order)
		if len(order.orderList) == 0:
			self.removePrice(order.price)
		del self.orderMap[idNum]
	def max(self):
		return min(self.priceTree)
	def min(self):
		return max(self.priceTree)

 def __init__(self):
     self.price_tree = FastRBTree()
     self.volume = 0
     self.price_map = {}  # Map from price -> order_list object
     self.order_map = {}  # Order ID to Order object
     self.min_price = None
     self.max_price = None

 def __init__(self):
     self.priceTree = FastRBTree()
     self.volume = 0
     self.priceMap = {}  # Map from price -> orderList object
     self.orderMap = {}  # Order ID to Order object
     self.minPrice = None
     self.maxPrice = None

 def __init__(self, f_price):
     '''
     A representation of a PriceLevel object
     '''
     self.f_price = f_price
     self.i_qty = 0
     self.order_tree = FastRBTree()

class Tree(object):
    def __init__(self):
        self.price_tree = FastRBTree()
        self.price_map = {}
        self.order_map = {}
        self.received_orders = {}

    def receive(self, order_id, size):
        self.received_orders[order_id] = size

    def create_price(self, price):
        new_list = []
        self.price_tree.insert(price, new_list)
        self.price_map[price] = new_list

    def remove_price(self, price):
        self.price_tree.remove(price)
        del self.price_map[price]

    def insert_order(self, order_id, size, price, initial=False):
        if not initial:
            del self.received_orders[order_id]
        if price not in self.price_map:
            self.create_price(price)
        order = {'order_id': order_id, 'size': size, 'price': price, 'price_map': self.price_map[price]}
        self.price_map[price].append(order)
        self.order_map[order_id] = order

    def match(self, maker_order_id, match_size):
        order = self.order_map[maker_order_id]
        original_size = order['size']
        new_size = original_size - match_size
        order['size'] = new_size

    def change(self, order_id, new_size):
        order = self.order_map[order_id]
        order['size'] = new_size

    def remove_order(self, order_id):
        if order_id in self.order_map:
            order = self.order_map[order_id]
            self.price_map[order['price']] = [o for o in self.price_map[order['price']] if o['order_id'] != order_id]
            if not self.price_map[order['price']]:
                self.remove_price(order['price'])
            del self.order_map[order_id]
        else:
            del self.received_orders[order_id]

class Tree(object):
    def __init__(self):
        self.price_tree = FastRBTree()
        self.price_map = {}  # Map from price -> order_list object
        self.order_map = {}  # Order ID to Order object
        self.received_orders = {}

    def receive(self, order_id, size):
        self.received_orders[order_id] = size

    def create_price(self, price):
        new_list = OrderList()
        self.price_tree.insert(price, new_list)
        self.price_map[price] = new_list

    def remove_price(self, price):
        self.price_tree.remove(price)
        del self.price_map[price]

    def insert_order(self, order_id, size, price, initial=False):
        if not initial:
            del self.received_orders[order_id]
        if price not in self.price_map:
            self.create_price(price)
        order = Order(order_id, size, price, self.price_map[price])
        self.price_map[order.price].append_order(order)
        self.order_map[order.order_id] = order

    def match(self, maker_order_id, size):
        order = self.order_map[maker_order_id]
        original_size = order.size
        new_size = original_size - size
        order.update_size(new_size)

    def change(self, order_id, new_size):
        order = self.order_map[order_id]
        order.update_size(new_size)

    def remove_order(self, order_id):
        if order_id in self.order_map:
            order = self.order_map[order_id]
            order.order_list.remove_order(order)
            if len(order.order_list) == 0:
                self.remove_price(order.price)
            del self.order_map[order_id]
        else:
            del self.received_orders[order_id]

    def __init__(self, node_timeout):
        self._logger = logging.getLogger(self.__class__.__name__)

        self._nodes = {}
        self._sessions = {}
        self._sessions_by_owner = {}
        self._keepalive_queue = FastRBTree()
        self._priority_queue = FastRBTree()
        self._node_timeout = node_timeout
        self._culling_timer = runtime.greenpool.spawn(self._cull_dead_nodes)

    def __init__(self, cache_entries_limit, ghost_entries_limit,
                 trace_size_limit, csv_suffix="_mmc.csv"):
        self.full_cache = FastRBTree()
        self.was_hit = None
        self.was_ghost_hit = None
        self.num_hits = 0
        self.num_requests = 0
        self.cache_entries_limit = cache_entries_limit
        self.ghost_entries_limit = ghost_entries_limit
        self.trace_size_limit = trace_size_limit
        self.trace = collections.deque()
        self.stack = RBTree()
        self.ranker = RBTree()
        self.generation = 0
        # During startup, this will act like an LRU.
        self.startup = True
        self.EM_period = 50 * int(np.ceil(np.log(trace_size_limit)))
        self.countdown_to_EM = trace_size_limit // 2
        self.tau = [0.25, 0.25, 0.25, 0.25]
        self.theta = [0.5, 0.5, 0.5, 0.5]
        self.acc_tau = [0.0, 0.0, 0.0, 0.0]
        self.acc_theta = [0.0, 0.0, 0.0, 0.0]
        self.num_in_cache = 0
        self.num_in_full_cache = 0
        self.num_reads = 0
        self.csv_suffix = csv_suffix

        self.ts_order = [
                'row', 'hit', 'ghost_hit',
                'tau_R_SDD', 'tau_R_IRM', 'tau_W_SDD', 'tau_W_IRM',
                'theta_R_SDD', 'theta_R_IRM', 'theta_W_SDD', 'theta_W_IRM',
                'depth', 'rank',
                'Z_R_SDD', 'Z_R_IRM', 'Z_W_SDD', 'Z_W_IRM', 'Z_sum'
            ]
        self.ts_datapoint = {key: None for key in self.ts_order}
        self.ts_datapoint['row'] = 0
        self.ts_file = open("csv/mmc_rw" + self.csv_suffix, "w")
        self.ts_writer = csv.writer(self.ts_file)
        self.ts_writer.writerow(self.ts_order)

        self.evict_order = [
                'row', 'depth', 'rank', 'value', 'opcode']
        self.evict_datapoint = {key: None for key in self.evict_order}
        self.evict_datapoint['row'] = 0
        self.evict_file = open("csv/mmc_rw_evict" + self.csv_suffix, "w")
        self.evict_writer = csv.writer(self.evict_file)
        self.evict_writer.writerow(self.evict_order)

        self.purge_order = ['row', 'depth', 'rank', 'value', 'opcode']
        self.purge_datapoint = {key: None for key in self.purge_order}
        self.purge_datapoint['row'] = 0
        self.purge_file = open("csv/mmc_rw_purge" + self.csv_suffix, "w")
        self.purge_writer = csv.writer(self.purge_file)
        self.purge_writer.writerow(self.purge_order)

    def __init__(self, s_side, fr_data, i_member=None):
        '''
        Initialize a BookSide object. Save all parameters as attributes

        :param s_side: string. BID or ASK
        :param fr_data: ZipExtFile object. data to read
        :param i_member*: integer. Member number to be used as a filter
        '''
        if s_side not in ['BID', 'ASK']:
            raise InvalidTypeException('side should be BID or ASK')
        self.i_member = i_member
        self.s_side = s_side
        self.price_tree = FastRBTree()
        self._i_idx = 0
        self.fr_data = fr_data
        self.parser = parser_data.LineParser(s_side)
        self.d_order_map = {}
        self.last_price = 0.

class OrderTree(object):
    def __init__(self):
        self.price_tree = FastRBTree()
        self.price_map = {} 
        self.min_price = None
        self.max_price = None


    def insert_price(self, price, amount):
        self.price_tree.insert(price, amount)
        self.price_map[price] = amount
        if self.max_price == None or price > self.max_price:
            self.max_price = price
        if self.min_price == None or price < self.min_price:
            self.min_price = price


    def update_price(self, price, amount):
        
        self.price_tree.insert(price, amount) #updates if key exists
        self.price_map[price] = amount
        
    def remove_price(self, price):
        self.price_tree.remove(price)
        del self.price_map[price]

        if self.max_price == price:
            try:
                self.max_price = max(self.price_tree)
            except ValueError:
                self.max_price = None
        if self.min_price == price:
            try:
                self.min_price = min(self.price_tree)
            except ValueError:
                self.min_price = None

    def price_exists(self, price):
        return price in self.price_map

    def max(self):
        return self.max_price

    def min(self):
        return self.min_price

#! /usr/bin/env python
# coding:utf-8

from __future__ import division
import heapq
import bintrees
import random


if __name__ == '__main__':
    from benchmarker import Benchmarker
    from itertools import repeat, izip
    from bintrees import FastRBTree

    # initialize heapq
    h = range(10000)
    heapq.heapify(h)
    # initialize AVLTree
    m = izip(xrange(10000), repeat(True))
    t = FastRBTree(m)

    for bm in Benchmarker(width=20, loop=100000, cycle=3, extra=1):
        for _ in bm.empty():
            pass
        for _ in bm('heapq'):
            heapq.heappop(h)
            heapq.heappush(h, random.randint(-100000, 100000))
        for _ in bm('FastRBTree'):
            t.pop_min()
            t[random.randint(-100000, 100000)] = True

class BookSide(object):
    '''
    A side of the lmit order book representation
    '''
    def __init__(self, s_side):
        '''
        Initialize a BookSide object. Save all parameters as attributes
        :param s_side: string. BID or ASK
        '''
        if s_side not in ['BID', 'ASK']:
            raise InvalidTypeException('side should be BID or ASK')
        self.s_side = s_side
        self.price_tree = FastRBTree()
        self._i_idx = 0
        self.d_order_map = {}
        self.last_price = 0.

    def update(self, d_data):
        '''
        Update the state of the order book given the data pased. Return if the
        message was handle successfully
        :param d_data: dict. data related to a single order
        '''
        # dont process aggresive trades
        if d_data['agressor_indicator'] == 'Agressive':
            return True
        # update the book information
        order_aux = Order(d_data)
        s_status = order_aux['order_status']
        b_sould_update = True
        b_success = True
        # check the order status
        if s_status != 'New':
            try:
                i_old_id = self.d_order_map[order_aux]['main_id']
            except KeyError:
                if s_status == 'Canceled' or s_status == 'Filled':
                    b_sould_update = False
                    s_status = 'Invalid'
                elif s_status == 'Replaced':
                    s_status = 'New'
        # process the message
        if s_status == 'New':
            b_sould_update = self._new_order(order_aux)
        elif s_status != 'Invalid':
            i_old_id = self.d_order_map[order_aux]['main_id']
            f_old_pr = self.d_order_map[order_aux]['price']
            i_old_q = self.d_order_map[order_aux]['qty']
            # hold the last traded price
            if s_status in ['Partially Filled', 'Filled']:
                self.last_price = order_aux['order_price']
            # process message
            if s_status in ['Canceled', 'Expired', 'Filled']:
                b_sould_update = self._canc_expr_filled_order(order_aux,
                                                              i_old_id,
                                                              f_old_pr,
                                                              i_old_q)
                if not b_sould_update:
                    b_success = False
            elif s_status == 'Replaced':
                b_sould_update = self._replaced_order(order_aux,
                                                      i_old_id,
                                                      f_old_pr,
                                                      i_old_q)
            elif s_status == 'Partially Filled':
                b_sould_update = self._partially_filled(order_aux,
                                                        i_old_id,
                                                        f_old_pr,
                                                        i_old_q)
        # remove from order map
        if s_status not in ['New', 'Invalid']:
            self.d_order_map.pop(order_aux)
        # update the order map
        if b_sould_update:
            f_qty = int(order_aux['total_qty_order'])
            self.d_order_map[order_aux] = {}
            self.d_order_map[order_aux]['price'] = d_data['order_price']
            self.d_order_map[order_aux]['order_id'] = order_aux.order_id
            self.d_order_map[order_aux]['qty'] = f_qty
            self.d_order_map[order_aux]['main_id'] = order_aux.main_id

        # return that the update was done
        return True

    def _canc_expr_filled_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
        '''
        Update price_tree when passed canceled, expried or filled orders
        :param order_obj: Order Object. The last order in the file
        :param i_old_id: integer. Old id of the order_obj
        :param f_old_pr: float. Old price of the order_obj
        :param i_old_q: integer. Old qty of the order_obj
        '''
        this_price = self.price_tree.get(f_old_pr)
        if this_price.delete(i_old_id, i_old_q):
            self.price_tree.remove(f_old_pr)
        # remove from order map
        return False

    def _replaced_order(self, order_obj, i_old_id, f_old_pr, i_old_q):
        '''
#.........这里部分代码省略.........

class MMCPolicy(object):
    def __init__(self, cache_entries_limit, ghost_entries_limit,
                 trace_size_limit, csv_suffix="_mmc.csv", draw_dump=False):
        self.full_cache = FastRBTree()
        self.was_hit = None
        self.was_ghost_hit = None
        self.num_hits = 0
        self.num_requests = 0
        self.cache_entries_limit = cache_entries_limit
        self.ghost_entries_limit = ghost_entries_limit
        self.trace_size_limit = trace_size_limit
        self.trace = collections.deque()
        self.stack = RBTree()
        self.ranker = RBTree()
        self.generation = 0
        # During startup, this will act like an LRU.
        self.startup = True
        self.EM_period = 50 * int(np.ceil(np.log(trace_size_limit)))
        self.countdown_to_EM = trace_size_limit // 2
        self.tau = [0.5, 0.5]
        self.theta = [0.5, 0.5]
        self.acc_tau = [0.0]
        self.acc_theta = [0.0, 0.0]
        self.num_in_cache = 0
        self.num_in_full_cache = 0
        self.csv_suffix = csv_suffix
        self.draw_dump = draw_dump

        self.ts_order = [
                'row', 'hit', 'ghost_hit', 'tau',
                'theta0', 'theta1', 'Z', 'depth', 'rank']
        self.ts_datapoint = {key: None for key in self.ts_order}
        self.ts_datapoint['row'] = 0
        self.ts_file = open("csv/mmc" + self.csv_suffix, "w")
        self.ts_writer = csv.writer(self.ts_file)
        self.ts_writer.writerow(self.ts_order)

        self.evict_order = [
                'row', 'depth', 'rank', 'value', 'Z', 'tau']
        self.evict_datapoint = {key: None for key in self.evict_order}
        self.evict_datapoint['row'] = 0
        self.evict_file = open("csv/mmc_evict" + self.csv_suffix, "w")
        self.evict_writer = csv.writer(self.evict_file)
        self.evict_writer.writerow(self.evict_order)

        self.purge_order = ['row', 'depth', 'rank', 'value', 'Z']
        self.purge_datapoint = {key: None for key in self.purge_order}
        self.purge_datapoint['row'] = 0
        self.purge_file = open("csv/mmc_purge" + self.csv_suffix, "w")
        self.purge_writer = csv.writer(self.purge_file)
        self.purge_writer.writerow(self.purge_order)

    def request(self, page):
        self.num_requests += 1
        self.was_hit = False
        self.was_ghost_hit = False
        node = self.get_node(page)
        if node:
            self.was_ghost_hit = True
            if not node.is_evicted:
                self.num_hits += 1
                self.was_hit = True
            node.hit_count += 1.0 - self.calculate_Z(node.depth, node.rank)
        else:
            node = Node(self)
            node.hit_count = self.tau[1]
            node.page_key = page
            self.full_cache[page] = node

        if not self.was_hit:
            self.num_in_cache += 1
        if not self.was_ghost_hit:
            self.num_in_full_cache += 1

        node.is_evicted = node.is_purged = False
        record = Record(self, node)
        self.add_trace_record(record)

        if len(self.trace) > self.trace_size_limit:
            popped_record = self.trace.popleft()
            self.update_tau_and_theta_accs(record, increment=True)
            self.update_tau_and_theta_accs(popped_record, increment=False)
            self.refresh_params()
            popped_record.node.hit_count -= 1.0 - popped_record.Z

        node.restack()
        node.rerank()

        self.countdown_to_EM -= 1
        if self.countdown_to_EM == 0:
            self.EM_algorithm(delta=0.00001)
            self.countdown_to_EM = self.EM_period
            self.startup = False

        if (
          self.num_in_cache > self.cache_entries_limit or
          self.num_in_full_cache >
          self.cache_entries_limit + self.ghost_entries_limit
        ):
            self.pageout()
#.........这里部分代码省略.........