Python Store.get方法代码示例

# 需要导入模块: from store import Store [as 别名]
# 或者: from store.Store import get [as 别名]
    def GET(self,id1,id2,id3,id4,type):
#		image_url_regex = r'/([a-z0-9]{2})/([a-z0-9]{2})/([a-z0-9]{19,36})(-[sc]\d{2,4})?\.(gif|jpg|jpeg|png)$'
		id = '{0}{1}{2}'.format(id1,id2,id3)
		from store import Store
		store = Store()
		file = store.get(id)
		if file is None:
		    store.close()
		    return render.error("not found",'/')

		org_path = '{0}/{1}/{2}.{4}'.format(id1,id2,id3,id4,type)
		org_file = '{0}/{1}'.format(THUMB_ROOT, org_path)
		if not os.path.exists(org_file):
		    save_file(file, org_file)
		if id4 is None:
		    dst_path = org_path
		    dst_file = org_file
		else:
		    dst_path = '{0}/{1}/{2}{3}.{4}'.format(id1,id2,id3,id4,type)
		    dst_file = '{0}/{1}'.format(THUMB_ROOT, dst_path)
		    #print(ids[3][1:])
		    size = int(id4[2:])
		    if size not in SUPPORTED_SIZE:
		        print('unsupported size: {0}'.format(size))
		        store.close()
		        return render.error("not found",'/')
		    thumb_image(org_file, size, dst_file)
#		print(org_file)
#		print(dst_file)
#		print web.ctx.env
		server_soft = web.ctx.env['SERVER_SOFTWARE']
#		print server_soft
		if server_soft[:5] == 'nginx' and os.name != 'nt':
			print("in")
			store.close()
			#start_response('200 OK', [('X-Accel-Redirect', '{0}/{1}'.format(THUMB_PATH, dst_path))])
			web.header('X-Accel-Redirect', '{0}/{1}'.format(THUMB_PATH, dst_path))
			return ;
		
#		print(file.type) 
		web.header('Content-Type',  str(file.type))
		web.header('Content-Length', '{0.length}'.format(file))
		web.header('Via','store')
		#print(headers)
		
		# TODO: response file content
		distfile = open(dst_file, 'rb')
		data = distfile.read()
		store.close()
		return data; #200OK
		#return [data]
		
		#fd = open(dst_file,'r')
		#return environ['wsgi.file_wrapper'](fd, 4096)
		return render.error("not found",'/')

# 需要导入模块: from store import Store [as 别名]
# 或者: from store.Store import get [as 别名]
#

# Measure time to insert
avg_time = 0
for k in keys:
    start = time.time()
    store.put(k, k)
    end = time.time()
    avg_time += (end - start)
print("Time Per Allocaiton (s): " + str(avg_time / store.count))

# Measure time to read
avg_time = 0
for k in keys:
    start = time.time()
    tmp = store.get(k)
    end = time.time()
    avg_time += (end - start)
print("Time Per Read (s): " + str(avg_time / store.count))

# Measure time to remove
avg_time = 0
for k in keys:
    start = time.time()
    store.delete(k)
    end = time.time()
    avg_time += (end - start)
print("Time Per Delete (s): " + str(avg_time / len(keys)))

#
# Testing API service

# 需要导入模块: from store import Store [as 别名]
# 或者: from store.Store import get [as 别名]
class NodeServer(object):
    def __init__(self, node_id, store_file=None, explicit_configuration=None, coordinator_addresses=[], var_directory='var',
                 configuration_update_interval=300):
        super(NodeServer, self).__init__()
        if __debug__: logging.debug('node_id = %s, store_file = %s, explicit_configuration = %s, coordinators = %s, var_directory = %s',
                  node_id, store_file, explicit_configuration, coordinator_addresses, var_directory)
        self.id = node_id
        self.node = None
        var_directory = paths.path(var_directory)
        store_file = store_file or os.path.join(var_directory, 'data', '%s.tcb' % self.id)
        self.__store = Store(store_file)
        self.__store.open()
        self.__node_clients = {}
        self.__internal_cluster_client = service.MulticastClient(InternalNodeServiceProtocol())
        configuration_directory = os.path.join(var_directory, 'etc')
        self.__repair_task = None
        self.__background_repair_enabled = False
        self.__background_repair_interval_seconds = None
        self.__read_repair_enabled = False
        self.__configuration = None
        self.__configuration_controller = ConfigurationController('nodeserver-%s' % self.id,
                                                                  coordinator_addresses, explicit_configuration,
                                                                  configuration_directory, self.__update_configuration,
                                                                  configuration_update_interval)


    def __initialize_node_client_pool(self):
        nodes = self.__configuration.find_neighbour_nodes_for_node(self.node) if self.node else {}

        recycled_node_clients = {}
        for node_id, client in self.__node_clients.iteritems():
            if node_id in nodes:
                recycled_node_clients[node_id] = client
            else:
                client.disconnect()

        new_node_clients = {}
        for node_id, node in nodes.iteritems():
            if node_id not in recycled_node_clients:
                new_node_clients[node_id] = service.Client((node.address, node.port), InternalNodeServiceProtocol, tag=node_id)

        self.__node_clients = dict(recycled_node_clients, **new_node_clients)

        # Blocking, do this after setting self.__node_clients
        for client in new_node_clients.itervalues():
            client.connect()

    def __update_configuration(self, new_configuration):
        if __debug__: logging.debug('New configuration: %s', new_configuration)
        self.__configuration = new_configuration
        deployment = None
        if self.id in new_configuration.active_deployment.nodes:
            deployment = new_configuration.active_deployment
        if new_configuration.target_deployment and self.id in new_configuration.target_deployment.nodes:
            deployment = new_configuration.target_deployment
        self.node = deployment.nodes.get(self.id, None) if deployment else None
        self.__read_repair_enabled = deployment.read_repair_enabled if deployment else False
        self.__background_repair_enabled = deployment.background_repair_enabled if deployment else False
        self.__background_repair_interval_seconds = deployment.background_repair_interval_seconds if deployment else 0
        self.__initialize_node_client_pool()
        # TODO: restart servers if addresses changed

    def __fetch_value(self, key, node_id):
        if __debug__: logging.debug('key: %s, node_id: %s', key, node_id)
        return self.__clients_for_nodes((node_id,))[0].get(key) or (None, None)

    def __fetch_timestamps(self, key, node_ids):
        if __debug__: logging.debug('key: %s', key)
        node_ids = dict(node_ids)
        node_ids.pop(self.node.id, None)
        if not node_ids:
            return []
        clients = self.__clients_for_nodes(node_ids)
        return self.__internal_cluster_client.stat(clients, key)

    def __clients_for_nodes(self, node_ids):
        return [self.__node_clients[node_id] for node_id in node_ids]

    def __propagate(self, key, timestamp, value, target_nodes):
        if __debug__: logging.debug('key: %s, target_nodes: %s', key, target_nodes)
        collector = self.__internal_cluster_client.set_collector(self.__clients_for_nodes(target_nodes), 1)
        self.__internal_cluster_client.set_async(collector, key, timestamp, value)

    def __read_repair(self, key, timestamp, value, node_ids):
        if __debug__: logging.debug('key: %s, timestamp: %s', key, timestamp)
        remote_timestamps = self.__fetch_timestamps(key, node_ids)
        if __debug__: logging.debug('remote: %s', [(client.tag, repr(remote_timestamp)) for client, remote_timestamp in remote_timestamps])
        newer = [(client, remote_timestamp) for client, remote_timestamp in remote_timestamps
                 if remote_timestamp and remote_timestamp > timestamp]

        if __debug__: logging.debug('newer: %s', [(client.tag, repr(remote_timestamp)) for client, remote_timestamp in newer])
        if newer:
            latest_client, latest_timestamp = newer[-1]
            latest_timestamp, latest_value = self.__fetch_value(key, latest_client.tag)
            if __debug__: logging.debug('latest_timestamp: %s', latest_timestamp)
            if latest_timestamp and latest_value:
                value = latest_value
                timestamp = latest_timestamp

        older = [(client, remote_timestamp) for client, remote_timestamp in remote_timestamps
#.........这里部分代码省略.........

# 需要导入模块: from store import Store [as 别名]
# 或者: from store.Store import get [as 别名]
def main(argv=None):
	if argv is None:
		argv = sys.argv
	
	try:
		try:
			opts, args = getopt.getopt(argv[1:], "hi:q:lt:v", ["help", "import=", "id=", "list", "test", "verbose", "limit=", "start="])
		except getopt.error, msg:
			raise Usage(msg)
		
		#print(opts)
		#print(args)
		action = None
		store_file = None
		# option processing
		for option, value in opts:
			if option == "-v":
				verbose = True
			if option in ("-h", "--help"):
				raise Usage(help_message)
			if option in ("-i", "--import"):
				store_file = value
				print('store file: {0}'.format(store_file))
				action = 'import'
			elif option in ("-l", "--list"):
				action = 'list'
			elif option in ("-t", "--test"):
				action = 'test'
				filename = value
			elif option in ("-q", "--id"):
				action = 'get'
				id = value
			else:
				pass
		
		print('action: {}'.format(action))
		if (action == 'list'):
			store = Store()
			gallery = store.browse()
			for img in gallery['items']:
				#print(img)
				print("{0[filename]}\t{0[length]:8,d}".format(img))
			return 0
		elif (action == 'get') and id is not None:
			store = Store()
			if not store.getFs().exists(id):
				print ('not found')
				return 1
			gf = store.get(id)
			#print(gf)
			print ("found: {0.name}\t{0.length}".format(gf))
			return 0
		elif (action == 'test'):
			print('filename: %r' % filename)
			fp = open(filename, 'rb')
			h = fp.read(32)
			print(getImageType(h))
			return 0
		elif (action == 'import'):
			print('storing: %r' % store_file)
			fp = open(store_file, 'rb')
			type = "IMAGE"
			data = fp.read()
#			h = fp.read(32)
#			type =  getImageType(h)
#			print(store_file+": "+type )
			store = Store()
			result = store.store(data, type, store_file)
			print result
			return 0

# 需要导入模块: from store import Store [as 别名]
# 或者: from store.Store import get [as 别名]
class KvPaxosServer:
  def __init__(self):
    with open(str(ME) + 'log.txt', 'w') as file:
      file.write('log:\n')
    self.px = Paxos.make(HOSTS_LIST, ME)
    self.http_server = BaseHTTPServer.HTTPServer(
        (HOST, int(PORT)), MyHTTPRequestHandler)
    self.kvstore = Store()
    self.keep_running = True
    self.lock = Lock()
    
    # do not modify these 2 vars outside the execute() function
    # lock required to access these values. # maybe unnecessary 
    self.executed_paxos_no = 0
    # contains all executed operations and their return values
    self.operation_log = []
    self.processed_requestid = dict()
    
  def start(self):
    log("HTTP Server Starts - %s:%s" % (HOST, PORT))
    maintainance_thread = Thread(target=self.maintainance, name='maintainance')
    maintainance_thread.start()
    try:
      while self.keep_running:
      # self.http_server.serve_forever()
        self.http_server.handle_request()
    except KeyboardInterrupt:
      sys.exit(0)

    maintainance_thread.join()
    os._exit(0)
  
  def maintainance(self):
    while self.keep_running:
      while self.px.max() < self.executed_paxos_no and self.keep_running:
        # print 'maintainance sleep'
        time.sleep(MAX_SLEEP_TIME)
      if self.keep_running:
        # print 'maintainance execute', self.px.max(), self.executed_paxos_no
        self.execute(self.executed_paxos_no, None)    
      
  
  ''' this function is only called by the handler class, & the maintainance thread'''
  def execute(self, seq, requestid):
      
    # catch up. this ensures that operations are executed in ascending order
    while self.executed_paxos_no < seq:
      time.sleep(MAX_SLEEP_TIME)

    with self.lock:
      # print 'lock acquired ============================================================'
      # if seq < self.executed_paxos_no:
      #   # the operations is done by other threads already, check the log directly
      #   return operation_log[seq].return_value
      if self.processed_requestid.has_key(requestid):
        assert self.processed_requestid[requestid] < self.executed_paxos_no
        return self.operation_log[self.processed_requestid[requestid]].return_value
      # this request has not been executed yet, or is being executed by maintainance thread.
      while True:
        # since not executed, it cannot be forgotten
        decided, op_jstr = self.px.status(seq)
        if decided:
          break 
        else:
          # print 'waiting for decided value', seq, op_jstr
          time.sleep(MAX_SLEEP_TIME)
      op = Operation.decode(op_jstr)
      assert decided
      if self.processed_requestid.has_key(op.requestid):
        success, value = self.operation_log[self.processed_requestid[op.requestid]].return_value
      else:
        if op.op_type == 'GET':
          success, value = self.kvstore.get(op.key)
        elif op.op_type == 'INSERT':
          success, value = self.kvstore.insert(op.key, op.value)
        elif op.op_type == 'DELETE':
          success, value = self.kvstore.delete(op.key)
        elif op.op_type == 'UPDATE':
          success, value = self.kvstore.update(op.key, op.value)
      self.executed_paxos_no += 1
      # self.px.done(seq)
      op.done((success, value))
      self.operation_log += [op]
      assert (not self.processed_requestid.has_key(op.requestid)) or requestid is None
      if not self.processed_requestid.has_key(op.requestid):
        self.processed_requestid[op.requestid] = seq
      # print self.processed_requestid
        with open(str(ME) + 'log.txt', 'a') as file:
          file.write(op.encode() + '\n')# + str(self.processed_requestid) + '\n')
      return success, value
  
  def handle_shutdown(self):
    self.keep_running = False