Python Packet.getDataNodes方法代码示例

# 需要导入模块: import Packet [as 别名]
# 或者: from Packet import getDataNodes [as 别名]
def copyFromDFS(address, fname, path):
	""" Contact the metadata server to ask for the file blocks of
	    the file fname.  Get the data blocks from the data nodes.
	    Saves the data in path.
	"""

   	# Contact the metadata server to ask for information of fname
	sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	sock.connect(address)


	# Create a packet object to get block ids from meta data
	p = Packet()
	p.BuildGetPacket(fname)
	sock.sendall(p.getEncodedPacket())


	# If there is no error response, retreive the data blocks
	r = sock.recv(1024)
	p.DecodePacket(r)
	dnList = p.getDataNodes()


	# Create file to store data from blocks
	f = open(path, 'wb')


	# Get data blocks from data servers
	for dnode in dnList:
		# Contact the data node
		sockdn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		sockdn.connect((dnode[0], dnode[1]))


		# Create a packet object to get data from data node
		p.BuildGetDataBlockPacket(dnode[2])
		sockdn.sendall(p.getEncodedPacket())

		# Get the data size of the data that will be receive
		dsize = sockdn.recv(1024)
		dsize = int(dsize)

		sockdn.sendall("OK")


		# Get data in 1024 size parts
		data = ""
		while(len(data) < dsize):
			r = sockdn.recv(1024)
			data = data + r
			sockdn.sendall("OK")

		# Write data to file
		f.write(data)
		sockdn.close()


	f.close()

# 需要导入模块: import Packet [as 别名]
# 或者: from Packet import getDataNodes [as 别名]
def copyFromDFS(address, dfs_path, filename, password, crypto):
    """ Contact the metadata server to ask for the file blocks of
        the file fname.  Get the data blocks from the data nodes.
        Saves the data in path.
    """
    # Contact the metadata server to ask for information of fname

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect(address)
    p = Packet()
    p.BuildGetPacket(dfs_path)
    sendall_with_size(sock, p.getEncodedPacket())

    # If there is no error response Retreive the data blocks

    # Save the file

    message = recv_with_size(sock)
    p.DecodePacket(message)
    #getDataNodes has ADDRESS, PORT, BLOCK_ID
    data_nodes = p.getDataNodes()

    """Not needed."""
    #In case the user specifies a diferent directory

    #filename = fname.split("/")

    #destination = "%s/%s" % (path, "copy_" + filename[-1])
    """Not needed."""

    wfile = open(filename, "w")
    for IP, PORT, BLOCK_ID in data_nodes:
        node_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        node_sock.connect((IP, PORT))

        p.BuildGetDataBlockPacket(BLOCK_ID)
        sendall_with_size(node_sock, p.getEncodedPacket())

        block = recv_with_size(node_sock)

        #added for decrypting
        if(crypto):
            decrpt_block = decrypt(password, block)

        else:
            decrpt_block = block

        wfile.write(decrpt_block)

        node_sock.close()

    wfile.close()

# 需要导入模块: import Packet [as 别名]
# 或者: from Packet import getDataNodes [as 别名]
def copyToDFS(address, fname, path):
	""" Contact the metadata server to ask to copu file fname,
	    get a list of data nodes. Open the file in path to read,
	    divide in blocks and send to the data nodes. 
	"""

	# Create a connection to the data server

	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	s.connect(address)

	# Read file
	# here I read the file and store the contents of the file in a variable named data
	# and store the data length to in a variable called dataLength :D

	theFile = open(path, 'rb')
	data = theFile.read()
	theFile.close()
	dataLength = len(data)

	# Create a Put packet with the fname and the length of the data,
	# and sends it to the metadata server 

	packet = Packet()
	packet.BuildPutPacket(fname, dataLength)
	s.sendall(packet.getEncodedPacket())

	# If no error or file exists
	# Get the list of data nodes.
	# Divide the file in blocks
	# Send the blocks to the data servers

	response = s.recv(1024)
	s.close()
	# lets check if the file already exists in the DB
	if response == "DUP":
		# if it does then let the user know
		print "The file already exists"
		return

	# if it isn't in the DB then we get our hands dirty
	else:
		# decode the packet and get the data nodes that the metadata server
		# sent you, also get its size.. this will be usefull later
		packet.DecodePacket(response)
		dataNodes = packet.getDataNodes()
		countDN = len(dataNodes)
		# now we have to divide the file into blocks. To do so we need t know
		# the block size, so that we can divide the file in that size
		blockSize = dataLength/countDN

		# now lets create a list that will hold the data of the file divided
		# representing diferent blocks
		blocks = []

		# if the blicksize is rounded to 0 then we wont divide the file because this means
		# the file is too small and we will send it as a whole
		if blockSize < 1:
			blocks.append(data)
		else:
			# now we can start filling the blocks list with data
			# I'll use range with 3 attributes.. the first one is start
			# from that number... the second one is stop in that number
			# and the third one is count in that interval
			for i in range(0, dataLength, blockSize):
				if len(blocks) + 1 == countDN:
					# in the last iteration we are going to append all as a
					# data block
					blocks.append(data[i:])
				else:
					blocks.append(data[i:i+blockSize])

		# now that we have the blocks list filled with data we can send the blocks
		# to the data nodes
		ctr = 0
		for node in dataNodes:
			# we connect to each datanode in the that the metadata server gave us
			sdn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
			sdn.connect((node[0], node[1]))
			# now we build a put packet that we will send to the datanode server
			packet.BuildPutPacket(fname, dataLength)
			sdn.sendall(packet.getEncodedPacket())

			#now we wait for the response of the server
			response = sdn.recv(1024)

			if response == "OK":
				block = blocks[ctr]
				blockSize = len(block)

				sdn.sendall(str(blockSize))
				response = sdn.recv(1024)

				# we have to send the block in chunks of data because the buffer size
				# is 1024 so that is what this while loop is for
				while len(block):
					# get a chunk
					dataChunk = block[0:1024]
					# send that chunk
					sdn.sendall(dataChunk)
#.........这里部分代码省略.........

# 需要导入模块: import Packet [as 别名]
# 或者: from Packet import getDataNodes [as 别名]
def copyFromDFS(address, fname, path):
	""" Contact the metadata server to ask for the file blocks of
	    the file fname.  Get the data blocks from the data nodes.
	    Saves the data in path.
	"""

   	# Contact the metadata server to ask for information of fname
   	# we first connect to the metadata server and build a get packet
   	# with the name of the file we want to copy
   	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	s.connect(address)
	packet = Packet()
	packet.BuildGetPacket(fname)
	# send the packet to the metadata server
	s.sendall(packet.getEncodedPacket())
	# wait for a response, when you get the response then close the socket
	response = s.recv(1024)
	s.close()

	# If the response is NFOUND is because the metadata server couldnt find
	# the file in the db, so lets notify the user
	if response == "NFOUND":
		print "File not found"
		return
	else:
		# if everything went well with the response then well do the following
		# we will create a file that will contain the data
		theFile = open(path, 'wb')

		# now we will decode the packet and get the list of the datanodes in wich
		# we stored a bloc of the file we want to copy
		packet.DecodePacket(response)
		metaList = packet.getDataNodes()

		# now that we have the datanodes we want to get the block of each datanode
		# so that at the end we can have a copy of the whole file
		for dataNode in metaList:
			# we connect to each datanode in the that the metadata server gave us
			sdn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
			sdn.connect((dataNode[0], dataNode[1]))

			# now we have to create something to send to de datanode
			# so that it knows what i want from it.. so lets build a 
			# Get Data Block Packet and send the block id that is in the
			# third position in the dataNode list
			packet.BuildGetDataBlockPacket(dataNode[2])
			sdn.sendall(packet.getEncodedPacket())
			# now we wait for the datanode to send us the size of the block
			blockSize = sdn.recv(1024)

			# this part is very important.. this gave us a lot of problems
			# because we didnt have the next sendall and if you dont have that
			# you might screw up things when you recieve the size you HAVE TO respond
			# that you recieved because if you don't then the datanode server will start
			# sending you chunks of the block and you might recieve the wrong data.. this
			# happened to us and we got a chun of the data block as a datasize and the DFS
			# exploded :( .. but we fixed it with this sendall right here :D
			sdn.sendall("recieved the size")

			# now we start to create the copy file.. we start to recieve the chunks
			blockData = ""
			blockData = sdn.recv(1024)
			sdn.sendall("Recieved first chunk")
			# while the length of the copy block isnt the same as the block in the datanode
			# then we will contiue recieving chunks and merging them with the previous chunk
			while len(blockData) < int(blockSize):
				blockData += sdn.recv(1024)
				# dont forget to let the datanode server that you recieved the chunk
				sdn.sendall("recieved another chunk")

			# once I'm out of the while, this means that we have the full block of data
			# and now we have to write this data in the file created at the beguining
			theFile.write(blockData)
			# dont forget to close the socket connection
			sdn.close()
		# when you get here that means that you already have all blocks and you have
		# successfuly created a copy file
		theFile.close()

# 需要导入模块: import Packet [as 别名]
# 或者: from Packet import getDataNodes [as 别名]
	contents = f.read()
	fsize = len(contents)

	# Create a Put packet with the fname and the length of the data,
	# and sends it to the metadata server 
	try:
		response = "DUP"
		sp = Packet()
		while response == "DUP":
			sp.BuildPutPacket(fname, fsize)
			sock.sendall(sp.getEncodedPacket())
			response = sock.recv(1024)

			if response != "DUP":
			 	sp.DecodePacket(response)
			 	dnservers = sp.getDataNodes()
			if response == "DUP":
				print "File exists or error."
				sys.exit(1)
	finally:
		sock.close()

	# If no error or file exists
	# Get the list of data nodes.
	# Divide the file in blocks
	# Send the blocks to the data servers

	# Fill code	
	blocksize = fsize/len(dnservers)
	extra = fsize%len(dnservers)
	blocks = []

# 需要导入模块: import Packet [as 别名]
# 或者: from Packet import getDataNodes [as 别名]
def copyToDFS(address, dfs_path, filename, password, crypto):
    """ Contact the metadata server to ask to copy file fname,
        get a list of data nodes. Open the file in path to read,
        divide in blocks and send to the data nodes.
    """

    # Create a connection to the data server

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect(address)

    #get filesize

    filesize = os.path.getsize(filename)

    # Create a Put packet with the filename and the length of the data,
    # and sends it to the metadata server

    p = Packet()
    p.BuildPutPacket(dfs_path, filesize)
    sendall_with_size(sock, p.getEncodedPacket())

    # If no error or file exists
    # Get the list of data nodes.
    # Divide the file in blocks
    # Send the blocks to the data servers

    message = recv_with_size(sock)
    #print message
    p.DecodePacket(message)

    #Packet.getDataNodes() returns a list of elements
    #They are of the form (address, port)
    data_nodes = p.getDataNodes()
    node_amount = len(data_nodes)

    #for distributing the workload
    if(filesize / node_amount == 0):
        partition_size = filesize
    else:
        partition_size = filesize / node_amount

    #blocks of about 4K
    block_size = 2 ** 12
    if(filesize > 40 * (10 ** 6)):
        #blocks of size about 4096K
        block_size = 2 ** 22

    blocks = [] #for the metadata server

    rfile = open(filename, "r")
    for IP, PORT in data_nodes:
        temp_load = partition_size
        while(temp_load):
            node_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            node_sock.connect((IP, PORT))

            if(temp_load < block_size):
                segment = rfile.read(temp_load)
                temp_load = 0
            else:
                segment = rfile.read(block_size)
                temp_load -= block_size

            #variable from the main function to indicate if crypto is needed
            if(crypto):
                crpt_seg = encrypt(password, segment)
            else:
                crpt_seg = segment

            #sending a put message to the data node
            p.BuildPutPacket(dfs_path, len(crpt_seg))
            sendall_with_size(node_sock, p.getEncodedPacket())

            #the OK message
            OK = recv_with_size(node_sock)

            #sends the block to the data node
            sendall_with_size(node_sock, crpt_seg)

            #receive the unique block ID
            blockid = recv_with_size(node_sock)

            #adding muh blocks
            blocks.append((IP, str(PORT), blockid))

            node_sock.close()

    #not everything was cleared, the last node gets the load
    #almost the same code as above
    if(filesize % node_amount != 0):
        while(1):
            node_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            node_sock.connect((data_nodes[-1][0], data_nodes[-1][1]))

            segment = rfile.read(block_size)
            #encountered an end of file
            if(segment == ""):
                break

#.........这里部分代码省略.........

# 需要导入模块: import Packet [as 别名]
# 或者: from Packet import getDataNodes [as 别名]
def copyToDFS(address, fname, path):
	""" Contact the metadata server to ask to copy file fname,
	    get a list of data nodes. Open the file in path to read,
	    divide in blocks and send to the data nodes. 
	"""

	# Create a connection to the data server
	sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

	# Read file
	fp = open(fname, 'rb')
	fsize = os.path.getsize(fname)

	# Create a Put packet with the fname and the length of the data,
	# and sends it to the metadata server 
	p = Packet()

	packet = p.BuildPutPacket(path+fname, fsize)

	sock.connect(address)
	sock.sendall(p.getEncodedPacket())

	msg = sock.recv(1024)

	# If no error or file exists
	# Get the list of data nodes.
	# Divide the file in blocks
	# Send the blocks to the data servers
	if msg == "DUP":
		print "The file '%s' is already in the file system." % fname
		return
	elif msg == "OK":
		p.RefreshPacket()
		p.BuildGetPacket(fname)
		sock.sendall(p.getEncodedPacket())
		p.RefreshPacket()

		msg = sock.recv(1024)
		sock.close()

		if msg == "NFOUND":
			print "The file '%s' is not in the file system." % fname
			return
		else:
			p.DecodePacket(msg)
			metalist = p.getDataNodes()
			blocks = []

			n = len(metalist)

			if n == 0:
				print "There was an error getting the list of data nodes."
				return
			else:
				i = 0
				for block in fileblocks(n, fsize, fp):
					# send the block to the data node
					p.RefreshPacket()
					p.BuildPutPacket(fname, fsize)

					node = metalist[i]
					sock.connect(node[0], node[1])
					sock.sendall(p.getEncodedPacket())

					msg = sock.recv(1024)

					if msg != "OK":
						print "There was an error saving a block to the data node."
						return
					else:
						# send block, receive id and save it to forward it to metadata server
						p.RefreshPacket()
						p.DecodePacket(msg)
						

					i = (i+1)%n

	# Notify the metadata server where the blocks are saved.

	# Fill code

	sock.close()

# 需要导入模块: import Packet [as 别名]
# 或者: from Packet import getDataNodes [as 别名]
def copyToDFS(address, fname, path):
	""" Contact the metadata server to ask to copy file fname,
	    get a list of data nodes. Open the file in path to read,
	    divide in blocks and send to the data nodes. 
	"""

	# Create a connection to the data server
	sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	sock.connect(address)


	# Read file
	f = open(path, 'rb')
	fdata = f.read()
	f.close()

	# Get the size of the file
	fsize = len(fdata)

	# Create a Put packet with the fname and the length of the data,
	# and sends it to the metadata server 
	p = Packet()

	p.BuildPutPacket(fname, fsize)
	sock.sendall(p.getEncodedPacket())

	# If no error or file exists
	r = sock.recv(1024)
	sock.close()

	if r == "DUP":
		print "Duplicate File"
		return

	# Get the list of data nodes.
	else:
		p.DecodePacket(r)
		dnodes = p.getDataNodes()

	# Divide the file in blocks
		blist = []
		dnsize = len(dnodes)
		bsize = (fsize / dnsize)

		for i in range(0, fsize, bsize):
			if (i / bsize) + 1 == dnsize:
				blist.append(fdata[i:])
				break

			else:
				blist.append(fdata[i:i + bsize])


	# Send the blocks to the data servers
	for i in dnodes:
		# Connecto to the data node
		sockdn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		sockdn.connect((i[0], i[1]))

		# Create a packet object to put data
		p.BuildPutPacket(fname, fsize)
		sockdn.sendall(p.getEncodedPacket())
		r = sockdn.recv(1024)

		# Take the data block that will be send to data node
		data = blist.pop(0)


		if r == "OK":
			size = len(data)

			sockdn.sendall(str(size))
			r = sockdn.recv(1024)

			# Send the data block into 1024 size parts
			while len(data) > 0:
				chunk = data[0:1024]
				data = data[1024:]

				sockdn.sendall(chunk)
				r = sockdn.recv(1024)

			#Adding the chunk id to the data nodes list
			sockdn.sendall("OK")
			r = sockdn.recv(1024)
			i.append(r)

		sockdn.close()


	# Connect to the meta data
	sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	sock.connect(address)

	# Create a packet object to register data blocks into meta data
	p.BuildDataBlockPacket(fname, dnodes)

	sock.sendall(p.getEncodedPacket())
	sock.close()

# 需要导入模块: import Packet [as 别名]
# 或者: from Packet import getDataNodes [as 别名]
def copyToDFS(address, fname, path):
	""" Contact the metadata server to ask to copu file fname,
	    get a list of data nodes. Open the file in path to read,
	    divide in blocks and send to the data nodes. 
	"""
	#----------------------
	# Create a connection to the data server
	sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	try: 
		sock.connect(address)
	except:
		print 'Could not connect to meta-data server! Are you sure you have the right address?'

	# Read file
	f = open(path, 'rb')
	file_data = f.read()
	f.close()
	
	# Create a Put packet with the fname and the length of the data, and sends it to the metadata server 
	#-----------------------
	p = Packet()
	p.BuildPutPacket(fname, len(file_data))
	sock.sendall(p.getEncodedPacket())

	nodes = sock.recv(1024)

	sock.close()
	#-----------------------

	#-----Data node Information--------
	if nodes == "DUP":
		print "Duplicate error. This file already exists!"
		return

	# Get the list of data nodes.
	p.DecodePacket(nodes)
	data_nodes = p.getDataNodes()

	if len(data_nodes) == 0:
		print 'No data nodes available!'
		return
	#----------------------------------
	#Information pertaining nodes and file.
	dnode_amount = len(data_nodes) #amount of nodes online atm
	#----------------------------------

	#----Splitting in chunks-----------
	# Divide the file in blocks
	variable = chunkiefier(file_data, dnode_amount) #divides the file into the amount of nodes
	#----------------------------------

	#-------Combination of division and sending-----------
	for i in range(dnode_amount):
		#----------------------------------------
		#Dividing each node block into 1024 blocks & attempting to connect to node
		sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		try:
			sock.connect((data_nodes[i][0], data_nodes[i][1]))
		except: 
			print "Could not connect to node server."
		#----------------------------------------

		#----------------------------------------
		#Packet with information about our file	
		p2 = Packet()
		p2.BuildPutPacket(fname, len(file_data))
	 	sock.sendall(p2.getEncodedPacket())
	 	ack = sock.recv(1024)
	 	#----------------------------------------

	 	#----------------------------------------
	 	#Sending Node the size of the data they will receive
	 	node_size = len(variable[i]) #Amount of data data node is waiting for
	 	sock.send(str(node_size))
	 	ack = sock.recv(1024) #Ensures one send per recv
	 	#----------------------------------------

	 	#----------------------------------------
		sock.send('SENDINGBLOCKSNOW')
		ready = sock.recv(1024)
	 	#----------------------------------------
	 	if ready == 'OKSENDTHEDATA':
	 		obj1 = ARC4.new(str(key)) #Generating encryption object with key
		 	#Sending packets to each node server
			sock.sendall(obj1.encrypt(str(variable[i]))) #Sending the encrypted data through socket.
			acks = sock.recv(1024) #Acknowledge to ensure one send per recv
		else:
			print 'Data node is not ready to receive the data.'
			exit(0)
			print ready
		#----------------------------------------
		
		#----------------------------------------
		#Receiving block id after all data has been sent to data node
		print "Done distributing data to node", i+1, "!"
		sock.send("OK")
		block_id = sock.recv(1024) #Receiving
		data_nodes[i].append(block_id) #Adding the block id to the data node information array
		sock.close() #Closing the socket since we do not need it anymore for that particular data node
		#----------------------------------------
#.........这里部分代码省略.........

# 需要导入模块: import Packet [as 别名]
# 或者: from Packet import getDataNodes [as 别名]
def copyFromDFS(address, fname, path):
	""" Contact the metadata server to ask for the file blocks of
	    the file fname.  Get the data blocks from the data nodes.
	    Saves the data in path.
	"""
	file_size = 0
	#--------------------------------------------
   	# Contact the metadata server to ask for information of fname
	sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	sock.connect(address)
	#--------------------------------------------

	#--------------------------------------------
	# Getting block ids from meta server
	p = Packet()
	p.BuildGetPacket(fname)
	sock.sendall(p.getEncodedPacket())
	#--------------------------------------------

	#--------------------------------------------
	# Receiving the meta server response
	bids = sock.recv(1024)
	sock.close() #We no longer need anything from the meta server
	p.DecodePacket(bids)
	data_nodes = p.getDataNodes()
	#--------------------------------------------

	#--------------------------------------------
	# File to store in incoming data
	f = open(path, 'wb')
	#--------------------------------------------

	#--------------------------------------------
	# Going through each node and receiving their blocks
	for i in data_nodes:

		#--------------------------------------------
		# Connecting to node
		sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		try:
			sock2.connect((i[0], i[1]))
		except: 
			print "Could not connect to datanode", i
		#--------------------------------------------

		#--------------------------------------------	
		# Letting datanode know we want some data.
		p.BuildGetDataBlockPacket(i[2])
		sock2.sendall(p.getEncodedPacket())
		#--------------------------------------------

		#--------------------------------------------
		# First thing received is size of data node will send us
		node_size = sock2.recv(1024)
		node_size = int(node_size)
		sock2.send("OK") #One recv per send
		#--------------------------------------------

		#--------------------------------------------
		# Get data in 1024 size parts
		data = ''
		while(len(data) < node_size):
			block = sock2.recv(node_size)
			data = data + block
		sock2.send("OK")	
		#--------------------------------------------

		#Decrypt file
		obj2 = ARC4.new(str(key))
		data = obj2.decrypt(data)

		# Save the file
		f.write(data)
		sock2.close()
		file_size = file_size + len(data)
	#--------------------------------------------

	f.close()

	print 'Successfully copied', file_size, 'bytes of data.'