Python Client.rename方法代码示例

# 需要导入模块: from snakebite.client import Client [as 别名]
# 或者: from snakebite.client.Client import rename [as 别名]
#!/usr/bin/env python

import os
from snakebite.client import Client


client = Client("trevally.amer.nevint.com", 9000, use_trash=False, effective_user='hadoop')


#for res in client.mkdir(['/user/hadoop/test/move/file'],create_parent=True, mode=755):
#    print res

for res in client.rename(['/user/hadoop/test.tar'],'/user/hadoop/test3.tar'):
    print res

# 需要导入模块: from snakebite.client import Client [as 别名]
# 或者: from snakebite.client.Client import rename [as 别名]

#.........这里部分代码省略.........
			while not self.day_in_topic(day) and parse(day) <= parse(date_to):
				print "WARNING: %s not found in %s, trying next day" % (day,self.topic)
				day=datetime.strftime((parse(day)+timedelta(days=1)), "%Y-%m-%d" )

			day=self.day_in_topic(day)
			if not day:
				return	
				
		return True

	#
	# Given a date, if there are files that are not .snappy download and remove them, then getmerge, and upload everything
	#
	def merge_day(self,date):
		print "INFO: processing ", date
		daytmp="%s/snappymerge-%s-tmp" % (self.hdfs_tmp,date)
		daypath=["%s/%s/%s/" % (self.base, self.topic,date)]
		#mergedfile="./%s-merged.snappy" % (date)
		mergedfile="./%s-merged.snappy" % (datetime.strftime(datetime.now(),"%Y-%d-%m.%f"))
		day_files=[x['path'] for x in self.client.ls(daypath)]
		print "INFO: DAYPATH: ", daypath
		try:
			os.remove(mergedfile)
		except:
			pass

		
		if len([ x for x in day_files if x.endswith('.snappy') ]) <= 1:
			print "WARNING: %s does not have enough files to getmerge, skipping" % (date)
			return

		if [ file for file in day_files if not file.endswith('.snappy') ]:
				print "WARNING: %s contains a non snappy file (%s), moving *snappy to %s getmerge there\n" % (daypath,file,daytmp)
				self.merge_with_move(daypath[0],daytmp,day_files,mergedfile)
		else:
			print "INFO: MERGING ", daypath[0]
			result=self.client.getmerge(daypath[0],mergedfile)
			print [x for x in result if not x['result']]
		
			print "INFO: DELETING original files in ", daypath[0]
			for file in day_files:
				print "INFO: Deleting original file ", file
				self.hdfsclient.delete(file)

			print "INFO: UPLOADING merged (%s) to %s" % (mergedfile,daypath[0])
			self.hdfsclient.upload(daypath[0],mergedfile,overwrite=True)
			os.remove(mergedfile)

		return

#
# When there are files that do not contain .snappy suffix, merge with move, first moves everyting to an hdfs temp dir, merges there, and uploads
#
	def merge_with_move(self,day_path,day_tmp,dayfiles,merged_file):
		self.hdfsclient.makedirs(day_tmp)


		print "INFO: MOVING files to ", day_tmp
		snap = [x for x in dayfiles if x.endswith(".snappy")]
		result=self.client.rename(snap,day_tmp)
		print [ x['path'] for x in result if not x['result']]

		print "INFO: MERGING files in ", day_tmp
		result=self.client.getmerge(day_tmp,merged_file)
		print [x['path'] for x in result if not x['result']]

		print "INFO: UPLOADING merged (%s) to %s"  % (merged_file,day_path)
		self.hdfsclient.upload(day_path,merged_file,overwrite=True)
		os.remove(merged_file)

		print "INFO: Deleting files on ", day_tmp
		self.hdfsclient.delete(day_tmp,recursive=True)


				
if __name__ == '__main__' :
	import argparse

	count=0

	parser = argparse.ArgumentParser(description="Merge daily historical snappy files into one to save hdfs space")
	parser.add_argument('topic', help="Topic name relative to --base")
	parser.add_argument('--hdfs_user', help="HDFS user name (default: current user)",default=None)
	parser.add_argument('--hdfs_server', help="HDFS server name or ip (default: aquhmstsys022001.c022.digitalriverws.net)",default="aquhmstsys022001.c022.digitalriverws.net")
	parser.add_argument('--hdfs_port', help="HDFS server port number (default:8020)", type=int, default=8020)
	parser.add_argument('--hdfs_tmp', help="HDFS temporary dir to store files to be merged (default:/user/hduser/tmp)", default="/user/hduser/tmp")
	parser.add_argument('--web_port', help="HDFS server WEB port number (default:50070)", type=int, default=50070)
	parser.add_argument('--base', help="Alternate hdfs base path for topic (default:/user/aqueduct/flume)",default="/user/aqueduct/flume")
	parser.add_argument('--start', help="Start Date inclusive  (default: from beginning)")
	parser.add_argument('--end', help="End Date inclusive (default: to end)")


	args = parser.parse_args()
	topic=HDFS_topic(topic=args.topic,user=args.hdfs_user,server=args.hdfs_server,port=args.hdfs_port,\
                     hdfs_tmp=args.hdfs_tmp,web_port=args.web_port,base=args.base)
	try:
		topic.merge(args.start,args.end)
	except Exception as err:
		print err
		exit