Python TBufferedTransport.close方法代码示例

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
def main(argv):
  p = argparse.ArgumentParser()
  add_common_args(p)
  # Since THeaderTransport acts as framed transport when detected frame, we
  # cannot use --transport=framed as it would result in 2 layered frames.
  p.add_argument('--override-transport')
  p.add_argument('--override-protocol')
  args = p.parse_args()
  assert args.protocol == 'header'
  assert args.transport == 'buffered'
  assert not args.ssl

  sock = TSocket(args.host, args.port, socket_family=socket.AF_INET)
  if not args.override_transport or args.override_transport == 'buffered':
    trans = TBufferedTransport(sock)
  elif args.override_transport == 'framed':
    print('TFRAMED')
    trans = TFramedTransport(sock)
  else:
    raise ValueError('invalid transport')
  trans.open()

  if not args.override_protocol or args.override_protocol == 'binary':
    proto = TBinaryProtocol(trans)
  elif args.override_protocol == 'compact':
    proto = TCompactProtocol(trans)
  else:
    raise ValueError('invalid transport')

  test_void(proto)
  test_void(proto)

  trans.close()

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
    def run(self):
        global work_mutex
        global work_numbers
                
        err_local = 0
        try:
    	    socket = TSocket(self.server_ip, int(self.server_port))
            transport = TFramedTransport(socket)
            transport.open()
            protocol = TBinaryProtocol.TBinaryProtocol(transport)
            client = ThriftNeloEventServer.Client(protocol)   


            stop_flag = True
            while stop_flag:
    	        #read thrift from file
                f = file(file_name, 'r')
                fd_transport = TFileObjectTransport(f)
                buffered_transport = TBufferedTransport(fd_transport)
                binary_protocol = TBinaryProtocol.TBinaryProtocol(buffered_transport)
                fd_transport.open()
                stop_flag = False
                try:
                    evt = ThriftNeloEvent()
                    while True:
                        evt.read(binary_protocol)
				        #send the log to each project name
                        for prjName, logCnt in prj_dict.items():
                            try:
                                if logCnt_dict.has_key(prjName):
                                    if int(logCnt_dict[prjName]) < int(logCnt):
                                        evt.projectName = prjName
                                        evt.sendTime = int(time.time() * 1000)
                                        err = client.ackedAppend(evt)
                                        tps_remember(err)
                                        err_local += err
                                        logCnt_dict[prjName] = logCnt_dict[prjName] + 1
                                        stop_flag = True
                                else:
                                    evt.projectName = prjName
                                    err = client.ackedAppend(evt)
                                    tps_remember(err)
                                    err_local += err
                                    logCnt_dict[prjName] = 1
                                    stop_flag = True
                            except TException, msg:
                                print msg, prjName
                except EOFError,msg:
                    buffered_transport.close() #close the transport
                    stop_flag = True

            work_mutex.acquire()
            work_numbers -= 1
            work_mutex.release()
 
            socket.close()

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
def getMasterTables(hbaseHost):
    transport = TBufferedTransport(TSocket(hbaseHost, 9090))
    transport.open()
    protocol = TBinaryProtocol.TBinaryProtocol(transport)
    client = Hbase.Client(protocol)

    for table in client.getTableNames():
        if 'master' in table:
            print table

    transport.close()

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
 def hs2_port_is_open(self):
   """Test if the HS2 port is open. Does not need to authenticate."""
   # Impyla will try to authenticate as part of connecting, so preserve previous logic
   # that uses the HS2 thrift code directly.
   try:
     socket = TSocket(self.hostname, self.hs2_port)
     transport = TBufferedTransport(socket)
     transport.open()
     transport.close()
     return True
   except Exception, e:
     LOG.info(e)
     return False

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
class ThriftClient(object):
    def __init__(self, port=6458, host="127.0.0.1"):
        self.host = host
        self.port = port
        self.client = None
        self.transport = None

    def connect(self):
        self.transport = TSocket(self.host, self.port)
        self.transport = TBufferedTransport(self.transport)
        protocol = TBinaryProtocol(self.transport)
        self.client = Client(protocol)
        self.transport.open()

    def execute(self, name, *args, **kwargs):
        if self.client is None:
            self.connect()
        result = getattr(self.client, name)(*args, **kwargs)
        return result

    def create(self, key, second, minute = -1, hour = -1, day = -1, month = -1, week = -1, action="shell", params={}):
        return self.execute("create", key, second, minute, hour, day, month, week, action, params)

    def create_timeout(self, key, second, minute = -1, hour = -1, day = -1, month = -1, week = -1, count=1, action="shell", params={}):
        return self.execute("createTimeout", key, second, minute, hour, day, month, week, count, action, params)

    def remove(self, key):
        return self.execute("remove", key)

    def get(self, key):
        return self.execute("get", key)

    def get_current(self):
        return self.execute("getCurrent")

    def get_time(self, timestamp):
        return self.execute("getTime", timestamp)

    def get_keys(self, prefix=''):
        return self.execute("getKeys", prefix)

    def __del__(self):
        if self.client:
            self.transport.close()
            self.transport = None
            self.client = None

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
class writeThread(threading.Thread):
    def __init__(self, threadname, RecordsThreadwillwrite):
        threading.Thread.__init__(self, name = threadname)
        bytesPerColumn = int(bytesPerRecord/columns) - 11 #suppose 3 columns

        self.columnvalue = "value_" + "x"*bytesPerColumn + "_endv"
        self.tbwBatch = int (RecordsThreadwillwrite / recordsPerBatch)
        
        self.transport = TBufferedTransport(TSocket('10.1.2.230', 9090), 40960)
        self.transport.open()
        protocol = TBinaryProtocol.TBinaryProtocol(self.transport)

        self.client = Hbase.Client(protocol)
                        
    def run(self):
        print "+%s start" % (self.getName())
        global gEndT
        global gWritenItems           
        
        threadWritenItem = 0
        for loopidx in xrange(0, self.tbwBatch):            
            self.write_hbase() #write                                           
            threadWritenItem += recordsPerBatch   
            
        mylock.acquire()
        gEndT = time.time()  
        gWritenItems += threadWritenItem
        print "%s done, %s seconds past, %d reocrds saved" % (self.getName(), gEndT-gStartT, gWritenItems)
        mylock.release()
        self.transport.close()
             
        
        
        
    def write_hbase(self): #write 50 rowkyes, and  3 column families in each rowkey
        print self.getName(), "Start write"
        batchmutations = []
        for i in xrange(0, recordsPerBatch): # write to db, 300 items together
            mutations = []
            rowkey = "RK_%s_%s" % (random.random(), time.time())       
            for ii in xrange(0, columns):
                mutations.append(Hbase.Mutation(column="f1:%s"%ii, value=self.columnvalue))
            batchmutations.append(Hbase.BatchMutation(rowkey, mutations))
        self.client.mutateRows("testdb1", batchmutations)        

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
class MyHbase:
    """
    the main class,use to connect to hbase,create ,select table etc.
    """

    def __init__(self, netloc, port, table="diracAccounting"):
        self.tableName = table

        self.transport = TBufferedTransport(TSocket(netloc, port))
        self.protocol = TBinaryProtocol(self.transport)
        self.client = Hbase.Client(self.protocol)
        self.transport.open()

        tables = self.client.getTableNames()
        # check if has table 'diracAccounting',if has then delete it and then recreate it
        for table in tables:
            if table == "diracAccounting":
                if self.client.isTableEnabled(table):
                    print "disabling table:%s" % (table)
                    self.client.disableTable(table)
            print "deleting table:%s" % (table)
            self.client.deleteTable(table)
            # if self.tableName not in tables:
            self.__createTable(["groupby", "generate"])

    def __del__(self):
        self.transport.close()

    def __createTable(self, columnfamilyList):
        """argument:
            columnfamilyList is a list
            the columnfanilyList=['gruopby','generate']
        """
        columns = []
        for name in columnfamilyList:
            col = ColumnDescriptor(name)
            columns.append(col)
        print "creating tables:%s" % (self.tableName)
        try:
            self.client.createTable(self.tableName, columns)
        except AlreadyExists, ae:
            print "WARN: " + ae.message

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
def main(args):
    
    if (len(args)<3):
        print "%s hbasehost tablename " % args[0]
        sys.exit(1)


    hbaseHost = args[1]

    table_name= args[2]

    transport = TBufferedTransport(TSocket(hbaseHost, 9090))
    transport.open()
    protocol = TBinaryProtocol.TBinaryProtocol(transport)
    
    global client
    client = Hbase.Client(protocol)
    
    updateColumn(table_name)
    
    transport.close()

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
def main(argv):
  p = argparse.ArgumentParser()
  add_common_args(p)
  # Since THeaderTransport acts as framed transport when detected frame, we
  # cannot use --transport=framed as it would result in 2 layered frames.
  p.add_argument('--override-transport')
  args = p.parse_args()
  assert args.protocol == 'header'
  assert args.transport == 'buffered'
  assert not args.ssl

  sock = TSocket(args.host, args.port, socket_family=socket.AF_INET)
  if not args.override_transport or args.override_transport == 'buffered':
    trans = TBufferedTransport(sock)
  elif args.override_transport == 'framed':
    trans = TFramedTransport(sock)
  else:
    raise ValueError('invalid transport')
  trans.open()
  proto = TBinaryProtocol(trans)
  proto.writeMessageBegin('testVoid', TMessageType.CALL, 3)
  proto.writeStructBegin('testVoid_args')
  proto.writeFieldStop()
  proto.writeStructEnd()
  proto.writeMessageEnd()
  trans.flush()

  _, mtype, _ = proto.readMessageBegin()
  assert mtype == TMessageType.REPLY
  proto.readStructBegin()
  _, ftype, _ = proto.readFieldBegin()
  assert ftype == TType.STOP
  proto.readFieldEnd()
  proto.readStructEnd()
  proto.readMessageEnd()

  trans.close()

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
class MyHbase():
    def __init__(self,netloc,port,tablename):
        self.tablename = tablename

        self.transport = TBufferedTransport(TSocket(netloc,port))
        self.protocol = TBinaryProtocol(self.transport)
        self.client = Hbase.Client(self.protocol)
        self.transport.open()
        if self.tablename in self.client.getTableNames():
            if not self.client.isTableEnabled(self.tablename):
                print 'table off_line, start bring it on_line now'
                self.client.enableTable(self.tablename)
            print "successfully connect table: %s"%(self.tablename)
            print
        else:
            print "wrong table name"
    def __del__(self):
        self.transport.close()

    def printRow(self,entry):
        print "row: " + entry.row + ", cols:",
        for k in sorted(entry.columns):
            print k + " => " + entry.columns[k].value
            print
    
    def showTableInfo(self):
        '''
        show some information of the hbase table
        '''
        #print self.client.getColumnDescriptors(self.tablename)
        temp = self.client.getColumnDescriptors(self.tablename)
        for (col,desc) in temp.items():
            print "column with name:"+col
            print desc                             

    def generatePlot(self,groupbyname,generatename):
        '''

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
def main(args):
    
    if(len(args) < 2):
        print "%s <verified file> -all"%(args[0])
        sys.exit(1)
    
    filename = args[1]
    opt_all = True if len(args)>2 and args[2] == "-all" else False

    filenamearray = filename.split("_")
    orgId = filenamearray[0]
    subOrgId = filenamearray[1]
    
    getConfiguration('host.properties')

    
    transport = TBufferedTransport(TSocket(hbaseHost, 9090))
    transport.open()
    protocol = TBinaryProtocol.TBinaryProtocol(transport)
    
    global client
    client = Hbase.Client(protocol)
        
    
    tablename = "%s_%s_master_%s"%(orgId,subOrgId,orgId);
    

    for line in open(filename,"r"):
        input=line.strip()
        row=client.getRow(tablename,input)
	print input 
        printRow(row)
	print ""
	if (not opt_all): break

    
    transport.close()

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
def main(args):
    
    if(len(args)!=2):
        print "%s tableName"%(args[0])
        sys.exit(1)
    
    tablename = args[1]
    
    getConfiguration('host.properties')

    
    transport = TBufferedTransport(TSocket(hbaseHost, 9090))
    transport.open()
    protocol = TBinaryProtocol.TBinaryProtocol(transport)
    
    global client
    client = Hbase.Client(protocol)
        
    
#    tablename = "%s_%s_master_%s"%(orgId,subOrgId,orgId);
    client.disableTable(tablename)
    client.deleteTable(tablename)

    transport.close()

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
def main(args):

    if len(args) != 3:
        print "%s orgId subOrgId" % (args[0])
        sys.exit(1)

    orgId = args[1]
    subOrgId = args[2]

    getConfiguration("host.properties")

    transport = TBufferedTransport(TSocket(hbaseHost, 9090))
    transport.open()
    protocol = TBinaryProtocol.TBinaryProtocol(transport)

    global client
    client = Hbase.Client(protocol)

    tablename = "%s_%s_master_%s" % (orgId, subOrgId, orgId)

    connectSolrServer(orgId, subOrgId)
    verifyData(tablename)

    transport.close()

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
class Client (object):
	'''
	This class handles all communication to and from the backend pyLoad server
	instance
	
	Events:
		on_connected: Raised when a connection to the backend server is made
		on_disconnected: Raised when the connection has been severed
	'''
	
	on_connected = None
	on_disconnected = None
	
	
	def __init__ (self):
		'''
		Constructor
		'''
		self.__connected = False
		
		self.on_connected = Event()
		self.on_disconnected = Event()
		
		# set all live properties in this class for polling
		self.properties_to_poll = [
			self.links_active,
			self.links_waiting,
			self.links_total,
			self.speed,
		]

		self.tasks = {}



	def connect (self, host, port, username, password):
		'''
		Connect to the specified backend
		'''
		self.__connected = False
		
		self.host = host
		self.port = port
		self.username = username
		self.password = password
		
		self.socket = TSocket (host, port)
		self.transport = TBufferedTransport (self.socket)
		self.protocol = TBinaryProtocol (self.transport)
		self.client = Pyload.Client (self.protocol)
		
		try:
			self.transport.open()
			self.__connected = True
			logging.info ("Connected to {0}:{1}".format(host, port))
		
		except:
			logging.warn ("Failed to connect to {0}:{1}".format(host, port))
			raise ConnectionFailed
		
		if self.client.login (username, password):
			logging.info ("Server version: {0}".format(self.version))
			self.on_connected()
			return True
		
		return False
		

	@login_required
	def disconnect (self):
		'''
		Disconnect and release the resources from the backend
		'''
		self.transport.close()
		self.__connected = False
		
		self.client = None
		self.protocol = None
		self.transport = None
		self.socket = None
		
		self.host = None
		self.port = None
		self.username = None
		self.password = None
		
		logging.info ("Disconnected from {0}:{1}".format(self.host, self.port))
		self.on_disconnected()
	
	
	@property
	def connected (self):
		'''
		Whether or not the client is connected to a server
		'''
		return self.__connected
		
	
	@property
	@login_required
#.........这里部分代码省略.........

# 需要导入模块: from thrift.transport.TTransport import TBufferedTransport [as 别名]
# 或者: from thrift.transport.TTransport.TBufferedTransport import close [as 别名]
class ESIndexCreator:
	def __init__(self):
		reComments = re.compile('<!--[^>]*-->')
		reHtml = re.compile('</?\w+[^>]*>')

		self.host = "172.20.6.61"
		self.port = 9090
		self.transport = TBufferedTransport(TSocket(self.host, self.port))
		self.transport.open()
		self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
		self.client = Hbase.Client(self.protocol)
		self.es = Elasticsearch(
			['172.20.8.162'],
			sniff_on_start = True,
			sniff_on_connection_fail = True,
			sniffer_timeout = 60
		)

	def __del__(self):
		self.transport.close()

	def filterTags(self,htmlstr):  
		#先过滤CDATA  
		re_cdata=re.compile('//<!\[CDATA\[[^>]*//\]\]>',re.I) #匹配CDATA  
		re_script=re.compile('<\s*script[^>]*>[^<]*<\s*/\s*script\s*>',re.I)#Script  
		re_style=re.compile('<\s*style[^>]*>[^<]*<\s*/\s*style\s*>',re.I)#style  
		re_br=re.compile('<br\s*?/?>')#处理换行  
		re_h=re.compile('</?\w+[^>]*>')#HTML标签  
		re_comment=re.compile('<!--[^>]*-->')#HTML注释  
		s=re_cdata.sub('',htmlstr)#去掉CDATA  
		s=re_script.sub('',s) #去掉SCRIPT  
		s=re_style.sub('',s)#去掉style  
		s=re_br.sub('',s)#将br转换为换行  
		s=re_h.sub('',s) #去掉HTML 标签  
		s=re_comment.sub('',s)#去掉HTML注释  
		#去掉多余的空行  
		blank_line=re.compile('\n+')  
		s=blank_line.sub('',s)  
		return s

	def getAllTablesInfo(self):
		#get table info
		listTables = self.client.getTableNames()
		print "="*40
		print "Show all tables information...."

		for tableName in listTables:
			print "TableName:" + tableName
			print " "
			listColumns = self.client.getColumnDescriptors(tableName)
			print listColumns
			print " "

			listTableRegions = self.client.getTableRegions(tableName)
			print listTableRegions
			print "+"*40

	def deleteIndex(self,indexName):
		print "Remove index...."
		self.es.indices.delete(index=indexName,ignore=[400,404])

	def createIndex(self,indexName):
		print "Create index...."
		#create indexs
		self.es.indices.create(index=indexName)

		print "Define mapping...."
		#define mapping
		self.es.indices.put_mapping(
				index=indexName,
				doc_type="article",
				ignore_conflicts='true',
				body={
					"article":{
						"properties":{
							"sitename":{ "type":"string", "store":"true", "index":"not_analyzed" },
							"addtime":{"type":"date", "store":'true' },
							"publishtime":{"type":"date", "store":'true' },
							"keywords":{"type":"string", "store":'true',"analyzer":"ik" },
							"content":{"type":"string", "store":'true',"analyzer":"ik" },
							"url":{"type":"string", "store":'true', "index":"not_analyzed"},
							"title":{"type":"string","store":'true', "analyzer":"ik"}
							}
						}
					}
				)
		self.es.indices.put_mapping(
				index=indexName,
				doc_type="baidu",
				ignore_conflicts='true',
				body={
					"baidu":{
						"properties":{
							"sitename":{ "type":"string", "store":"true", "index":"not_analyzed" },
							"addtime":{"type":"date", "store":'true' },
							"publishtime":{"type":"date", "store":'true' },
							"keywords":{"type":"string", "store":'true',"analyzer":"ik" },
							"content":{"type":"string", "store":'true',"analyzer":"ik" },
							"url":{"type":"string", "store":'true', "index":"not_analyzed"},
							"title":{"type":"string","store":'true', "analyzer":"ik"}
#.........这里部分代码省略.........