Python CommonUtils类代码示例

def parsePartInfo(filename=None):

    if filename:
        cmd = shlex.split('cat ' + filename)
    else:
        cmd = shlex.split('sinfo -h -o "%20P %5a %25C %25l %25L %25s %25F %25B %25z"')
    
    container = PartitionInfoHandler()
    CommonUtils.parseStream(cmd, container)
    return container

 def __init__( self , session ):
     """Input: coral session handle
     """
     self.__session = session
     self.__tagInventoryTableName=CommonUtils.inventoryTableName()
     self.__tagInventoryIDName=CommonUtils.inventoryIDTableName()
     self.__tagInventoryTableColumns = {'tagid':'unsigned long', 'tagname':'string', 'pfn':'string','recordname':'string', 'objectname':'string', 'labelname':'string'}
     self.__tagInventoryTableNotNullColumns = ['tagname','pfn','recordname','objectname']
     #self.__tagInventoryTableUniqueColumns = ['tagname']
     self.__tagInventoryTablePK = ('tagid')

def requestUrlContent(url, cache_dir=None, filename=None):
    if cache_dir != None and not os.path.isdir(cache_dir):
        os.makedirs(cache_dir)

    ext = os.path.splitext(url)[-1]
    if filename == None:
        filename = CommonUtils.md5(url) + ext

    target_path = None
    if cache_dir == None:
        target_path = tempfile.mktemp()
    else:
        target_path = os.path.join(".", cache_dir)
        target_path = os.path.join(target_path, filename)

    if target_path == None:
        target_path = tempfile.mktemp()

    wget = 'wget --user-agent="Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)"'
    command = wget + ' "%s" -O %s --timeout=60 --tries=2'%(url, target_path)

    if (not os.path.isfile(target_path)):
        state = os.system(command.encode("utf-8")) # 在windows下是gb2312, 在ubuntu主机上应该是utf-8
        if state != 0:
            print url, "download failed!"
    else:
        pass

    # 需要保证返回的字符串是ascii编码，否则lxml xpath解析可能会出问题
    # phantomjs保存的文件是utf-8编码，所以需要进行转码为ascii编码
    return open(target_path).read()

def insertPlayer(players):
    insertSql = "insert into player(player_zn_name,player_eng_name,team,location,height,weight,birthday,money) " \
                "values(""?,?,?,?,?,?,?,?)"
    selectSql = "select * from player where player_eng_name = ? and birthday=?"
    updateSql = "update player set team=?,location=?,height =?,weight=?,birthday=?,money=?,player_zn_name=? where player_eng_name=? and birthday=?"

    conn = CommonUtils.getConnect()
    cu = conn.cursor()

    #cu.execute("delete from player")

    insertValues = []
    updateValues = []
    for player in players:
        # 先判断是否存在此球员
        cu.execute(selectSql, (player.playerEngName, player.birthday))
        result = cu.fetchall()
        if(len(result) == 0):
            #若不存在，则直接插入
            insertValues.append((player.playerZnName, player.playerEngName, player.teamName, player.location, player.height, player.weight, player.birthday, player.money))
        else:
            # 若存在，则直接更新即可
            updateValues.append((player.teamName,player.location,player.height,player.weight,player.birthday,player.money,player.playerZnName,player.playerEngName,player.birthday))

    # before insert,delete data that is older
    #cu.execute("delete from player")
    # insert player data
    print "新增数据" + str(len(insertValues))
    print "更新数据" + str(len(updateValues))
    cu.executemany(insertSql,insertValues)
    cu.executemany(updateSql,updateValues)
    conn.commit()
    conn.close()

    def downloadThumbnail(self):
        """
        下载缩略图
        """
        cacheDir = "cache" + os.path.sep + self._infoType + os.path.sep + "img" + os.path.sep + str(self._newsId)
        url = self._thumbnail
        thumbnailPath = self.requestUrlContent(url, cacheDir, CommonUtils.md5(url) + os.path.splitext(url)[-1])

        if not os.path.isfile(thumbnailPath):
            return None

        # 处理成5种尺寸, 按宽度自适应
        # XS:100, S:200, M:400, L:600, XL:900
        im = Image.open(thumbnailPath)
        width, height = im.size
        filename, ext = os.path.splitext(os.path.basename(thumbnailPath))

        thumbnailCachePath = self._thumbnailCachePath

        # XS
        target_file = os.path.join(thumbnailCachePath, filename+"_portfolio_"+"XS"+ext)
        try:
            _width = 100
            _heigh = _width * height / width
            out = im.resize((_width, _heigh), Image.ANTIALIAS)
            out.save(target_file)
        except Exception, e:
            shutil.copyfile(thumbnailPath, target_file)

 def getCommentsForTable( self, tableName ):
     """get all comments for given table
     result=[(entryid,comment)]
     """
     transaction=self.__session.transaction()
     result=[]
     
     try:
         transaction.start(True)
         schema = self.__session.nominalSchema()
         query = schema.tableHandle(CommonUtils.commentTableName()).newQuery()
         condition='tablename = :tablename'
         conditionbindDict=coral.AttributeList()
         conditionbindDict.extend('tablename','string')
         conditionbindDict['tablename'].setData(tableName)
         query.addToOutputList('entryid')
         query.addToOutputList('comment')
         query.setCondition(condition,conditionbindDict)
         cursor=query.execute()
         while cursor.next():
             comment=cursor.currentRow()['comment'].data()
             entryid=cursor.currentRow()['entryid'].data()
             result.append((entryid,comment))  
         cursor.close()
         transaction.commit()
         del query
         return result
     except Exception, e:
         transaction.rollback()
         raise Exception, str(e)

 def getCommentForId( self, tableName, entryid ):
     """get comment for given id in given table
     """
     transaction=self.__session.transaction()
     comment=''
     try:
         transaction.start(True)
         schema = self.__session.nominalSchema()
         query = schema.tableHandle(CommonUtils.commentTableName()).newQuery()
         condition='entryid = :entryid AND tablename = :tablename'
         conditionbindDict=coral.AttributeList()
         conditionbindDict.extend('entryid','unsigned long')
         conditionbindDict.extend('tablename','string')
         conditionbindDict['entryid'].setData(entryid)
         conditionbindDict['tablename'].setData(tableName)
         query.addToOutputList('comment')
         query.setCondition(condition,conditionbindDict)
         cursor=query.execute()
         if cursor.next():
             comment=cursor.currentRow()['comment'].data()
             cursor.close()
         transaction.commit()
         del query
         return comment
     except Exception, e:
         transaction.rollback()
         raise Exception, str(e)

 def createEntryCommentTable(self):
     """Create entry comment able.Existing table will be deleted.
     """
     try:
        transaction=self.__session.transaction()
        transaction.start()
        schema = self.__session.nominalSchema()
        schema.dropIfExistsTable(CommonUtils.commentTableName())
        description = coral.TableDescription()
        description.setName(CommonUtils.commentTableName())
        for columnName, columnType in self.__entryCommentTableColumns.items():
            description.insertColumn(columnName,columnType)
        for columnName in self.__entryCommentTableNotNullColumns:
            description.setNotNullConstraint(columnName,True)
        description.setPrimaryKey(self.__entryCommentTablePK)
        tablehandle=schema.createTable(description)
        tablehandle.privilegeManager().grantToPublic(coral.privilege_Select)
        transaction.commit()
     except Exception, e:
        transaction.rollback() 
        raise Exception, str(e)

 def existCommentTable(self):
     """Check if entry comment table exists
     """
     try:
         transaction=self.__session.transaction()
         transaction.start(True)
         schema = self.__session.nominalSchema()
         result=schema.existsTable(CommonUtils.commentTableName())
         transaction.commit()
         #print result
     except Exception, er:
         transaction.rollback()
         raise Exception, str(er)

def insertSchedules(schedules):
	insertSql = "insert into point(type,date,time,weekday,guest,home) values (?,?,?,?,?,?)"

	insertValus = []
	for point in schedules:
		schedule = (point.type,point.date,point.time,point.weekday,point.guest,point.home)
		insertValus.append(schedule)

	conn = CommonUtils.getConnect()
	cu = conn.cursor()
	cu.execute("delete from point")
	cu.executemany(insertSql,insertValus)
	conn.commit()
	conn.close()

def insertTeam(teams):
    insertSql = "insert into team(eng_name,chinese_name,city,short_name) values(?,?,?,?)"

    insertValues = []
    for team in teams:
        value = (team.engName, team.chineseName, team.city, team.shortName)
        insertValues.append(value)

    conn = CommonUtils.getConnect()
    cu = conn.cursor()
    cu.execute("delete from team")
    cu.executemany(insertSql, insertValues)
    conn.commit()
    conn.close()

 def bulkinsertComments( self, tableName,bulkinput):
     """bulk insert comments for a given table
     bulkinput [{'entryid':unsigned long, 'tablename':string,'comment':string}]
     """
     transaction=self.__session.transaction()
     try:
         transaction.start(False)
         schema = self.__session.nominalSchema()
         dbop=DBImpl.DBImpl(schema)
         dbop.bulkInsert(CommonUtils.commentTableName(),self.__entryCommentTableColumns,bulkinput)
         transaction.commit()  
     except Exception, e:
         transaction.rollback()
         raise Exception, str(e)    

 def insertComment( self, tablename, entryid,comment ):
     """insert comment on the given entry of given table
     """
     transaction=self.__session.transaction()
     try:
         transaction.start(False)
         tabrowValueDict={'entryid':entryid,'tablename':tablename,'comment':comment}
         schema = self.__session.nominalSchema()
         dbop=DBImpl.DBImpl(schema)
         dbop.insertOneRow(CommonUtils.commentTableName(),self.__entryCommentTableColumns,tabrowValueDict)
         transaction.commit()
     except Exception, e:
         transaction.rollback()
         raise Exception, str(e)

 def clearAllEntriesForTable( self, tablename ):
     """delete all entries related with given table
     """
     transaction=self.__session.transaction()
     try:
         transaction.start(False)
         dbop=DBImpl.DBImpl(self.__session.nominalSchema())
         condition='tablename = :tablename'
         conditionbindDict=coral.AttributeList()
         conditionbindDict.extend('tablename','string')
         conditionbindDict['tablename'].setData(tablename)
         dbop.deleteRows(CommonUtils.commentTableName(),condition,conditionbindDict)
         transaction.commit()
     except Exception, e:
         transaction.rollback()
         raise Exception, str(e)

def getTeamsInfo():
    # base url
    url = "http://g.hupu.com/nba/players/"
    # teams
    teams = [
        "Spurs",
        "Rockets",
        "Mavericks",
        "Grizzlies",
        "Pelicans",
        "Clippers",
        "Warriors",
        "Suns",
        "Lakers",
        "Kings",
        "Blazers",
        "Thunder",
        "Nuggets",
        "Timberwolves",
        "Jazz",
        "Raptors",
        "Nets",
        "Knicks",
        "Celtics",
        "76ers",
        "Heat",
        "Hawks",
        "Wizards",
        "Bobcats",
        "Magic",
        "Pacers",
        "Bulls",
        "Pistons",
        "Cavaliers",
        "Bucks",
    ]
    # team dictionary
    teamDic = {}

    # get all html info divide by teamName
    for team in teams:
        teamUrl = url + team
        teamPlayerInfos = CommonUtils.getSoupFromUrl(teamUrl)
        teamDic[team] = teamPlayerInfos
    return teamDic