Python FileSync类代码示例

def get_team_score_json(global_config, name, comp, store_json_file=False):
        
    global_config['logger'].debug( 'GET Team %s Score For Competition %s', name, comp )
    
    season = WebCommonUtils.map_comp_to_season(comp)
    session = DbSession.open_db_session(global_config['db_name'] + season)
    
    result = []
    result.append('{ "competition" : "%s", "team" : "%s", ' % (comp,name))
    team_scores = DataModel.getTeamScore(session, name, comp)
    if len(team_scores)==1:
        result.append('"score": "%s" }' % team_scores[0].score)
    else:
        result.append('  "score": [')
        for score in team_scores:
            result.append(score.json())
            result.append(',\n')
        if len(team_scores) > 0:
            result = result[:-1]
        result.append(']}')
        
    json_str = ''.join(result)
    
    if store_json_file is True:
        try:
            FileSync.put( global_config, '%s/EventData/TeamData/team%s_scouting_score.json' % (comp,name), 'text', json_str)
        except:
            raise
        
    session.remove()
    return json_str

def get_geo_location_json(include_teams=True, include_events=True):

    geo_locations = {}
    filename = "geo_coordinates_for"

    my_config = ScoutingAppMainWebServer.global_config
    session = DbSession.open_db_session(my_config["db_name"] + my_config["this_season"])

    if include_events:
        filename += "_Events"
        events = session.query(DataModel.EventInfo).all()
        for event in events:
            if event.geo_location is not None:
                geo_locations[event.event_key] = json.loads(event.geo_location)

    if include_teams:
        filename += "_Teams"
        teams = session.query(DataModel.TeamInfo).all()
        for team in teams:
            if team.geo_location is not None:
                team_key = "frc%d" % team.team
                geo_locations[team_key] = json.loads(team.geo_location)

    geo_location_json = json.dumps(geo_locations)
    geo_location_js = "var %s = '%s';" % (filename, geo_location_json)

    # store the geo location information to a file, too
    try:
        FileSync.put(my_config, "GlobalData/%s.json" % (filename), "text", geo_location_json)
        FileSync.put(my_config, "GlobalData/%s.js" % (filename), "text", geo_location_js)
    except:
        raise

    return geo_location_json

def get_team_scouting_notes_json(global_config, comp, name, store_json_file=False):
    
    global_config['logger'].debug( 'GET Team %s Scouting Notes For Competition %s', name, comp )
    
    season = WebCommonUtils.map_comp_to_season(comp)
    session = DbSession.open_db_session(global_config['db_name'] + season)

    result = []

    result.append('{ "competition" : "%s", "team" : "%s",\n' % (comp,name))
    result.append('  "scouting_notes" : [\n')

    team_notes = DataModel.getTeamNotes(session, name, comp)
    for note in team_notes:
        result.append('   { "tag": "%s", "note": "%s" }' % (note.tag,note.data))
        result.append(',\n')
        
    if len(team_notes) > 0:         
        result = result[:-1]

    result.append(' ] }\n')
    
    json_str = ''.join(result)

    if store_json_file is True:
        try:
            FileSync.put( global_config, '%s/EventData/TeamData/team%s_scouting_notes.json' % (comp,name), 'text', json_str)
        except:
            raise
        
    session.remove()
    return json_str

def get_team_participation_json():

    team_participation = {}
    filename = "team_participation"

    my_config = ScoutingAppMainWebServer.global_config
    session = DbSession.open_db_session(my_config["db_name"] + my_config["this_season"])

    teams = session.query(DataModel.TeamInfo).all()
    for team in teams:
        team_key = "frc%d" % team.team
        if team.first_competed is not None:
            info = {}
            info["first_competed"] = team.first_competed
            info["last_competed"] = team.last_competed
            team_participation[team_key] = info

    team_participation_json = json.dumps(team_participation)
    team_participation_js = "var %s = '%s';" % (filename, team_participation_json)

    # store the geo location information to a file, too
    try:
        FileSync.put(my_config, "GlobalData/%s.json" % (filename), "text", team_participation_json)
        FileSync.put(my_config, "GlobalData/%s.js" % (filename), "text", team_participation_js)
    except:
        raise

    return team_participation_json

def get_team_score_breakdown_json(global_config, name, comp=None, store_json_file=False):
        
    global_config['logger'].debug( 'GET Team Score Breakdown: %s', name )
    
    if comp == None:
        comp = global_config['this_competition'] + global_config['this_season']
        season = global_config['this_season']
    else:
        season = WebCommonUtils.map_comp_to_season(comp)
    session = DbSession.open_db_session(global_config['db_name'] + season)
    
    attrdef_filename = WebCommonUtils.get_attrdef_filename(comp=comp)
    attr_definitions = AttributeDefinitions.AttrDefinitions(global_config)
    attr_definitions.parse(attrdef_filename)
    
    result = []

    result.append('{ "score_breakdown": [\n')

    team_attributes = DataModel.getTeamAttributesInOrder(session, name, comp)
    for attribute in team_attributes:
        attr_def = attr_definitions.get_definition( attribute.attr_name )
        if attr_def:
            try:
                stat_type = attr_def['Statistic_Type']
            except:
                stat_type = 'Total'

            weight = int(float(attr_def['Weight']))
            if weight != 0:
                if stat_type == 'Average':
                    value = int(attribute.cumulative_value/attribute.num_occurs)
                else:
                    value = int(attribute.cumulative_value)
                data_str = '{"attr_name": "%s", "raw_score": %d, "weighted_score": %d}' % (attribute.attr_name,int(value),int(weight*value)) 
                result.append(data_str)
                result.append(',\n')
    if len(team_attributes) > 0:
        result = result[:-1]
        result.append('\n')
    result.append(']}')
    
    json_str = ''.join(result)
    
    if store_json_file is True:
        try:
            FileSync.put( global_config, '%s/EventData/TeamData/team%s_scouting_scorebreakdown.json' % (comp,name), 'text', json_str)
        except:
            raise
        
    session.remove()
    return json_str

def get_team_scouting_mediafiles_json(global_config, comp, name, store_json_file=False):
    
    global_config['logger'].debug( 'GET Team %s Scouting Mediafiles For Competition %s', name, comp )

    result = []

    result.append('{ "competition" : "%s", "team" : "%s",\n' % (comp,name))
    result.append('  "scouting_mediafiles" : [\n')

    input_dir = './static/data/' + comp + '/ScoutingPictures/'
    pattern = 'Team' + name + '_' + '[a-zA-Z0-9_]*.jpg|mp4'
    mediafiles = get_datafiles(input_dir, re.compile(pattern), False, global_config['logger'])

    for filename in mediafiles:
        segments = filename.split('/')
        basefile = segments[-1]
        
        result.append('   { "filename": "%s" }' % (basefile))
        result.append(',\n')
    
    if len(mediafiles) > 0:         
        result = result[:-1]
        
    result.append(' ],\n')
    result.append('  "thumbnailfiles" : [\n')

    ImageFileUtils.create_thumbnails(mediafiles)
    thumbnail_dir = input_dir + "Thumbnails/"
    pattern = '[0-9]*x[0-9]*_Team' + name + '_' + '[a-zA-Z0-9_]*.jpg|mp4'
    thumbnailfiles = get_datafiles(thumbnail_dir, re.compile(pattern), False, global_config['logger'])
    
    for filename in thumbnailfiles:
        segments = filename.split('/')
        basefile = segments[-1]
        
        result.append('   { "filename": "%s" }' % (basefile))
        result.append(',\n')
    
    if len(thumbnailfiles) > 0:         
        result = result[:-1]

    result.append(' ] }\n')
    json_str = ''.join(result)
    
    if store_json_file is True:
        try:
            FileSync.put( global_config, '%s/EventData/TeamData/team%s_scouting_mediafiles.json' % (comp,name), 'text', json_str)
        except:
            raise
        
    return json_str

def get_event_rank_list_json(global_config, year, event_code):
        
    global_config['logger'].debug( 'GET Event Rank List Json' )
    
    # derive our competition name from the FIRST event code 
    competition = WebCommonUtils.map_event_code_to_comp(year+event_code)

    #return get_data_from_first(global_config, year, event_code, 'rankings')
    store_data_to_file = False
    result = []
    
    rankings_data = get_event_data_from_tba( '%s%s/rankings' % (year,event_code.lower()) )

    result.append('{ "event" : "%s",\n' % (event_code.lower()))
    
    rankings = rankings_data.get('rankings', [])
    if len(rankings):
        # rankings is now a list of lists, with the first element of the list being the list of column headings
        # take the list of columngs and apply to each of the subsequent rows to build the json response
        result.append('  "last_updated": "%s",\n' % time.strftime('%c'))
        result.append('  "rankings" : [\n')
        
        for team_rank in rankings:
            result.append('       { "rank": %d, "team_number": %s, "status": "available" }' % (team_rank['rank'],team_rank['team_key'].replace('frc','')))
            result.append(',\n')
                
        if len(rankings) > 1:         
            result = result[:-1]
        result.append(' ]\n')
        store_data_to_file = True
    else:
        # we were not able to retrieve the data from FIRST, so let's return any stored file with the 
        # information, otherwise we will return an empty json payload
        stored_file_data = FileSync.get( global_config, '%s/EventData/ranklist.json' % (competition) )
        if stored_file_data != '':
            return stored_file_data
        else:
            # no stored data either, so let's just return a formatted, but empty payload
            result.append('  "last_updated": "%s",\n' % time.strftime('%c'))
            result.append('  "rankings" : []\n')        

    result.append(' }\n')
    json_str = ''.join(result)
    if store_data_to_file:
        try:
            FileSync.put( global_config, '%s/EventData/ranklist.json' % (competition), 'text', json_str)
        except:
            raise
    return json_str

def get_team_list_json(global_config, season, event, store_json_file=False):
    global team_info_dict
    
    global_config['logger'].debug( 'GET Team List For Competition %s', event )

    comp = WebCommonUtils.map_event_code_to_comp(event, season)
    
    session = DbSession.open_db_session(global_config['db_name'] + season)

    result = []

    result.append('{ "teams" : [\n')

    '''
    team_list = DataModel.getTeamsInNumericOrder(session, comp)
    for team in team_list:
        team_info = None
        # TODO - Remove this hardcoded number for the valid team number. This check prevents
        # requesting information for invalid team numbers, which has been known to happen when
        # tablet operators enter bogus team numbers by mistake
        if team.team < 10000:
            team_info = DataModel.getTeamInfo(session, int(team.team))
            
        if team_info:
            result.append('   { "team_number": "%s", "nickname": "%s" }' % (team.team,team_info.nickname))
            result.append(',\n')
        else:
            result.append('   { "team_number": "%s", "nickname": "%s" }' % (team.team,'Unknown'))
            result.append(',\n')
        
    if len(team_list) > 0:         
        result = result[:-1]
        result.append(' ] }\n')
        json_str = ''.join(result)
    else:
    '''
    json_str = get_team_list_json_from_tba(global_config, comp)

    if store_json_file is True:
        try:
            FileSync.put( global_config, '%s/EventData/%s.json' % (comp,'teams'), 'text', json_str)
        except:
            raise
        
    session.remove()
    return json_str

def get_team_datafile_json(global_config, filename, store_json_file=False):
        
    global_config['logger'].debug( 'GET Team Data File Json: %s', filename )
        
    comp, fname = filename.split('/', 1)
    filepath = './static/data/' + comp + '/ScoutingData/' + fname
    datafile = open( filepath, "r" )
    
    team = fname.split('_')[0].lstrip('Team')

    result = []
    result.append('{ "competition": "%s",\n' % comp)
    result.append('  "team": "%s",\n' % team)
    result.append('  "filename": "%s",\n' % fname)
    result.append('  "scouting_data": [\n')
    
    while 1:
        lines = datafile.readlines(500)
        if not lines:
            break
        for line in lines:
            line = line.rstrip('\n')
            try:
                name, value = line.split(':',1)
            except:
                pass

            result.append('   { "name": "%s", "value": "%s" }' % (name,value))
            result.append(',\n')
            
        if len(lines) > 0:
            result = result[:-1]
    result.append('] }\n')
    
    json_str = ''.join(result)
    
    if store_json_file is True:
        try:
            short_fname = fname.replace('.txt','')
            FileSync.put( global_config, '%s/EventData/TeamData/team%s_scouting_file_%s.json' % (comp,team,short_fname), 'text', json_str)
        except:
            raise
        
    return json_str

def create_picklist_json(global_config, comp=None, store_json_file=False):
        
    global_config['logger'].debug( 'Create Picklist Json' )
    
    global local_picklist
    store_data_to_file = False
    
    if comp == None:
        comp = global_config['this_competition'] + global_config['this_season']
        season = global_config['this_season']
    else:
        season = WebCommonUtils.map_comp_to_season(comp)

    session = DbSession.open_db_session(global_config['db_name'] + season)
    
    result = []
    result.append('{ "picklist": [\n')
            
    local_picklist = DataModel.getTeamsInRankOrder(session, comp, True)
    rank = 1
    for team in local_picklist:
        # round the score to an integer value
        team.score = float(int(team.score))
        if team.score > 0:
            row = '{ "rank" : %d, "team" : %d, "score" : %d, "competition" : "%s" }' % (rank, team.team, int(team.score), team.competition)
            result.append(row)
            result.append(',\n')
            rank += 1
    if len(result) > 0:
        result = result[:-1]

    result.append(']}')
    
    json_str = ''.join(result)
    
    if store_json_file is True:
        try:
            FileSync.put( global_config, '%s/EventData/%s.json' % (comp,'picklist'), 'text', json_str)
        except:
            raise
        
    session.remove()
    return json_str

def update_picklist_json(global_config, from_position, to_position, comp=None, store_json_file=True):
        
    global_config['logger'].debug( 'Create Picklist Json' )

    global local_picklist
    
    if local_picklist is None:
        create_picklist_json(global_config, comp, store_json_file=True)
                
    result = []
    result.append('{ "picklist": [\n')
    
    if comp == None:
        comp = global_config['this_competition'] + global_config['this_season']
    
    item_to_update = local_picklist.pop( from_position-1 )
    local_picklist.insert(to_position-1, item_to_update)    
    rank = 1
    for team in local_picklist:
        # round the score to an integer value
        team.score = float(int(team.score))
        if team.score > 0:
            row = '{ "rank" : %d, "team" : %d, "score" : %d, "competition" : "%s" }' % (rank, team.team, int(team.score), team.competition)
            result.append(row)
            result.append(',\n')
            rank += 1
    if len(result) > 0:
        result = result[:-1]

    result.append(']}')
    
    json_str = ''.join(result)
    
    if store_json_file is True:
        try:
            FileSync.put( global_config, '%s/EventData/%s.json' % (comp,'picklist'), 'text', json_str)
        except:
            raise
        
    return json_str

def get_team_scouting_datafiles_json(global_config, comp, name, store_json_file=False):
    
    global_config['logger'].debug( 'GET Team %s Scouting Datafiles For Competition %s', name, comp )

    result = []

    result.append('{ "competition" : "%s", "team" : "%s",\n' % (comp,name))
    result.append('  "scouting_datafiles" : [\n')

    input_dir = './static/data/' + comp + '/ScoutingData/'
    pattern = 'Team' + name + '_' + '[a-zA-Z0-9_]*.txt'
    datafiles = get_datafiles(input_dir, re.compile(pattern), False, global_config['logger'])

    for filename in datafiles:
        segments = filename.split('/')
        basefile = segments[-1]
        
        result.append('   { "filename": "%s" }' % (basefile))
        result.append(',\n')
        
        if store_json_file is True:
            get_team_datafile_json( global_config, comp + '/' + basefile, store_json_file )
    
    if len(datafiles) > 0:         
        result = result[:-1]

    result.append(' ] }\n')
    json_str = ''.join(result)
    
    if store_json_file is True:
        try:
            FileSync.put( global_config, '%s/EventData/TeamData/team%s_scouting_datafiles.json' % (comp,name), 'text', json_str)
        except:
            raise
        
    return json_str

                    
                result.append(', ')
            if len(line) > 0:         
                result = result[:-1]
            result.append(' ],\n')
                
        if len(rankings) > 1:         
            result = result[:-1]
        result.append(' ]\n')
        store_data_to_file = True
    except Exception, err:
        print 'Caught exception:', err
    except:
        # we were not able to retrieve the data from FIRST, so let's return any stored file with the 
        # information, otherwise we will return an empty json payload
        stored_file_data = FileSync.get( global_config, '%s/EventData/%s%s.json' % (competition,query_str,round_str) )
        if stored_file_data != '':
            return stored_file_data
    
    result.append(' ] }\n')
    json_str = ''.join(result)
    if store_data_to_file:
        try:
            FileSync.put( global_config, '%s/EventData/%s%s.json' % (competition,query_str,round_str), 'text', json_str)
        except:
            raise
    return json_str

        
def update_event_data_files( global_config, year, event, directory ):
    

def process_json_files(global_config, competition, output_file, input_dir, reprocess_files=False):
    
    # Initialize the database session connection
    db_name  = global_config['db_name'] + global_config['this_season']
    session  = DbSession.open_db_session(db_name)

    # get all the verified files from the input directory. These files are
    # candidates to be processed
    verified_files = FileSync.get_file_list(input_dir, ext='.verified', recurse=True )
    verified_files.sort(key=match_sort)
    
    # For the normal case, get all the processed files, too. We'll use the processed list to 
    # determine which files are actually newly verified and need to be processed. If the 
    # reprocess flag is true, then we'll process all verified files.
    if reprocess_files is not True:
        processed_files = FileSync.get_file_list(input_dir, ext='.processed', recurse=True )
        for processed_file in processed_files:
            verified_files.remove( processed_file.replace('processed','verified') )
    
    xlsx_workbook = None
    excel_intf_ctrl = global_config.get('excel_sheets_intf', 'Disabled')
    if excel_intf_ctrl == 'Enabled':
        # read in the output file, which is expected to be an XLSX file
        try:
            xlsx_workbook = openpyxl.load_workbook(output_file)
        except:
            print 'Error Reading Spreadsheet %s For Input' % output_file
    
    google_intf_ctrl = global_config.get('google_sheets_intf', 'Disabled')

    '''
    # took out for now until we have local dictionary storage
    events = global_config.get('events')
    if events is None:
        events = {}
        global_config['events'] = events
    event_data = events.get( competition )
    if event_data is None:
        events[competition] = { 'ScoutingData': { 'TeamData': {} } }
        event_data = events[competition]

    event_scouting_data = event_data['ScoutingData']['TeamData']
    '''

    for verified_file in verified_files:
        filename = verified_file.split('/')[-1]

        # read the file into a dictionary
        with open(input_dir+verified_file) as fd:
            scouting_data = json.load(fd)
            
            if filename.startswith('Match'):
                team = scouting_data['Setup'].get('Team')
                category = 'Match'
            elif filename.startswith('Pit'):
                team = scouting_data['Pit'].get('Team')
                category = 'Pit'                
            else:
                category = 'Unknown'

            if team is not None and len(team) > 0:
                # ######################################################### #
                # store the scouting data to the local database

                DataModel.addTeamToEvent(session, int(team), competition)

                attr_definitions = AttributeDefinitions.AttrDefinitions(global_config)
                for section_name, section_data in scouting_data.iteritems():
                    if isinstance(section_data,dict):
                        for attr_name, attr_value in section_data.iteritems():
                            
                            # use the attribute definitions to control whether information gets 
                            # stored to the database rather than the hard coded stuff here.
                            # also need to consider the section/category name as the attributes
                            # and definitions are processed
                            
                            # don't store the team number in the database
                            if attr_name == 'Team':
                                continue
                            
                            # augment the attribute name with the section name in order to make the attribute
                            # unique
                            attr_name = '%s:%s' % (section_name, attr_name)
                            
                            attribute_def = {}
                            attribute_def['Name'] = attr_name
                            if attr_value.isdigit():
                                attribute_def['Type'] = 'Integer'
                                attribute_def['Weight'] = 1.0
                            else:
                                attribute_def['Type'] = 'String'
                                attribute_def['Weight'] = 0.0
                            attribute_def['Statistic_Type'] = 'Average'
                            attr_definitions.add_definition(attribute_def)

                            try:
                                DataModel.createOrUpdateAttribute(session, int(team), competition, category, 
                                                                  attr_name, attr_value, attribute_def)
                            except Exception, exception:
                                traceback.print_exc(file=sys.stdout)
#.........这里部分代码省略.........

def get_team_attr_rankings_json(global_config, comp=None, attr_name=None):
        
    global_config['logger'].debug( 'GET Team Attribute Rankings Json' )
    store_data_to_file = False
    
    if comp == None:
        comp = global_config['this_competition'] + global_config['this_season']
        season = global_config['this_season']
    else:
        season = WebCommonUtils.map_comp_to_season(comp)

    session = DbSession.open_db_session(global_config['db_name'] + season)

    attrdef_filename = WebCommonUtils.get_attrdef_filename(comp=comp)
    attr_definitions = AttributeDefinitions.AttrDefinitions()
    attr_definitions.parse(attrdef_filename)
    attr_def = attr_definitions.get_definition(attr_name)
    try:
        stat_type = attr_def['Statistic_Type']
    except:
        stat_type = 'Total'
        
    web.header('Content-Type', 'application/json')
    result = []
    result.append('{  "attr_name" : "%s",\n' % attr_name)
    
    # add the columns bases on the attribute definition type
    result.append('   "columns" : [\n')
    result.append('      { "sTitle": "Team" }')
    result.append(',\n')
    columns = []
    if attr_def['Type'] == 'Map_Integer':
        map_values = attr_def['Map_Values'].split(':')
        for map_value in map_values:
            item_name = map_value.split('=')[0]
            columns.append(item_name)
            result.append('      { "sTitle": "%s" }' % item_name)
            result.append(',\n')
        result = result[:-1]
        result.append('\n')
        result.append('   ],\n')

    if stat_type == 'Average':
        team_rankings = DataModel.getTeamAttributesInAverageRankOrder(session, comp, attr_name)        
    else:
        team_rankings = DataModel.getTeamAttributesInRankOrder(session, comp, attr_name)

    result.append('   "rankings" : [\n')
    for team_attr in team_rankings:
        data_str = '      [ %d,' % team_attr.team
        value_dict = DataModel.mapAllValuesToDict(attr_def, team_attr.all_values)
        for column in columns:
            try:
                value = value_dict[column]
            except:
                value = 0
            data_str += ' %d,' % value
        data_str = data_str.rstrip(',')
        data_str += ' ]'
        result.append(data_str)
        result.append(',\n')

    if len(team_rankings) > 0:
        result = result[:-1]
    result.append('\n')
    result.append('   ]\n}')    
    
    json_str = ''.join(result)
    
    if store_data_to_file is True:
        try:
            file_name = 'attrrankings_%s' % attr_name
            FileSync.put( global_config, '%s/EventData/%s.json' % (comp,file_name), 'text', json_str)
        except:
            raise
        
    session.remove()
    return json_str