Python Fetcher.fetch方法代碼示例

# 需要導入模塊: from fetcher import Fetcher [as 別名]
# 或者: from fetcher.Fetcher import fetch [as 別名]
	def __init__(self, year, month, day = None):
		"""
		Constructor
		
		Arguments:
		year: The... year!
		month: The... month!
		day: The... day! (or None for all days of the month)
		"""
		days = []
		
		if day is None:
			for d in xrange(1, calendar.mdays[month] + 1):
				days.append(datetime.date(year, month, d))
		else:
			days.append(datetime.date(year, month, day))

		begin = days[0]
		end = days[-1]

		f = Fetcher(Fetcher.MLB_TRANSACTION_URL, start=begin.strftime("%Y%m%d"), end=end.strftime("%Y%m%d"))
		try:
		 	obj = f.fetch()
			if obj['transaction_all']['queryResults']['totalSize'] == 0: return
			results = obj['transaction_all']['queryResults']['row']
			
			if type(results) is dict:
				self.append(results)
			else:
				for row in results:
					self.append(row)
		except (ValueError, KeyError), e:
			logger.error("ERROR %s on %s" % (e, f.url))
			pass

# 需要導入模塊: from fetcher import Fetcher [as 別名]
# 或者: from fetcher.Fetcher import fetch [as 別名]
    def loadGamelogs(self, year=None):
        """
        Loads gamelogs for the player for a given year

        Arguments:
        year : The season desired. Defaults to the current year if not specified
        """
        if year is None: year = datetime.datetime.now().year
        if year not in self.logs: self.logs[year] = []

        if 'primary_position' not in self:
            logger.error("no primary position attribute for " % self)
            return False

        url = Fetcher.MLB_PITCHER_URL if self['primary_position'] == 1 else Fetcher.MLB_BATTER_URL

        f = Fetcher(url, player_id=self.player_id, year=year)
        j = f.fetch()

        try:
            if self['primary_position'] == 1:
                parent = j['mlb_bio_pitching_last_10']['mlb_individual_pitching_game_log']['queryResults']
            else:
                if 'mlb_individual_hitting_last_x_total' in j:
                    parent = j['mlb_individual_hitting_last_x_total']['mlb_individual_hitting_game_log']['queryResults']
                else:
                    parent = j['mlb_bio_hitting_last_10']['mlb_individual_hitting_game_log']['queryResults']
        except KeyError, e:
            logger.error('no key for gamelogs found in %s' % f.url)
            return False

# 需要導入模塊: from fetcher import Fetcher [as 別名]
# 或者: from fetcher.Fetcher import fetch [as 別名]
    def loadYearlies(self):
        """
        Loads yearly and career totals for a player
        """
        if self['primary_position'] == 1 and not self.force_batting:
            f = Fetcher(Fetcher.MLB_PITCHER_SUMMARY_URL, player_id=self.player_id)
        else:
            f = Fetcher(Fetcher.MLB_BATTER_SUMMARY_URL, player_id=self.player_id)

        j = f.fetch()

        # if the JSON object is empty, bail
        if len(j.keys()) == 0:
            return

        # get yearly totals
        if self['primary_position'] == 1 and not self.force_batting:
            parent = j['mlb_bio_pitching_summary']['mlb_individual_pitching_season']['queryResults']
        else:
            parent = j['mlb_bio_hitting_summary']['mlb_individual_hitting_season']['queryResults']

        if parent['totalSize'] > 0:
            records = parent['row']

            # accounting for player with only one row
            if type(records) is dict:
                records = [records]

            for row in records:
                log = {}
                for key, value in row.iteritems():
                    log[key] = value

                # handle each season as a list, so
                # players with multiple team seasons
                # get each team for that year
                # accounted for
                if row['season'] in self.totals:
                    self.totals[row['season']].append(log)
                else:
                    self.totals[row['season']] = [log]

        # get career totals
        if self['primary_position'] == 1 and not self.force_batting:
            parent = j['mlb_bio_pitching_summary']['mlb_individual_pitching_career']['queryResults']
        else:
            parent = j['mlb_bio_hitting_summary']['mlb_individual_hitting_career']['queryResults']

        if parent['totalSize'] > 0:
            for key, value in parent['row'].iteritems():
                self.career[key] = value

# 需要導入模塊: from fetcher import Fetcher [as 別名]
# 或者: from fetcher.Fetcher import fetch [as 別名]
	def load(self, loadRosters = False):
		"""
		Calls MLB.com server and loads all team information

		Arguments:
		loadRosters : If true, rosters will automatically be loaded (more HTTP requests!)
		"""
		f = Fetcher(Fetcher.MLB_LEAGUE_URL)

		for item in f.fetch():
			t = team.Team(item)
			if loadRosters:
				t.loadRoster()
			self.teams[t['team_code']] = t

# 需要導入模塊: from fetcher import Fetcher [as 別名]
# 或者: from fetcher.Fetcher import fetch [as 別名]
 def loadRoster(self):
     """
     Calls MLB.com servers to obtain the complete roster for the team. If call fails, '_error' property is set.
     """
     f = Fetcher(Fetcher.MLB_ROSTER_URL, team_id=self['team_id'])
     j = f.fetch()
     
     if 'roster_40' not in j:
         self._error = "ERROR on %s: key roster_40 not found (cannot load 40 man roster)" % (f.url)            
         return False
     
     parent = j['roster_40']['queryResults']
     
     if parent['totalSize'] > 0:
         for record in parent['row']:
             player_id = record['player_id']
             self.roster[player_id] = player.Player(player_id)

# 需要導入模塊: from fetcher import Fetcher [as 別名]
# 或者: from fetcher.Fetcher import fetch [as 別名]
    def __init__(self, year, month, day=None):
        """
        Constructor
        
        Arguments:
        year: The... year!
        month: The... month!
        day: The... day! (or None for all days of the month)
        
        Schedule is a standard dictionary: each day is a key in the format of 'YYYY-MM-DD', each value
        a list of game dictionaries.
        """
        days = []
        
        if day is None:
            for d in xrange(1, calendar.mdays[month] + 1):
                days.append(datetime.date(year, month, d))
        else:
            days.append(datetime.date(year, month, day))

        for d in days:
            key = d.strftime("%Y-%m-%d")
            if key not in self.keys():
                self[key] = []

            f = Fetcher(Fetcher.MLB_SCHEDULE_URL, date=d.strftime("%Y%m%d"))
            try:
                content = f.fetch(True)
                if len(content) == 0:
                    continue
                content = re.sub(r'\t+', '\t', content)
                content = content.replace('"', '\\"')
                content = content.replace("'", "\"")
                content = re.sub(r'\t([\w,_]+):\s', r'"\1":', content)
                obj = json.loads(content)
                self[key] = obj
            except ValueError, e:
                print "ERROR %s on %s" % (e, f.url)
                pass

# 需要導入模塊: from fetcher import Fetcher [as 別名]
# 或者: from fetcher.Fetcher import fetch [as 別名]
	def loadYearlies(self):
		"""
		Loads yearly and career totals for a player
		"""
		if self['primary_position'] == 1:
			f = Fetcher(Fetcher.MLB_PITCHER_SUMMARY_URL, player_id=self.player_id)
		else:
			f = Fetcher(Fetcher.MLB_BATTER_SUMMARY_URL, player_id=self.player_id)
		
		j = f.fetch()
		
		# if the JSON object is empty, bail
		if len(j.keys()) == 0: return
		
		# get yearly totals
		if self['primary_position'] == 1:
			parent = j['mlb_bio_pitching_summary']['mlb_individual_pitching_season']['queryResults']
		else:
			parent = j['mlb_bio_hitting_summary']['mlb_individual_hitting_season']['queryResults']
		
		if parent['totalSize'] > 0:
			records = parent['row']

			# accounting for player with only one row
			if type(records) is dict: records = [records]
			
			for row in records:
				log = {}
				for key, value in row.iteritems(): log[key] = value
				self.totals[row['season']] = log
			
		# get career totals
		if self['primary_position'] == 1:
			parent = j['mlb_bio_pitching_summary']['mlb_individual_pitching_career']['queryResults']
		else:
			parent = j['mlb_bio_hitting_summary']['mlb_individual_hitting_career']['queryResults']
			
		if parent['totalSize'] > 0:
			for key, value in parent['row'].iteritems(): self.career[key] = value

# 需要導入模塊: from fetcher import Fetcher [as 別名]
# 或者: from fetcher.Fetcher import fetch [as 別名]
    def load(self, load_yearlies=False, id=None):
        """
        Calls MLB.com server and loads player information. If call fails, '_error' property is set.

        Arguments:
        id : The MLB.com player ID
        """
        if id is None and self.player_id is not None:
            id = self.player_id
            self['player_id'] = self.player_id
        else:
            raise Exception('No player_id specified')

        f = Fetcher(Fetcher.MLB_PLAYER_URL, player_id=self.player_id)
        j = f.fetch()

        try:
            records = j['player_info']['queryResults']['totalSize']
        except KeyError, e:
            msg = 'ERROR on %s: totalSize not returned for call' % f.url
            self._error = msg
            logger.error(msg)
            return False

# 需要導入模塊: from fetcher import Fetcher [as 別名]
# 或者: from fetcher.Fetcher import fetch [as 別名]
class Coffer:
    def __init__(self,config_parser):
        # Connect to engine
        database_path  = get_from_config_parser(config_parser,'Database','path','database')
        database_debug = get_boolean_from_config_parser(config_parser,'Database','debug',False)
        dir = os.path.dirname(database_path)
        if not os.path.exists(dir):
            mkdir(dir)
        sys.stderr.write('Connecting to database at "%s"\n' % database_path)
        self._engine = create_engine('sqlite:///%s' % database_path,echo=database_debug)

        # Start session
        Session = sessionmaker(bind=self._engine)
        self._session = Session()
        # Initialize feed storage
        self._feed_storage = FeedStorage(self._engine,self._session)
        # Initialize item storage
        self._item_storage = ItemStorage(self._engine,self._session)
        # A list of subprocess.Popen processes that will be maintained
        # by the Coffer object.
        self._external_processes = []
        # File storage (data dump)
        file_storage_path = get_from_config_parser(config_parser,'FileStorage','path','datadump')
        max_block_size    = get_int_from_config_parser(config_parser,'FileStorage','max-block-size',
                                                       file_storage.DEFAULT_MAX_BLOCK_SIZE)
        bzip2_path = get_from_config_parser(config_parser,'FileStorage','bzip2-path','/usr/bin/bzip2')
        self._file_storage = FileStorage(self._external_processes,file_storage_path,
                                         max_block_size,bzip2_path)
        # Content fetcher configuration
        self._fetcher = Fetcher(config_parser)

    def clone_db_session(self):
        clone_session = sessionmaker(bind=self._engine)
        return clone_session()

    def finish(self):
        '''
        Waits for all external processes started by coffer to finish.
        '''
        sys.stderr.write('Waiting for sub-processes to finish..\n')
        for process in self._external_processes:
            process.wait()
        sys.stderr.write('  ..finished.\n\n')

    def check_processes(self):
        '''
        Checks if some of the external processes have finished and
        removes them from the external-process list if they have.
        '''
        end_i = len(self._external_processes)
        i     = 0
        while i < end_i:
            if self._external_processes[i].poll() is not None:
                del self._external_processes[i]
                end_i -= 1
            else:
                i += 1

    def run_command_shell(self):
        shell = CommandShell(self)
        shell.cmdloop()

    def get_feed_info(self,url):
        '''
        Obtain information on an RSS feed, given its URL. The
        information will be obtained directly from the URL,
        not from our database. This works for feeds regardless
        of whether they are stored in our database.
        '''
        feed_results = feedparser.parse(url)
        sys.stderr.write(str(feed_results))
        if 'title' in feed_results.feed:
            return feed_results.feed.title
        else:
            return None

    def current_items_feed(self,
                           session,
                           feed,
                           enable_ad_filter = False,
                           check_existence  = False,
                           debug_enabled    = False):
        '''
        Returns a generator for the list of current items, i.e. the
        current list of fresh items returned by all known feeds.
        @param enable_ad_filter: if True, advertisements will be filtered out
                       using the predefined regex
        @param check_existence: if True, only entries that are not already
                       stored in the items database will be returned.
        '''
        if enable_ad_filter and len(feed.ad_filters) > 0:
            exclude_pattern = re.compile(u'|'.join(feed.ad_filters))
        feed_results = feedparser.parse(feed.get_url())
        for entry in feed_results.entries:
            if 'link' not in entry.keys():
                sys.stderr.write((u'No link found in this item: "%s"\n' \
                                  % entry.title).encode('utf-8'))
                if debug_enabled:
                    sys.stderr.write('Keys:\n%s\n' % str(entry.keys()))
                continue
#.........這裏部分代碼省略.........

# 需要導入模塊: from fetcher import Fetcher [as 別名]
# 或者: from fetcher.Fetcher import fetch [as 別名]
from fetcher import Fetcher
import sys

num = int( sys.argv[1] )
code = Fetcher.fetch( num )

print "----------------------------------------------------"
print 
print
print code
print "----------------------------------------------------"

# 需要導入模塊: from fetcher import Fetcher [as 別名]
# 或者: from fetcher.Fetcher import fetch [as 別名]
from fetcher import Fetcher

num = 1

while True:
    x = raw_input( ">> " )
    if x == "q":
        submission = Fetcher.fetch( num )
        if submission:
            num += 1
        else:
            print "No new submission"