Python fetcher.Fetcher类代码示例

    def loadGamelogs(self, year=None):
        """
        Loads gamelogs for the player for a given year

        Arguments:
        year : The season desired. Defaults to the current year if not specified
        """
        if year is None: year = datetime.datetime.now().year
        if year not in self.logs: self.logs[year] = []

        if 'primary_position' not in self:
            logger.error("no primary position attribute for " % self)
            return False

        url = Fetcher.MLB_PITCHER_URL if self['primary_position'] == 1 else Fetcher.MLB_BATTER_URL

        f = Fetcher(url, player_id=self.player_id, year=year)
        j = f.fetch()

        try:
            if self['primary_position'] == 1:
                parent = j['mlb_bio_pitching_last_10']['mlb_individual_pitching_game_log']['queryResults']
            else:
                if 'mlb_individual_hitting_last_x_total' in j:
                    parent = j['mlb_individual_hitting_last_x_total']['mlb_individual_hitting_game_log']['queryResults']
                else:
                    parent = j['mlb_bio_hitting_last_10']['mlb_individual_hitting_game_log']['queryResults']
        except KeyError, e:
            logger.error('no key for gamelogs found in %s' % f.url)
            return False

def run(limit = -1):
  if limit != -1 and store.count() > limit:
    print 'Limit: ' + str(limit) + ' Count: ' + str(store.count())
    print 'Voila, We are done !!'
    import sys
    sys.exit()
    return

  if swap.current.is_empty():
    if swap.next.is_empty() :
      print 'Somehow, We are done'
    else:
      # we exchange current and new, run again
      swap.exchange()
      return run(limit)
    

  #get the url from current-to be crawled list of urls
  url = swap.current.pop()
  
  if not url or not valid_url(url):
    print '"'+url+'" is not a valid url, giving up on it'
    return run(limit)

  f = Fetcher(url)

  next_urls = store.process(f.url(), f.child_urls())
  
  for url in next_urls:
    swap.next.push(url)

  return run(limit)

	def start(self):
		url_queue = Queue.Queue()
		url_queue.put((self.root_request_info.url, 0))

		request_info = RequestInfo('', None, self.root_request_info.headers)
		fetcher = Fetcher()

		while not url_queue.empty():
			curr_url, depth = url_queue.get()		
			#print 'url=%s, depth=%d' % (curr_url, depth)
			print curr_url

			if depth > self.depth_limit:
				continue
			
			depth += 1
			request_info.url = curr_url
			page_content = fetcher.request(request_info)

			## parse page
			## Content.parse(page_content)

			url_list = HtmlParser.extract_url(curr_url, page_content)
			if url_list:
				for url in url_list:
					url_queue.put((url, depth))

	def __init__(self, year, month, day = None):
		"""
		Constructor
		
		Arguments:
		year: The... year!
		month: The... month!
		day: The... day! (or None for all days of the month)
		"""
		days = []
		
		if day is None:
			for d in xrange(1, calendar.mdays[month] + 1):
				days.append(datetime.date(year, month, d))
		else:
			days.append(datetime.date(year, month, day))

		begin = days[0]
		end = days[-1]

		f = Fetcher(Fetcher.MLB_TRANSACTION_URL, start=begin.strftime("%Y%m%d"), end=end.strftime("%Y%m%d"))
		try:
		 	obj = f.fetch()
			if obj['transaction_all']['queryResults']['totalSize'] == 0: return
			results = obj['transaction_all']['queryResults']['row']
			
			if type(results) is dict:
				self.append(results)
			else:
				for row in results:
					self.append(row)
		except (ValueError, KeyError), e:
			logger.error("ERROR %s on %s" % (e, f.url))
			pass

class Tips(object):
    """ Manage Tips Events. """

    def __init__(self, enable):
        self.enable = enable
        self._tips = {}
        self._new_tips = set()
        self.lock = Lock()
        if self.enable:
            self.fetcher = Fetcher(self._tips, self.lock, self._new_tips)
            self.cleaner = Cleaner(self._tips, self.lock, self._new_tips)
            self.fetcher.start()
            self.cleaner.start()

    def tips(self):
        return self._tips.values()

    def new_tips(self):
        if self._new_tips:
            wait_free_acquire(self.lock)
            res = [self._tips[x] for x in self._new_tips]
            self._new_tips.clear()
            self.lock.release()
            return res
        else:
            return []

    def stop(self):
        if self.enable:
            self.fetcher.finnish()
            self.cleaner.finnish()

class GlobalPlugin(globalPluginHandler.GlobalPlugin):

	def __init__(self):
		super(globalPluginHandler.GlobalPlugin, self).__init__()

		# Creating the configuration directory
		configDir = os.path.join(config.getUserDefaultConfigPath(),
				"owm")
		if not os.path.exists(configDir):
			os.mkdir(configDir)

		# Add the settings in the NVDA menu
		self.prefsMenu = gui.mainFrame.sysTrayIcon.menu.GetMenuItems()[0].GetSubMenu()
		self.owmSettingsItem = self.prefsMenu.Append(wx.ID_ANY, "OWM Settings...", "Set OWM location")
		gui.mainFrame.sysTrayIcon.Bind(wx.EVT_MENU, self.onOWMSettings, self.owmSettingsItem)

		# Create the client to retrieve information from the API
		self.fetcher = Fetcher()
		self.fetcher.start()

	def script_announceOWMForecast(self, gesture):
		if self.fetcher.client is None:
			ui.message("Loading, please wait and try again in a few seconds...")
			return

		client = self.fetcher.client
		if client.error:
			ui.message("{0} {1}".format(client.statusCode, client.errorReason))
			self.fetcher.valid = False
			self.fetcher = Fetcher()
			self.fetcher.start()
		else:
			forecast = client.forecast
			message = forecast.getMessage()
			ui.message(message)
			log.info(message)

	def onOWMSettings(self, event):
		"""Pop a dialog with OWM settings."""
		locations = locationList.retrieve()
		selected = configFile['location']
		locationName = locationList.get(selected).name
		locationValues = {}
		for location in locations:
			locationValues[location.id] = (location.name, location.country)

		dialog = LocationDialog(gui.mainFrame, -1, "Select OWM Location",
				locationValues, locationName)
		gui.mainFrame.prePopup()
		ret = dialog.ShowModal()
		gui.mainFrame.postPopup()
		if ret == wx.ID_OK:
			log.info("Focused {0}, {1}".format(locationList.path,
					dialog.location.focusedLocationName))

	__gestures={
			"kb:NVDA+w": "announceOWMForecast",
	}

def update(new=True, begin=0, end=0, ids=[]):
    fetcher = Fetcher()
    if new: 
        count=fetcher.fetchNew()
    elif len(ids) == 0:
        count=fetcher.fetchAll(begin, end)
    else:
        count=fetcher.fetchSelected(ids)
    print "%d items updated" % count

def main():
    url_file = 'url.conf'
    url_list = get_url_list(url_file)
    cache = PriorityCache()
    fetchers = []
    for url in url_list:
        f = Fetcher(cache, url)
        fetchers.append(f)

    for f in fetchers:
        f.run()

    def loadYearlies(self):
        """
        Loads yearly and career totals for a player
        """
        if self['primary_position'] == 1 and not self.force_batting:
            f = Fetcher(Fetcher.MLB_PITCHER_SUMMARY_URL, player_id=self.player_id)
        else:
            f = Fetcher(Fetcher.MLB_BATTER_SUMMARY_URL, player_id=self.player_id)

        j = f.fetch()

        # if the JSON object is empty, bail
        if len(j.keys()) == 0:
            return

        # get yearly totals
        if self['primary_position'] == 1 and not self.force_batting:
            parent = j['mlb_bio_pitching_summary']['mlb_individual_pitching_season']['queryResults']
        else:
            parent = j['mlb_bio_hitting_summary']['mlb_individual_hitting_season']['queryResults']

        if parent['totalSize'] > 0:
            records = parent['row']

            # accounting for player with only one row
            if type(records) is dict:
                records = [records]

            for row in records:
                log = {}
                for key, value in row.iteritems():
                    log[key] = value

                # handle each season as a list, so
                # players with multiple team seasons
                # get each team for that year
                # accounted for
                if row['season'] in self.totals:
                    self.totals[row['season']].append(log)
                else:
                    self.totals[row['season']] = [log]

        # get career totals
        if self['primary_position'] == 1 and not self.force_batting:
            parent = j['mlb_bio_pitching_summary']['mlb_individual_pitching_career']['queryResults']
        else:
            parent = j['mlb_bio_hitting_summary']['mlb_individual_hitting_career']['queryResults']

        if parent['totalSize'] > 0:
            for key, value in parent['row'].iteritems():
                self.career[key] = value

	def load(self, loadRosters = False):
		"""
		Calls MLB.com server and loads all team information

		Arguments:
		loadRosters : If true, rosters will automatically be loaded (more HTTP requests!)
		"""
		f = Fetcher(Fetcher.MLB_LEAGUE_URL)

		for item in f.fetch():
			t = team.Team(item)
			if loadRosters:
				t.loadRoster()
			self.teams[t['team_code']] = t

 def handle(self):
     # self.request is the TCP socket connected to the client
     data = self.request.recv(1024).strip()
     if data.startswith("GET /favicon.ico"):
         return
     print '--------------data---------------- ' + data
     query_components = get_query_parameters(data)
     print "query %s" % query_components
     # just send back the same data, but upper-cased
     # date_days_ago = Fetcher.DEFAULT_DATE
     result = {"result":"empty"}
     if "days" in query_components:
         from fetcher import Fetcher
         date_days_ago = float(query_components["days"][0])
         result = Fetcher.get_ranked_pages(date_days_ago)
     elif "category" in query_components:
         sort = "score"
         if "sort" in query_components:
             sort = query_components["sort"][0]
         result = social_db.read_from_spikedate(query_components["category"][0], sort)
     elif "channel" in query_components:
         channel = query_components["channel"][0]
         result = get_channel_serving.sort_channel_by_field(channel, max_count=100, remove_adult=False)
     elif "translate" in query_components:
         text = query_components["translate"][0]
         if self.token_expired is None or self.token_expired:
             self.update_token()
             self.token_expired = False
         if "from" in query_components and "to" in query_components:
             from_lang = query_components["from"][0]
             to_lang = query_components["to"][0]
             result = MTPythonSampleCode.translate(self.final_token, textToTranslate=text, fromLangCode=from_lang, toLangCode=to_lang)
         else:
             result = MTPythonSampleCode.translate(self.final_token, text)
     self.request.sendall(json.dumps(result, encoding="utf-8", ensure_ascii=False))

    def __init__(self,config_parser):
        # Connect to engine
        database_path  = get_from_config_parser(config_parser,'Database','path','database')
        database_debug = get_boolean_from_config_parser(config_parser,'Database','debug',False)
        dir = os.path.dirname(database_path)
        if not os.path.exists(dir):
            mkdir(dir)
        sys.stderr.write('Connecting to database at "%s"\n' % database_path)
        self._engine = create_engine('sqlite:///%s' % database_path,echo=database_debug)

        # Start session
        Session = sessionmaker(bind=self._engine)
        self._session = Session()
        # Initialize feed storage
        self._feed_storage = FeedStorage(self._engine,self._session)
        # Initialize item storage
        self._item_storage = ItemStorage(self._engine,self._session)
        # A list of subprocess.Popen processes that will be maintained
        # by the Coffer object.
        self._external_processes = []
        # File storage (data dump)
        file_storage_path = get_from_config_parser(config_parser,'FileStorage','path','datadump')
        max_block_size    = get_int_from_config_parser(config_parser,'FileStorage','max-block-size',
                                                       file_storage.DEFAULT_MAX_BLOCK_SIZE)
        bzip2_path = get_from_config_parser(config_parser,'FileStorage','bzip2-path','/usr/bin/bzip2')
        self._file_storage = FileStorage(self._external_processes,file_storage_path,
                                         max_block_size,bzip2_path)
        # Content fetcher configuration
        self._fetcher = Fetcher(config_parser)

    def on_search(self, w):
        mode = self.mode.get_active()
        txt = self.entry.get_text()
        self.musiclist.set_loading(False)
        self.musiclist.empty_message = "Searching..."
        self.musiclist.get_model().clear()

        if self.fetcher:
            self.fetcher.stop()
            self.fetcher = None

        itemgen = None
        if mode == 0:
            itemgen = lambda: jamaendo.search_artists(query=txt)
        elif mode == 1:
            itemgen = lambda: jamaendo.search_albums(query=txt)
        elif mode == 2:
            itemgen = lambda: jamaendo.search_tracks(query=txt)
        else:
            return

        self.fetcher = Fetcher(itemgen, self,
                               on_item = self.on_add_result,
                               on_ok = self.on_add_complete,
                               on_fail = self.on_add_complete)
        self.fetcher.start()
        '''

class RadiosWindow(hildon.StackableWindow):
    def __init__(self):
        hildon.StackableWindow.__init__(self)
        self.fetcher = None
        self.radios = {}

        self.set_title("Radios")
        self.connect('destroy', self.on_destroy)

        # Results list
        self.panarea = hildon.PannableArea()
        self.radiolist = RadioList()
        self.radiolist.connect('row-activated', self.row_activated)
        self.panarea.add(self.radiolist)
        self.add(self.panarea)

        self.start_radio_fetcher()

    def on_destroy(self, wnd):
        if self.fetcher:
            self.fetcher.stop()
            self.fetcher = None

    def row_activated(self, treeview, path, view_column):
        name, _id = self.radiolist.get_radio_id(path)
        wnd = open_playerwindow()
        wnd.play_radio(name, _id)

    def start_radio_fetcher(self):
        if self.fetcher:
            self.fetcher.stop()
            self.fetcher = None
        self.fetcher = Fetcher(jamaendo.starred_radios, self,
                               on_item = self.on_radio_result,
                               on_ok = self.on_radio_complete,
                               on_fail = self.on_radio_complete)
        self.fetcher.start()

    def on_radio_result(self, wnd, item):
        if wnd is self:
            self.radios[item.ID] = item
            self.radiolist.add_radios([item])

    def on_radio_complete(self, wnd, error=None):
        if wnd is self:
            self.fetcher.stop()
            self.fetcher = None

 def loadRoster(self):
     """
     Calls MLB.com servers to obtain the complete roster for the team. If call fails, '_error' property is set.
     """
     f = Fetcher(Fetcher.MLB_ROSTER_URL, team_id=self['team_id'])
     j = f.fetch()
     
     if 'roster_40' not in j:
         self._error = "ERROR on %s: key roster_40 not found (cannot load 40 man roster)" % (f.url)            
         return False
     
     parent = j['roster_40']['queryResults']
     
     if parent['totalSize'] > 0:
         for record in parent['row']:
             player_id = record['player_id']
             self.roster[player_id] = player.Player(player_id)