Golang go-pkg-rss.New函数代码示例

func main() {
	feeds, err := hathor.GetFeeds()
	if err != nil {
		errlog <- err
	}
	fmt.Println("--- Feeds ---")
	for key := range feeds {
		fmt.Printf(" * %s\n", key)

		go func(uri string, timeout int) {
			feed := rss.New(timeout, true, chanHandler, itemHandler)
			for {
				if err := feed.Fetch(uri, nil); err != nil {
					errlog <- fmt.Errorf("%s: %s", err, uri)
					return
				}
				<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
			}
		}(feeds[key].Source, 5)
	}

	for {
		select {
		case res := <-log:
			fmt.Println(res)
		case res := <-errlog:
			fmt.Printf("[e] %s\n", res)
		}
	}
}

func main() {
	/*
	  resources := []Resource{
	    Resource{ Url: "http://zhihu.com/rss" },
	    Resource{ Url: "http://feed.zackyang.com/articles.xml" },
	  }
	*/
	for {
		resources := getResources()
		resources = []Resource{
			Resource{Url: "http://zhihu.com/rss"},
			Resource{Url: "http://feed.zackyang.com/articles.xml"},
		}
		// 获取所有订阅内容
		for _, resource := range resources {
			feed := rss.New(5, true, channelHandler, itemHandler)
			if err := feed.Fetch(resource.Url, nil); err != nil {
				fmt.Printf("Error: %s - %s", resource.Url, err)
			}
		}

		<-time.After(time.Minute * 10)
	}

}

func botPollSubscribe(bot tgbot.TgBot, msg tgbot.Message, uri string, timeout int, cr xmlx.CharsetFunc) {
	// If already a rss, only add him to the db
	// First, in cache:
	id := fmt.Sprintf("%d", msg.Chat.ID)
	ok := checkCache(uri, id)
	if ok {
		appendToCacheUsers(uri, msg.Chat.ID)
		bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
		return
	}
	ok = checkDb(uri, id)
	if ok {
		appendToCacheUsers(uri, msg.Chat.ID)
		bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
		return
	}

	firsttime := true
	n_errors := 0
	// Adding new RSS
	feed := rss.New(timeout, true, chanHandler, botItemHandler(bot, firsttime))

	for {
		if err := feed.Fetch(uri, cr); err != nil {
			// fmt.Fprintf(os.Stderr, "[e] %s: %s\n", uri, err)
			if !firsttime && n_errors < MAX_RETRIES {
				fmt.Fprintf(os.Stderr, "[e] (%d/%d) %s: %s\n", n_errors, MAX_RETRIES, uri, err)
				n_errors++
				<-time.After(time.Duration(10) * time.Second)
				continue
			} else {
				if firsttime {
					bot.Answer(msg).Text(fmt.Sprintf("Bad RSS: %s, maybe the URL is bad.\nError msg: %s", uri, err.Error())).ReplyToMessage(msg.ID).End()
				} else {
					sendToAll(bot, uri, []NewStruct{
						NewStruct{
							Text: fmt.Sprintf("Hi! I had too many failures reading the RSS %s and I'm going to remove it, you can subscribe againg using:\n/sub %s", uri, uri),
						},
					})
				}
				fmt.Fprintf(os.Stderr, "Removing url %s\n", uri)
				cleanBadUrl(uri)
				return
			}
		}

		n_errors = 0
		if firsttime {
			saveAllValues(uri, id)
			appendToCacheUsers(uri, msg.Chat.ID)
			setRssValue(buildKey("rss", uri, ""), "true")

			firsttime = false
			bot.Answer(msg).Text(fmt.Sprintf("You have been subscribed to %s", uri)).ReplyToMessage(msg.ID).End()
		}
		// Every MINS_SLEEP mins
		// <-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
		<-time.After(time.Duration(MINS_SLEEP) * time.Minute)
	}
}

func main() {
	config, err := GetConfig()
	if err != nil {
		fmt.Printf("[e] %s\n", err)
		return
	}
	for key := range config {

		go func(uri string, timeout int) {
			rssfeed := NewRssFeed(key, config[key])
			feed := rss.New(timeout, true, rssfeed.Channels, rssfeed.Items)
			for {
				if err := feed.Fetch(uri, nil); err != nil {
					fmt.Printf("[e] %s: %s\n", err, uri)
					return
				}
				update := feed.SecondsTillUpdate()
				fmt.Printf("%s - Updating again in %d seconds\n", key, update)
				<-time.After(time.Duration(update) * time.Second)
			}
		}(config[key].Source, 5)
	}

	// Wait for episodes to arrive and download 'em
	go ProcessEpisodes(download)
	ServeFeeds()
}

// NewFetcher returns a Fetcher for uri.
func NewFetcher(uri string) Fetcher {
	f := &fetcher{
		uri: uri,
	}
	newChans := func(feed *rss.Feed, chans []*rss.Channel) {}
	newItems := func(feed *rss.Feed, ch *rss.Channel, items []*rss.Item) {
		for _, item := range items {
			// Initialize guid to Atom Id
			// item.Id is empty string if item comes from RSS feed
			guid := item.Id
			// if item comes from RSS feed Guid must be provided
			if item.Guid != nil {
				guid = *(item.Guid)
			}
			f.items = append(f.items, Item{
				Channel: ch.Title,
				Title:   item.Title,
				GUID:    guid,
			})
		}
	}
	// min interval 1min, respect limits
	f.feed = rss.New(1, true, newChans, newItems)
	return f
}

// Return response
func (c *Postman) Run(user *commons.User, params []string) (string, error) {
	feed := rss.New(10, false, nil, nil)

	if err := feed.Fetch(c.cfg.Url, nil); err != nil {
		return "", err
	}

	if len(feed.Channels) == 0 {
		return "", errors.New("No items found in feed")
	}

	var response string
	for _, item := range feed.Channels[0].Items {
		if len(item.Links) == 0 {
			continue
		}

		url := item.Links[0].Href
		if _, ok := c.cache[url]; !ok {
			response = fmt.Sprintf("%s: %s", item.Title, url)
			c.cache[url] = 1
			break
		}
	}

	if response == "" {
		return "", errors.New("You've read all news. Please update feed later.")
	}

	return response, nil
}

// PollFeed fetches the podcast feed at the given uri
func PollFeed(uri string, timeout int, cr xmlx.CharsetFunc) {
	feed := rss.New(timeout, true, chanHandler, itemHandler)
	if err := feed.Fetch(uri, cr); err != nil {
		logger.Warning.Println("Feed parsing failure with "+uri, err)
		return
	}
	//<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
}

func fetch(feed sourceFeed, master *feeds.Feed) {
	fetcher := rss.New(5, true, chanHandler, makeHandler(master, feed.name))
	client := &http.Client{
		Timeout: time.Second,
	}

	fetcher.FetchClient(feed.uri, client, nil)
}

func fetchFeed(uri string, timeout int, cr xmlx.CharsetFunc) (err error) {
	feed := rss.New(timeout, true, chanHandler, itemHandler)
	err = feed.Fetch(uri, cr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "[e] %s: %s", uri, err)
		return
	}
	return
}

func (fm *FeedMailer) fetch(url string) {
	feed := rss.New(fm.prof.FetchTimeout, true, fm.chanHandler, fm.itemHandler)
	for {
		if err := feed.Fetch(url, nil); err != nil {
			log.Println(err)
		}
		<-time.After(time.Duration(feed.SecondsTillUpdate()) * time.Second)
	}
}

func readAllDbRss(bot tgbot.TgBot) {
	allrss := loadFromDbPrefix("rss:")
	// nmax := len(allrss)
	blocks := 60.0 * float64(MINS_SLEEP) // Now every second in 9 mins. Every half second in a minute, 120 blocks
	nseconds := float64(1)               //60 / float64(blocks)
	// module := int(math.Ceil(float64(nmax) / blocks))
	i := 0
	for urlkey := range allrss {
		splitted := strings.Split(urlkey, ":")
		if len(splitted) != 3 {
			continue
		}
		uri := strings.Join(splitted[1:], ":")

		j := i
		i++
		go func(uri string, firsttime bool) {
			n_errors := 0
			feed := rss.New(5, true, chanHandler, botItemHandler(bot, firsttime))

			// Start calcule by groups, calculate how many sleep before execute
			timeofsleep := math.Mod(float64(j), blocks) * nseconds // float64(int(j/module)) * nseconds
			start := time.Now()
			<-time.After(time.Duration(int(timeofsleep*1000)) * time.Millisecond)
			fmt.Printf("%d (%s) started after %v seconds\n", j, uri, time.Since(start))

			for {
				if err := feed.Fetch(uri, charsetReader); err != nil {
					// fmt.Fprintf(os.Stderr, "[e] %s: %s\n", uri, err)
					// if isAffordableNetworkError(err) && n_errors < MAX_RETRIES {
					if n_errors < MAX_RETRIES {
						fmt.Fprintf(os.Stderr, "[e] (%d/%d) %s: %s\n", n_errors, MAX_RETRIES, uri, err)
						n_errors++
						// <-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
						<-time.After(time.Duration(MINS_SLEEP) * time.Minute)
						continue
					} else {
						fmt.Fprintf(os.Stderr, "[e] (%d/%d) %s: %s\n", n_errors, MAX_RETRIES, uri, err)
						fmt.Printf("Removing url %s\n", uri)
						cleanBadUrl(uri)
						return
					}
				}
				n_errors = 0

				if firsttime {
					firsttime = false
					setRssValue(urlkey, "true")
				}
				// Every MINS_SLEEP mins
				// <-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
				<-time.After(time.Duration(MINS_SLEEP) * time.Minute)
			}
		}(uri, true)
	}
}

func fetch(url string) {
	feed := feeder.New(60, false, handleChannel, handleItem)
	err := feed.Fetch(url, nil)

	if err != nil {
		log.Println("error", url, err)
	}

	FinishedChannel <- url
}

func PollFeed(uri string, timeout int) {
	feed := rss.New(timeout, true, nil, itemHandler)
	for {
		log.Print("Polling for new posts")
		if err := feed.Fetch(uri, nil); err != nil {
			log.Printf("Error fetching Feed : %s", err)
		}
		<-time.After(time.Minute * 5)
	}
}

func PollFeed(wg *sync.WaitGroup, uri string, timeout int, cr xmlx.CharsetFunc) {
	feed := rss.New(timeout, true, chanHandler, itemHandler)
	// Decrement the counter when the goroutine completes.
	defer wg.Done()

	if err := feed.Fetch(uri, cr); err != nil {
		log.Printf("[e] %s: %s", uri, err)
		return
	}

}

func main() {
	feed := rss.New(5, true, chanHandler, itemHandler)
	for {
		refreshInterests()
		if err := feed.Fetch("http://feeds.feedburner.com/TheFlightDeal", nil); err != nil {
			fmt.Fprintf(os.Stderr, "[e] %s\n", err)
			return
		}
		<-time.After(1 * time.Hour)
	}
}