Golang rss.Fetch函数代码示例

func doWork() (err error) {

	feed, err := rss.Fetch(BBC_FEED_URL)
	if err != nil {
		return err
	}

	for _, item := range feed.Items[:10] {
		if _, ok := visited[item.ID]; ok {
			continue
		}
		n := shruti.Notification{
			Title:        item.Title,
			Url:          item.Link,
			Key:          PROVIDER_NAME + item.ID,
			Priority:     shruti.PRIO_MED,
			Action:       shruti.ACT_PUSH,
			ProviderName: PROVIDER_NAME,
		}
		err = sClient.PushNotification(n)
		msg := "submitted"
		if err != nil {
			msg = err.Error()
		}
		visited[item.ID] = true
		log.Println(msg)
	}

	return nil
}

// RSS抓取
func FetchRss(url string, category string) {
	feed, err := rss.Fetch(url)
	if err != nil {
		return
	}

	err = feed.Update()
	if err != nil {
		return
	}

	for k, _ := range feed.Items {
		words := strings.TrimSpace(feed.Items[k].Title)

		good, details, wordSlice := Scoring(words)

		result := &models.YuqingResult{}

		// 插入到结果表
		result.Id = words
		result.Category = category
		result.Good = good
		result.Detail = details
		result.WordSlice = wordSlice
		result.Insert()
	}
}

func main() {
	urls := []string{
		"http://rss.slashdot.org/Slashdot/slashdotMain",
		"http://www.osnews.com/files/recent.xml",
	}

	feeds := make([]*rss.Feed, 0)

	for _, url := range urls {
		feed, err := rss.Fetch(url)
		if err != nil {
			fmt.Printf("%v: %v\n", url, err)
		}

		feeds = append(feeds, feed)

		err = feed.Update()
		if err != nil {
			fmt.Printf("%v: %v\n", feed.Link, err)
		}
	}

	fmt.Println("LATEST NEWS\n")
	t := template.Must(template.New("webpage").Parse(TEMPLATE))
	for _, feed := range feeds {
		err := t.Execute(os.Stdout, feed)
		if err != nil {
			fmt.Printf("error executing template: %v\n", err)
		}
	}
}

func asyncFetchFeeds(feeds []*rss.Feed) []*HttpResponse {
	ch := make(chan *HttpResponse)
	responses := []*HttpResponse{}

	for _, feed := range feeds {
		go func(f *rss.Feed) {
			fmt.Printf("Fetching %s\n", f.UpdateURL)
			feed, err := rss.Fetch(f.UpdateURL)

			if err != nil {
				fmt.Printf("Error in response %s. Using old feed.\n", err)
				ch <- &HttpResponse{f, err}
			} else {
				ch <- &HttpResponse{feed, err}
			}

		}(feed)
	}

	for {
		select {
		case r := <-ch:
			fmt.Printf("%s was fetched\n", r.feed.UpdateURL)
			responses = append(responses, r)
			if len(responses) == len(feeds) {
				return responses
			}
		case <-time.After(5 * time.Second):
			return responses
		}
	}
}

// newfeedsplitter constructs a new feedsplitter
func newfeedsplitter(conn irc.SafeConn, url, name, color string, freq time.Duration, channels []string) (*feedspitter, error) {
	f, err := rss.Fetch(url)

	if err != nil {
		return nil, err
	}

	fs := &feedspitter{
		conn:     conn,
		feed:     f,
		name:     name,
		color:    color,
		freq:     freq,
		seen:     make(map[string]bool),
		channels: channels,
		stop:     make(chan bool),
	}

	// initial sweep of seen items
	for _, i := range fs.feed.Items {
		fs.seen[i.ID] = true
	}

	return fs, nil
}

func getNewRedditPosts(feedUrl, source string) ([]*models.Post, error) {
	conn := models.CloneConnection()
	defer conn.Close()

	// get the feed
	feed, err := rss.Fetch(feedUrl)
	if err != nil {
		return []*models.Post{}, err
	}

	// iterate the feed and save the new items
	posts := make([]*models.Post, 0)
	for _, item := range feed.Items {
		exists, err := models.PostExists(conn, item.ID)
		if err != nil {
			return []*models.Post{}, err
		}

		if !exists {
			post := &models.Post{
				Title:       item.Title,
				Source:      source,
				Url:         item.Link,
				UniqueId:    item.ID,
				PublishedAt: item.Date,
			}

			posts = append(posts, post)
		}
	}

	return posts, nil
}

func getFeed(w http.ResponseWriter, r *http.Request) {
	feedName := chi.URLParam(r, "feedName")

	feedSettings, err := config.LoadWidget(feedName)
	if err != nil {
		render.JSON(w, r, map[string]interface{}{"success": false, "errors": []string{err.Error()}})
		return
	}

	var items config.FeedItems

	feed, err := rss.Fetch(feedSettings.RssURL)
	if err != nil {
		render.JSON(w, r, map[string]interface{}{"success": false, "errors": []string{err.Error()}})
		return
	}

	for _, i := range feed.Items {
		items.ParseItem(i)
	}

	items, err = config.SaveFeed(feedName, items, feedSettings.Count)
	if err != nil {
		render.JSON(w, r, map[string]interface{}{"success": false, "errors": []string{err.Error()}})
		return
	}

	render.JSON(w, r, map[string]interface{}{"feed": items, "success": true})
}

func AddFeed(writer http.ResponseWriter, req *http.Request) {
	body, _ := ioutil.ReadAll(req.Body)
	var objmap map[string]string
	json.Unmarshal([]byte(body), &objmap)

	var url = objmap["url"]
	feed, err := rss.Fetch(url)

	if err != nil {
		fmt.Printf("Error fetching feed %s\n", err)
		writer.WriteHeader(http.StatusInternalServerError)
		return
	}

	err = db.Add(feed)

	if err != nil {
		fmt.Printf("%s\n", err)
		writer.WriteHeader(http.StatusBadRequest)
		return
	}

	writer.WriteHeader(http.StatusCreated)
	jsonResult, _ := json.Marshal(feed)
	fmt.Fprintf(writer, string(jsonResult))
}

func init() {
	var err error
	news, err = rss.Fetch("http://ranssi.paivola.fi/rss.php")
	if err != nil {
		log.Errorf("[Posts] Failed to load front-page posts: %s", err)
	}
	lastupdate = util.Timestamp()
}

func fetchFeed(url string, ch chan *rss.Feed) {
	fmt.Printf("Fetching %s \n", url)

	feed, err := rss.Fetch(url)

	if err != nil {
		fmt.Println("Failed to get feed %s\n", url)
	}

	ch <- feed
}

func GetRss(entry Entry) (to_scrape []Target) {
	feed, err := rss.Fetch(entry.Url)
	if err != nil {
		fmt.Println("Entry:", entry.Name, "Error:", err)
	} else {
		for _, item := range feed.Items {
			scrape_item := Target{item.Link, item.Title, entry.Language, item.Date}
			to_scrape = append(to_scrape, scrape_item)
		}
	}
	return
}

func checkFeed(rssURL string) (newEntryArr []*feedEl, err error) {

	if isFirstTime {
		isFirstTime = false
		_, err := rss.Fetch(rssURL)
		if err != nil {
			return nil, err
		}
	}
	log.Println("Checking for new rss items...")

	feed, err := rss.Fetch(rssURL)
	if err != nil {
		return nil, err
	}

	for _, e := range feed.Items {
		newEntryArr = append(newEntryArr,
			&feedEl{time: e.Date, title: e.Title, url: e.Link})
	}
	return
}

func main() {
	if len(os.Args) == 1 {
		fmt.Fprintf(os.Stderr, "Usage:\n\t%s [URL]\n", filepath.Base(os.Args[0]))
		os.Exit(2)
	}

	feed, err := rss.Fetch(os.Args[1])
	if err != nil {
		panic(err)
	}

	fmt.Printf("%+v\n", feed)
}

func (c City) Search(category, query string) ([]Post, error) {
	feed, err := rss.Fetch(fmt.Sprintf(searchCityStr, c.URL, category, query))
	if err != nil {
		fmt.Println(fmt.Sprintf(searchCityStr, c.URL, category, query))
		return nil, err
	}

	results := make([]Post, 0)
	for _, item := range feed.Items {
		results = append(results, newPost(item))
	}
	return results, nil
}

func (feed *Feed) Fetch() error {
	var err error
	logger.Info("Started fetching feed: %s", feed.Url)

	rss.CacheParsedItemIDs(false)
	feed.RawData, err = rss.Fetch(feed.Url)
	if err != nil {
		return err
	}

	logger.Info("Finished fetching feed: %s", feed.Url)
	logger.Info("There are %d items in %s", len(feed.RawData.Items), feed.Url)
	return nil
}