Golang goriakpbc.Client类代码示例

func RssMasterPollFeeds(con *riak.Client, InputCh chan<- url.URL, OutputCh <-chan FeedError) {
	bucket, err := con.NewBucket("feeds")
	if err != nil {
		log.Println("Failed to get feed bucket:", err)
	}
	// -62135596800 is Go's zero time according to Unix's time format.  This is what empty feeds have for their check time.
	// Nothing should appear before that.
	keys_to_poll, err := bucket.IndexQueryRange(NextCheckIndexName, "-62135596800", strconv.FormatInt(time.Now().Unix(), 10))
	var errors []error

	valid_keys := 0
	for _, key := range keys_to_poll {
		var loadFeed Feed
		if err := con.LoadModel(key, &loadFeed); err != nil {
			errors = append(errors, err)
		} else {
			log.Println(loadFeed.Url)
			valid_keys++
			go func(Url url.URL, inputCh chan<- url.URL) {
				inputCh <- Url
			}(loadFeed.Url, InputCh)
		}
	}
	for i := 0; i < valid_keys; i++ {
		if err := <-OutputCh; err.Err != nil {
			errors = append(errors, err)
		}
	}
	if len(errors) != 0 {
		log.Println(MultiError(errors))
	}
}

func killBucket(con *riak.Client, bucketName string) error {
	bucket, err := con.NewBucket(bucketName)
	if err != nil {
		return err
	}

	keys, err := bucket.ListKeys()
	if err != nil {
		return err
	}

	wg := sync.WaitGroup{}
	wg.Add(len(keys))
	for _, key := range keys {
		go func(key string) {
			defer wg.Done()
			obj, err := bucket.Get(string(key))
			if obj == nil {
				panic(err)
			}
			err = obj.Destroy()
			if err != nil {
				panic(err)
			}
		}(string(key))
	}
	wg.Wait()

	return nil
}

func mustCreateEmptyItemAt(t *testing.T, con *riak.Client, itemKey ItemKey) {
	itemModel := FeedItem{}
	if err := con.LoadModel(itemKey.GetRiakKey(), &itemModel); err != riak.NotFound {
		t.Fatalf("Failed to preload item to create an empty item at %s (%s)", itemKey.GetRiakKey(), err)
	} else if err = itemModel.Save(); err != nil {
		t.Fatalf("Failed to save an empty item at %s (%s)", itemKey.GetRiakKey(), err)
	}
}

func RssMasterHandleAddRequest(con *riak.Client, Url url.URL) error {
	feedModel := &Feed{Url: Url}
	if err := con.LoadModel(feedModel.UrlKey(), feedModel); err != nil && err != riak.NotFound {
		return err
	} else if err == nil {
		return nil
	} else { // Implicitly err == riak.NotFound
		feedModel.Indexes()[NextCheckIndexName] = strconv.FormatInt(time.Time{}.Unix(), 10)
		if err = feedModel.Save(); err != nil {
			return err
		}
	}
	return nil
}

func CreateFeed(t *testing.T, con *riak.Client, Url *url.URL) *Feed {
	feedModel := &Feed{Url: *Url}
	if err := con.LoadModel(feedModel.UrlKey(), feedModel); err != nil && err != riak.NotFound {
		t.Fatalf("Failed to initialize feed model (%s)!", err)
	} else {
		modelElement := feedModel.Model
		*feedModel = Feed{Url: *Url}
		feedModel.Model = modelElement
		if err = feedModel.Save(); err != nil {
			t.Fatalf("Failed to store feed model (%s)!", err)
		}
	}
	return feedModel
}

func InsertItem(con *riak.Client, itemKey ItemKey, item ParsedFeedItem) error {
	itemModel := FeedItem{
		Title:   item.Title,
		Author:  item.Author,
		Content: item.Content,
		Url:     item.Url,
		PubDate: item.PubDate,
	}
	if err := con.LoadModel(itemKey.GetRiakKey(), &itemModel); err != riak.NotFound {
		return err
	} else if err = itemModel.Save(); err != nil {
		return err
	}
	return nil
}

func checkAllItemsDeleted(t *testing.T, itemKeyList ItemKeyList, con *riak.Client) bool {
	ch := make(chan bool)
	for _, itemKey := range itemKeyList {
		go func(itemKey ItemKey, ch chan<- bool) {
			modelItem := FeedItem{}
			if err := con.LoadModel(itemKey.GetRiakKey(), &modelItem); err == riak.NotFound {
				ch <- false
			} else {
				ch <- true
			}
		}(itemKey, ch)
	}

	problems := 0
	for _, _ = range itemKeyList {
		found := <-ch
		if found {
			problems++
		}
	}
	return problems == 0
}

func compareParsedToFinalFeed(t *testing.T, data *ParsedFeedData, model *Feed, con *riak.Client) bool {
	// Compare basics:
	if data.Title != model.Title {
		t.Errorf("Feed title didn't match '%s' vs '%s'!", data.Title, model.Title)
		return false
	}
	if !data.NextCheckTime.Equal(model.NextCheck) {
		t.Errorf("Next time to check feed doesn't match %#v vs %#v!", data.NextCheckTime, model.NextCheck)
	}
	if !(strconv.FormatInt(data.NextCheckTime.Unix(), 10) == model.Indexes()[NextCheckIndexName]) {
		t.Errorf("Next time(in 2i) to check feed doesn't match %v vs %v!", data.NextCheckTime.Unix(), model.Indexes()[NextCheckIndexName])
	}
	if !data.FetchedAt.Equal(model.LastCheck) {
		t.Errorf("Fetch time from feed doesn't match %#v vs %#v!", data.NextCheckTime, model.NextCheck)
	}

	if len(data.Items) != len(model.ItemKeys) {
		if len(data.Items) > MaximumFeedItems {
			t.Errorf("Item count differs due to items count greater then Maximum number of feed items (%v of %v)", len(data.Items), MaximumFeedItems)
		} else {
			t.Errorf("Item count is different %v vs %v!", len(data.Items), len(model.ItemKeys))
		}
		return false
	}
	if len(model.InsertedItemKeys) != 0 || len(model.DeletedItemKeys) != 0 {
		t.Error("There are left over inserted or deleted item keys!")
		return false
	}

	type FeedItemCh struct {
		item FeedItem
		ch   chan FeedItemCh
	}

	itemChOut := make(chan FeedItemCh)
	itemCh := make(chan FeedItem)

	go func(itemCh chan FeedItem, itemChOut chan FeedItemCh) {
		defer close(itemCh)
		for item, ok := <-itemChOut; ok; item, ok = <-itemChOut {
			itemCh <- item.item
			itemChOut = item.ch
		}
	}(itemCh, itemChOut)

	for _, itemKey := range model.ItemKeys {
		itemChIn := make(chan FeedItemCh)

		go func(itemKey ItemKey, itemChOut, itemChIn chan FeedItemCh) {
			defer close(itemChOut)

			modelItem := FeedItem{}
			if err := con.LoadModel(itemKey.GetRiakKey(), &modelItem, riak.R1); err != nil {
				t.Errorf("Failed to load item! Error: %s item %s", err, itemKey.GetRiakKey())
			}
			itemChOut <- FeedItemCh{modelItem, itemChIn}
		}(itemKey, itemChOut, itemChIn)

		itemChOut = itemChIn
	}
	// This ensures the feeder go routine will eventually quit.  It either closes its initial input,
	// or the final channel it will get (since itemChOut is equal to the last channel the go routine
	// will read from, since it stores the last itemChIn from above).
	close(itemChOut)

	//Compare saved feed items.  This means a trip through riak!  The order should match though ...
	for i, _ := range model.ItemKeys {
		var modelItem FeedItem
		select {
		case modelItem = <-itemCh:
		case <-time.After(time.Minute * 5):
			t.Fatalf("Failed to get an item before timeout, item %v", i)
		}

		dataItem := data.Items[i]

		if dataItem.Title != modelItem.Title ||
			dataItem.Author != modelItem.Author ||
			dataItem.Content != modelItem.Content ||
			dataItem.Url != modelItem.Url ||
			!dataItem.PubDate.Equal(modelItem.PubDate) {
			t.Errorf("Item data didn't match! Original:\n%#v\nLoaded:\n%#v", dataItem, modelItem)
			return false
		}
	}

	return true
}

func updateFeed(con *riak.Client, feedUrl url.URL, feedData ParsedFeedData, ids <-chan uint64) (*Feed, error) {
	feed := &Feed{Url: feedUrl}
	if err := con.LoadModel(feed.UrlKey(), feed); err == riak.NotFound {
		return nil, FeedNotFound
	} else if err != nil {
		return nil, err
	}
	// First clean out inserted item keys.  This handles unfinished previous operations.
	itemsBucket, err := con.Bucket("items")
	if err != nil {
		return nil, err
	}

	// Note, this insert items without caring about the 10,000 limit.  Of course, any regular inserted
	// item will force the limit back down.
	for _, itemKey := range feed.InsertedItemKeys {
		// Does this item exist?
		if ok, err := itemsBucket.Exists(itemKey.GetRiakKey()); err != nil {
			return nil, err
		} else if ok {
			// Yep, so add it to the list.
			feed.ItemKeys = append(feed.ItemKeys, itemKey)
		}
		// Otherwise non-existent items are dropped.  This is to avoid
	}
	feed.InsertedItemKeys = nil

	// Next update the basic attributes
	feed.Title = feedData.Title
	feed.NextCheck = feedData.NextCheckTime
	feed.LastCheck = feedData.FetchedAt
	// Also set 2i to appropriate values!
	feed.Indexes()[NextCheckIndexName] = strconv.FormatInt(feed.NextCheck.Unix(), 10)

	/* Next find all the feed items to insert/update.  If the item doesn't exist, create it's id and
	 * mark for insert.  Otherwise mark it for an read/update/store pass.  Make sure to mark for
	 * deletion items as necessary.
	 */
	// This struct holds an ItemKey and a ParsedFeedItem for later parsing.
	type ToProcess struct {
		ItemKey ItemKey
		Data    ParsedFeedItem
		Model   *FeedItem
	}
	NewItems := make([]ToProcess, 0)
	UpdatedItems := make([]ToProcess, 0)
	SeenNewItemKeys := make(map[string]bool)

	for _, rawItem := range feedData.Items {
		// Try to find the raw Item in the Item Keys list.
		index := feed.ItemKeys.FindRawItemId(rawItem.GenericKey)
		if index != -1 {
			// Found it!  Load the details.  Also load the model, which will be re-used later.
			p := ToProcess{
				ItemKey: feed.ItemKeys[index],
				Data:    rawItem,
				Model:   &FeedItem{},
			}

			if err := con.LoadModel(p.ItemKey.GetRiakKey(), p.Model); err != nil {
				return nil, err
			}

			// Ok, now is this have a new pub date?  If so, pull it out of its current position, and
			// move it up the chain.  Otherwise, just update the content.  If an item has no pub date,
			// assume that it has changed if the any part of the item changed.
			if p.Model.PubDate.Equal(p.Data.PubDate) && !(p.Data.PubDate.IsZero() && itemDiffersFromModel(p.Data, p.Model)) {
				// Pub dates are the same.  Just modify the item to match what is in the feed.
				UpdatedItems = append(UpdatedItems, p)
			} else {
				// Pub dates differ.  Delete the item, and re-insert it.
				feed.DeletedItemKeys = append(feed.DeletedItemKeys, p.ItemKey)
				feed.ItemKeys.RemoveAt(index)

				// Delete the model from the to process struct.
				p.Model = &FeedItem{}

				NewItems = append(NewItems, p) // This gives us the new id.
			}
		} else {
			// Nope, lets insert it!  First, should we knock off an item?  e need to stay below MaximumFeedItems.
			for (len(feed.ItemKeys)+len(NewItems)) >= MaximumFeedItems && len(feed.ItemKeys) > 0 {
				// Need to kill an item.  So get the last key
				lastKey := feed.ItemKeys[len(feed.ItemKeys)-1]
				// insert it onto the end of the deleted item list.
				feed.DeletedItemKeys = append(feed.DeletedItemKeys, lastKey)
				// If we are updating this key, then remove it from this list.  No need to waste
				// time.
				for i, item := range UpdatedItems {
					if item.ItemKey.Equal(lastKey) {
						UpdatedItems = append(UpdatedItems[:i], UpdatedItems[i+1:]...)
					}
				}
				// And finally, pop the item
				feed.ItemKeys = feed.ItemKeys[:len(feed.ItemKeys)-1]
			}
			// Only insert if there are less then MaximumFeedItems already to be inserted.
			// This works since any later item will have been updated after.
			if len(NewItems) < MaximumFeedItems {
				// Also, make sure we aren't inserting the same item twice.  If it is duplicated, the
//.........这里部分代码省略.........