Golang AppContext.NewHttpClient方法代码示例

func updateAuctionsByIdList(appCtx *lib.AppContext, idList []*backfillTarget) {
	aidriver := NewAuctionItemDriver(appCtx)
	assocdriver := NewAssocCrawlerKeywordAuctionItemDriver(appCtx)
	crawler := yauction.NewCrawler(appCtx.NewHttpClient())
	auctionKeys := make([]*ds.Key, 0)
	auctions := make([]*yauction.AuctionItem, 0)
	assocKeys := make([]*ds.Key, 0)
	assocs := make([]*yauction.AssocCrawlerKeywordAuctionItem, 0)

	for _, target := range idList {
		auction, err := crawler.GetAuction(target.AuctionId)
		if err != nil {
			target.Error = err
			continue
		}
		auctionKeys = append(auctionKeys, aidriver.NewKey(
			auction.Id,
			0,
			nil,
		))
		auctions = append(auctions, auction)
		assocKeys = append(assocKeys, assocdriver.NewKey(
			fmt.Sprintf("%s-%s", target.Keyword, auction.Id),
			0,
			nil,
		))
		assocs = append(assocs, &yauction.AssocCrawlerKeywordAuctionItem{
			Keyword:   target.Keyword,
			AuctionId: auction.Id,
		})
	}

	aidriver.PutMulti(auctionKeys, auctions)
	assocdriver.PutMulti(assocKeys, assocs)
}

func crawlKeywords(appCtx *lib.AppContext) error {
	var keywords []yauction.CrawlerKeyword
	var logger = appCtx.Logger
	_now := now()

	ckdriver := NewCrawlerKeywordDriver(appCtx)

	q := ckdriver.NewQuery()
	q = q.Filter("EndAt >=", _now)
	_, err := q.GetAll(&keywords)
	if err != nil {
		return err
	}

	// TODO: optimize query result to remove keywords crawled within an hour.
	if len(keywords) == 0 {
		logger.Info("No keywords need to be checked")
		return nil
	}
	logger.Debug("%d Keywords start to crawl", len(keywords))
	var wg sync.WaitGroup
	auctionList := make([][]*yauction.AuctionItem, len(keywords))
	errorList := make([]error, len(keywords))
	for i, keyword := range keywords {
		wg.Add(1)
		iter := yauction.NewAuctionSearchIterator(
			yauction.NewCrawler(appCtx.NewHttpClient()),
			keyword.Keyword,
		)
		go func(j int, iter *yauction.AuctionSearchIterator) {
			list, err := iterateSearchKeyword(iter, logger)
			if err == yauction.ErrNoMoreIds {
				err = nil
				// Update Auctions
				aidriver := NewAuctionItemDriver(appCtx)
				assocdriver := NewAssocCrawlerKeywordAuctionItemDriver(appCtx)
				auctionKeys := make([]*ds.Key, len(list))
				assocKeys := make([]*ds.Key, len(list))
				assocs := make([]*yauction.AssocCrawlerKeywordAuctionItem, len(list))
				for k, auction := range list {
					auctionKeys[k] = aidriver.NewKey(auction.Id, 0, nil)
					assocKeys[k] = assocdriver.NewKey(
						fmt.Sprintf("%s-%s", iter.Keyword, auction.Id),
						0,
						nil,
					)
					assocs[k] = &yauction.AssocCrawlerKeywordAuctionItem{
						Keyword:   iter.Keyword,
						AuctionId: auction.Id,
					}
				}
				_, err = aidriver.PutMulti(auctionKeys, list)
				if err == nil {
					_, err = assocdriver.PutMulti(assocKeys, assocs)
					if err == nil {
						auctionList[j] = list
					}
				}
			}
			errorList[j] = err
			wg.Done()
		}(i, iter)
	}
	wg.Wait()

	numErrors := 0
	numUpdates := 0
	for i, list := range auctionList {
		err = errorList[i]
		if err != nil {
			logger.Error(
				"An error occurred while crawling the keyword %q: %v",
				keywords[i].Keyword,
				errorList[i],
			)
			numErrors += 1
			continue
		} else {
			numUpdates += len(list)
		}
	}
	logger.Info(
		"Finished to crawl %d keywords. Found %d AuctionItems, %d errors.",
		len(keywords), numUpdates, numErrors,
	)
	if numErrors > 0 {
		return fmt.Errorf("Keyword Crawling failed with %d errors", numErrors)
	} else {
		return nil
	}
}

func crawlClosedKeyword(appCtx *lib.AppContext, keyword string) ([]string, error) {
	var ckeyword yauction.CrawlerKeyword
	var logger = appCtx.Logger

	ckdriver := NewCrawlerKeywordDriver(appCtx)

	if err := ckdriver.Get(
		ckdriver.NewKey(keyword, 0, nil),
		&ckeyword,
	); err != nil {
		return nil, err
	}
	iter := yauction.NewAuctionClosedSearchIterator(
		yauction.NewCrawler(appCtx.NewHttpClient()),
		keyword,
	)
	logger.Info("Start Backfilling for %q", keyword)
	ids, err := iterateClosedSearchKeyword(iter, logger)
	if err != nil {
		return nil, err
	}
	logger.Info("Found %d AuctionIds", len(ids))

	// Group by 20
	updateGroups := make([][]*backfillTarget, 0)
	updateIdList := make([]*backfillTarget, 0)
	for _, id := range ids {
		updateIdList = append(updateIdList, &backfillTarget{
			AuctionId: id,
			Keyword:   keyword,
		})
		if len(updateIdList) == 20 {
			updateGroups = append(updateGroups, updateIdList)
			updateIdList = make([]*backfillTarget, 0)
		}
	}
	if len(updateIdList) > 0 {
		updateGroups = append(updateGroups, updateIdList)
	}

	var wg sync.WaitGroup
	for i, group := range updateGroups {
		wg.Add(1)
		go func(j int, idList []*backfillTarget) {
			updateAuctionsByIdList(appCtx, idList)
			wg.Done()
		}(i, group)
	}
	wg.Wait()

	result := make([]string, 0)
	numErrors := 0
	for _, group := range updateGroups {
		for _, target := range group {
			if target.Error != nil {
				numErrors += 1
			} else {
				result = append(result, target.AuctionId)
			}
		}
	}
	logger.Info(
		"Finished to update %d auctions, %d errors.",
		len(result), numErrors,
	)
	if numErrors > 0 {
		return result, fmt.Errorf("Keyword Crawling failed with %d errors", numErrors)
	} else {
		return result, nil
	}
}

func crawlExpiredAuctions(appCtx *lib.AppContext) error {
	var auctions []yauction.AuctionItem
	var logger = appCtx.Logger
	_now := now()

	aidriver := NewAuctionItemDriver(appCtx)

	q := aidriver.NewQuery()
	q = q.Filter("EndAt <", _now).Filter("Status =", yauction.STATUS_OPEN).Limit(50)
	keys, err := q.GetAll(&auctions)
	if err != nil {
		return err
	}
	if len(auctions) == 0 {
		logger.Info("No auctions need to be checked")
		return nil
	}

	logger.Debug("Found %d expired auctions", len(auctions))

	var wg sync.WaitGroup
	errorList := make([]error, len(auctions))
	for i, auction := range auctions {
		wg.Add(1)
		go func(j int, origAuction *yauction.AuctionItem) {
			crawler := yauction.NewCrawler(appCtx.NewHttpClient())
			_auction, err := crawler.GetAuction(auction.Id)
			if err == nil {
				if _auction.Status == yauction.STATUS_DELETED {
					origAuction.Status = yauction.STATUS_DELETED
				} else {
					origAuction.Title = _auction.Title
					origAuction.Url = _auction.Url
					origAuction.ImageUrl = _auction.ImageUrl
					origAuction.Seller = _auction.Seller
					origAuction.InitPrice = _auction.InitPrice
					origAuction.CurrentPrice = _auction.CurrentPrice
					origAuction.BidOrBuy = _auction.BidOrBuy
					origAuction.Quantity = _auction.Quantity
					origAuction.Bids = _auction.Bids
					origAuction.StartAt = _auction.StartAt
					origAuction.EndAt = _auction.EndAt
					origAuction.UpdatedAt = now()
					origAuction.Status = _auction.Status
				}
				_, err = aidriver.Put(keys[j], origAuction)
			}
			errorList[j] = err
			wg.Done()
		}(i, &auction)
	}
	wg.Wait()
	numErrors := 0
	numUpdates := 0
	for i, auction := range auctions {
		err = errorList[i]
		if err != nil {
			logger.Error(
				"An error occurred while updating an auction %q: %v",
				auction.Id,
				errorList[i],
			)
			numErrors += 1
			continue
		} else {
			numUpdates += 1
		}
	}
	logger.Info(
		"Finished to update %d auctions, %d errors.",
		numUpdates, numErrors,
	)
	if numErrors > 0 {
		return fmt.Errorf("Crawling expired auctions failed with %d errors", numErrors)
	} else {
		return nil
	}
}