Golang DomainDAO.SaveMany方法代碼示例

func calculateDomainDAODurations(domainDAO dao.DomainDAO, numberOfItems int) (totalDuration, insertDuration,
	queryDuration, removeDuration time.Duration) {

	beginTimer := time.Now()
	sectionTimer := beginTimer

	// Build array to create many at once
	var domains []*model.Domain
	for i := 0; i < numberOfItems; i++ {
		domain := model.Domain{
			FQDN: fmt.Sprintf("test%d.com.br", i),
		}

		domains = append(domains, &domain)
	}

	errorInDomainsCreation := false
	domainResults := domainDAO.SaveMany(domains)

	// Check if there was any error while creating them
	for _, domainResult := range domainResults {
		if domainResult.Error != nil {
			errorInDomainsCreation = true
			utils.Errorln(fmt.Sprintf("Couldn't save domain %s in database during the performance test",
				domainResult.Domain.FQDN), domainResult.Error)
		}
	}

	if errorInDomainsCreation {
		utils.Fatalln("Due to errors in domain creation, the performance test will be aborted", nil)
	}

	insertDuration = time.Since(sectionTimer)
	sectionTimer = time.Now()

	// Try to find domains from different parts of the whole range to check indexes
	queryRanges := numberOfItems / 4
	fqdn1 := fmt.Sprintf("test%d.com.br", queryRanges)
	fqdn2 := fmt.Sprintf("test%d.com.br", queryRanges*2)
	fqdn3 := fmt.Sprintf("test%d.com.br", queryRanges*3)

	if _, err := domainDAO.FindByFQDN(fqdn1); err != nil {
		utils.Fatalln(fmt.Sprintf("Couldn't find domain %s in database during "+
			"the performance test", fqdn1), err)
	}

	if _, err := domainDAO.FindByFQDN(fqdn2); err != nil {
		utils.Fatalln(fmt.Sprintf("Couldn't find domain %s in database during "+
			"the performance test", fqdn2), err)
	}

	if _, err := domainDAO.FindByFQDN(fqdn3); err != nil {
		utils.Fatalln(fmt.Sprintf("Couldn't find domain %s in database during "+
			"the performance test", fqdn3), err)
	}

	queryDuration = time.Since(sectionTimer)
	sectionTimer = time.Now()

	errorInDomainsRemoval := false
	domainResults = domainDAO.RemoveMany(domains)

	// Check if there was any error while removing them
	for _, domainResult := range domainResults {
		if domainResult.Error != nil {
			errorInDomainsRemoval = true
			utils.Errorln(fmt.Sprintf("Error while trying to remove a domain %s during the performance test",
				domainResult.Domain.FQDN), domainResult.Error)
		}
	}

	if errorInDomainsRemoval {
		utils.Fatalln("Due to errors in domain removal, the performance test will be aborted", nil)
	}

	removeDuration = time.Since(sectionTimer)
	totalDuration = time.Since(beginTimer)

	return
}

// Test all phases from a domain lyfe cycle, but now working with a group of domains
func domainsLifeCycle(domainDAO dao.DomainDAO) {
	domains := newDomains()

	// Create domains
	domainResults := domainDAO.SaveMany(domains)

	for _, domainResult := range domainResults {
		if domainResult.Error != nil {
			utils.Fatalln(fmt.Sprintf("Couldn't save domain %s in database",
				domainResult.Domain.FQDN), domainResult.Error)
		}
	}

	for _, domain := range domains {
		// Search and compare created domains
		if domainRetrieved, err := domainDAO.FindByFQDN(domain.FQDN); err != nil {
			utils.Fatalln(fmt.Sprintf("Couldn't find created domain %s in database", domain.FQDN), err)

		} else if !utils.CompareDomain(*domain, domainRetrieved) {
			utils.Fatalln(fmt.Sprintf("Domain %s created is being persisted wrongly", domain.FQDN), nil)
		}
	}

	// Update domains
	for _, domain := range domains {
		domain.Owners = []model.Owner{}
	}

	domainResults = domainDAO.SaveMany(domains)

	for _, domainResult := range domainResults {
		if domainResult.Error != nil {
			utils.Fatalln(fmt.Sprintf("Couldn't update domain %s in database",
				domainResult.Domain.FQDN), domainResult.Error)
		}
	}

	for _, domain := range domains {
		// Search and compare updated domains
		if domainRetrieved, err := domainDAO.FindByFQDN(domain.FQDN); err != nil {
			utils.Fatalln(fmt.Sprintf("Couldn't find updated domain %s in database", domain.FQDN), err)

		} else if !utils.CompareDomain(*domain, domainRetrieved) {
			utils.Fatalln(fmt.Sprintf("Domain %s updated in being persisted wrongly", domain.FQDN), nil)
		}
	}

	// Check if find all really return all domains
	allDomainsChannel, err := domainDAO.FindAllAsync()
	if err != nil {
		utils.Fatalln("Error while retrieving all domains from database", err)
	}

	var allDomains []model.Domain
	for {
		domainRetrieved := <-allDomainsChannel
		if domainRetrieved.Error != nil {
			utils.Fatalln("Error while retrieving all domains from database", err)
		} else if domainRetrieved.Domain == nil {
			break
		}

		allDomains = append(allDomains, *domainRetrieved.Domain)
	}

	if len(allDomains) != len(domains) {
		utils.Fatalln(fmt.Sprintf("FindAll method is not returning all domains we expected %d but got %d",
			len(domains), len(allDomains)), nil)
	}

	// Detected a problem in FindAsync method on 2014-01-17 where we were returning the same
	// object many times because we were reusing the same pointer. For that reason we are
	// going to add a test to check if the items returned are the same set of the inserted
	// ones
	for _, domain := range domains {
		found := false
		for _, domainRetrieved := range allDomains {
			if domainRetrieved.Id.Hex() == domain.Id.Hex() {
				found = true
				break
			}
		}

		if !found {
			utils.Fatalln("FindAll method is not returning all objects "+
				"that were inserted, apparently there are duplicated objects in the result set", nil)
		}
	}

	// Remove domains
	domainResults = domainDAO.RemoveMany(domains)

	for _, domainResult := range domainResults {
		if domainResult.Error != nil {
			utils.Fatalln(fmt.Sprintf("Error while trying to remove domain %s from database",
				domainResult.Domain.FQDN), domainResult.Error)
		}
	}

//.........這裏部分代碼省略.........

func domainsExpand(domainDAO dao.DomainDAO) {
	newDomains := newDomains()
	domainsResult := domainDAO.SaveMany(newDomains)
	for _, domainResult := range domainsResult {
		if domainResult.Error != nil {
			utils.Fatalln("Error creating domains", domainResult.Error)
		}
	}

	pagination := dao.DomainDAOPagination{}
	domains, err := domainDAO.FindAll(&pagination, false, "")

	if err != nil {
		utils.Fatalln("Error retrieving domains", err)
	}

	for _, domain := range domains {
		if len(domain.Owners) > 0 {
			utils.Fatalln("Not compressing owners in results", nil)
		}

		for _, nameserver := range domain.Nameservers {
			if len(nameserver.Host) > 0 ||
				nameserver.IPv4 != nil ||
				nameserver.IPv6 != nil ||
				!nameserver.LastCheckAt.Equal(time.Time{}) ||
				!nameserver.LastOKAt.Equal(time.Time{}) {
				utils.Fatalln("Not compressing nameservers in results", nil)
			}
		}

		for _, ds := range domain.DSSet {
			if ds.Algorithm != 0 ||
				len(ds.Digest) > 0 ||
				ds.DigestType != 0 ||
				ds.Keytag != 0 ||
				!ds.ExpiresAt.Equal(time.Time{}) ||
				!ds.LastCheckAt.Equal(time.Time{}) ||
				!ds.LastOKAt.Equal(time.Time{}) {
				utils.Fatalln("Not compressing ds set in results", nil)
			}
		}
	}

	domains, err = domainDAO.FindAll(&pagination, true, "")

	if err != nil {
		utils.Fatalln("Error retrieving domains", err)
	}

	for _, domain := range domains {
		if len(domain.Owners) == 0 {
			utils.Fatalln("Compressing owners in results when it shouldn't", nil)
		}

		for _, nameserver := range domain.Nameservers {
			if len(nameserver.Host) == 0 ||
				nameserver.IPv4 == nil ||
				nameserver.IPv6 == nil ||
				nameserver.LastCheckAt.Equal(time.Time{}) ||
				nameserver.LastOKAt.Equal(time.Time{}) ||
				nameserver.LastStatus != model.NameserverStatusOK {
				utils.Fatalln("Compressing nameservers in results when it shouldn't", nil)
			}
		}

		for _, ds := range domain.DSSet {
			if ds.Algorithm == 0 ||
				len(ds.Digest) == 0 ||
				ds.DigestType == 0 ||
				ds.Keytag == 0 ||
				ds.ExpiresAt.Equal(time.Time{}) ||
				ds.LastCheckAt.Equal(time.Time{}) ||
				ds.LastOKAt.Equal(time.Time{}) ||
				ds.LastStatus != model.DSStatusOK {
				utils.Fatalln("Compressing ds set in results when it shouldn't", nil)
			}
		}
	}

	domainsResult = domainDAO.RemoveMany(newDomains)
	for _, domainResult := range domainsResult {
		if domainResult.Error != nil {
			utils.Fatalln("Error removing domains", domainResult.Error)
		}
	}
}

// This method is the last part of the scan, when the new state of the domain object is
// persisted back to the database. It receives a go routine control group to sinalize to
// the main thread when the scan ends, a domain channel to receive each domain that need
// to be save and an error channel to send back all errors while persisting the data. It
// was created to be asynchronous and finish after receiving a poison pill from querier
// dispatcher
func (c *Collector) Start(scanGroup *sync.WaitGroup,
	domainsToSaveChannel chan *model.Domain, errorsChannel chan error) {

	// Add one more to the group of scan go routines
	scanGroup.Add(1)

	go func() {
		// Initialize Domain DAO using injected database connection
		domainDAO := dao.DomainDAO{
			Database: c.Database,
		}

		// Add a safety check to avoid an infinite loop
		if c.SaveAtOnce == 0 {
			c.SaveAtOnce = 1
		}

		finished := false
		nameserverStatistics := make(map[string]uint64)
		dsStatistics := make(map[string]uint64)

		for {
			// Using make for faster allocation
			domains := make([]*model.Domain, 0, c.SaveAtOnce)

			for i := 0; i < c.SaveAtOnce; i++ {
				domain := <-domainsToSaveChannel

				// Detect poison pill. We don't return from function here because we can still
				// have some domains to save in the domains array
				if domain == nil {
					finished = true
					break
				}

				// Count this domain for the scan information to estimate the scan progress
				model.FinishAnalyzingDomainForScan(len(domain.DSSet) > 0)

				// Keep track of nameservers statistics
				for _, nameserver := range domain.Nameservers {
					status := model.NameserverStatusToString(nameserver.LastStatus)
					nameserverStatistics[status] += 1
				}

				// Keep track of DS statistics
				for _, ds := range domain.DSSet {
					status := model.DSStatusToString(ds.LastStatus)
					dsStatistics[status] += 1
				}

				domains = append(domains, domain)
			}

			domainsResults := domainDAO.SaveMany(domains)
			for _, domainResult := range domainsResults {
				if domainResult.Error != nil {
					// Error channel should have a buffer or this will block the collector until
					// someone check this error. One question here is that we are returning the
					// error, but not telling wich domain got the error, we should improve the error
					// communication system between the go routines
					errorsChannel <- domainResult.Error
				}
			}

			// Now that everything is done, check if we received a poison pill
			if finished {
				model.StoreStatisticsOfTheScan(nameserverStatistics, dsStatistics)
				scanGroup.Done()
				return
			}
		}
	}()
}