Golang Group.Error方法代碼示例

func TestOneError(t *testing.T) {
	t.Parallel()
	e := errors.New("")
	var g errgroup.Group
	g.Error(e)
	ensure.True(t, g.Wait() == e)
}

func TestTwoErrors(t *testing.T) {
	t.Parallel()
	e1 := errors.New("e1")
	e2 := errors.New("e2")
	var g errgroup.Group
	g.Error(e1)
	g.Error(e2)
	ensure.DeepEqual(t, g.Wait().Error(), "multiple errors: e1 | e2")
}

// DumpTablesToS3 dumps multiple tables to s3 in parallel.
func (db *DB) DumpTablesToS3(tables []string, s3prefix string) error {
	group := new(errgroup.Group)
	for _, table := range tables {
		group.Add(1)
		go func(table string) {
			if err := db.DumpTableToS3(table, S3Filename(s3prefix, table)); err != nil {
				group.Error(err)
			}
			group.Done()
		}(table)
	}
	return group.Wait()
}

// GetTableSchemas returns a map from a tablename to its schema. Gets schemas for different tables
// in parallel.
func (db *DB) GetTableSchemas(tables []string, namespace string) (map[string]TableSchema, error) {
	group := new(errgroup.Group)
	tsmap := map[string]TableSchema{}
	for _, table := range tables {
		group.Add(1)
		go func(table string) {
			ts, err := db.GetTableSchema(table, namespace)
			if err != nil {
				group.Error(err)
			}
			tsmap[table] = ts
			group.Done()
		}(table)
	}
	err := group.Wait()
	return tsmap, err
}

func (h *herokuLink) herokuAppNames(ids []string, e *parsecli.Env) (nameIDs, []string, error) {
	var wg errgroup.Group
	wg.Add(len(ids))
	maxParallel := make(chan struct{}, maxRequests)

	var (
		ret               nameIDs
		deletedLinks      []string
		retMutex          sync.Mutex
		deletedLinksMutex sync.Mutex
	)

	getAppName := func(id string) {
		defer func() {
			wg.Done()
			<-maxParallel
		}()
		appName, err := parsecli.FetchHerokuAppName(id, e)
		if err != nil {
			if stackerr.HasUnderlying(err, stackerr.MatcherFunc(parsecli.HerokuAppNotFound)) {
				deletedLinksMutex.Lock()
				defer deletedLinksMutex.Unlock()
				deletedLinks = append(deletedLinks, id)
				return
			}
			wg.Error(err) // ignore error if corresponding heroku app was deleted
			return
		}

		retMutex.Lock()
		defer retMutex.Unlock()
		ret = append(ret, nameID{id: id, name: appName})
	}

	for _, id := range ids {
		go getAppName(id)
	}
	err := wg.Wait()
	sort.Sort(ret)
	return ret, deletedLinks, stackerr.Wrap(err)
}

// RefreshTables refreshes multiple tables in parallel and returns an error if any of the copies
// fail.
func (r *Redshift) RefreshTables(
	tables map[string]postgres.TableSchema, schema, tmpschema, s3prefix, awsRegion string, delim rune) error {
	if _, err := r.logAndExec(fmt.Sprintf(`CREATE SCHEMA "%s"`, tmpschema), false); err != nil {
		return err
	}
	group := new(errgroup.Group)
	for name, ts := range tables {
		group.Add(1)
		go func(name string, ts postgres.TableSchema) {
			if err := r.refreshTable(schema, name, tmpschema, postgres.S3Filename(s3prefix, name), awsRegion, ts, delim); err != nil {
				group.Error(err)
			}
			group.Done()
		}(name, ts)
	}
	errs := new(errgroup.Group)
	if err := group.Wait(); err != nil {
		errs.Error(err)
	}
	if _, err := r.logAndExec(fmt.Sprintf(`DROP SCHEMA "%s" CASCADE`, tmpschema), false); err != nil {
		errs.Error(err)
	}
	// Use errs.Wait() to group the two errors into a single error object.
	return errs.Wait()
}

func (d *deployCmd) computeChecksums(files []string,
	normalizeName func(string) string) (map[string]string, error) {
	var wg errgroup.Group
	maxParallel := make(chan struct{}, maxOpenFD)
	wg.Add(len(files))
	var mutex sync.Mutex

	checksums := make(map[string]string)

	computeChecksum := func(name string) {
		defer func() {
			wg.Done()
			<-maxParallel
		}()

		file, err := os.Open(name)
		defer file.Close()
		if err != nil {
			wg.Error(stackerr.Wrap(err))
			return
		}
		h := md5.New()
		if _, err := io.Copy(h, file); err != nil {
			wg.Error(stackerr.Wrap(err))
			return
		}
		if err := file.Close(); err != nil {
			wg.Error(stackerr.Wrap(err))
			return
		}
		mutex.Lock()
		checksums[normalizeName(name)] = fmt.Sprintf("%x", h.Sum(nil))
		defer mutex.Unlock()
	}

	for _, file := range files {
		maxParallel <- struct{}{}
		go computeChecksum(file)
	}

	err := wg.Wait()
	if err != nil {
		return checksums, err
	}

	return checksums, nil
}

func uploadSymbolFiles(files []string, commonHeaders map[string]string, removeFiles bool, e *env) error {
	var wg errgroup.Group
	uploadFile := func(filename string, e *env) {
		defer wg.Done()
		name := filepath.Base(filepath.Clean(filename))
		file, err := os.Open(filename)
		if err != nil {
			wg.Error(stackerr.Wrap(err))
			return
		}
		defer file.Close()

		req, err := http.NewRequest("POST", path.Join("symbolFiles", name), bufio.NewReader(file))
		if err != nil {
			wg.Error(stackerr.Wrap(err))
			return
		}

		if req.Header == nil {
			req.Header = make(http.Header)
		}
		for key, val := range commonHeaders {
			req.Header.Add(key, val)
		}
		hash, err := base64MD5OfFile(filename)
		if err != nil {
			wg.Error(err)
			return
		}
		req.Header.Add("Content-MD5", hash)
		mimeType := mime.TypeByExtension(filepath.Ext(name))
		if mimeType == "" {
			mimeType = "application/octet-stream"
		}
		req.Header.Add("Content-Type", mimeType)
		res := make(map[string]interface{})
		if _, err := e.ParseAPIClient.Do(req, nil, &res); err != nil {
			wg.Error(err)
			return
		}
		if removeFiles {
			if err := file.Close(); err != nil {
				wg.Error(err)
				return
			}
			if err := os.Remove(filename); err != nil {
				wg.Error(err)
				return
			}
		}
	}
	for _, file := range files {
		wg.Add(1)
		go uploadFile(file, e)
	}
	err := wg.Wait()
	if err != nil {
		return err
	}

	fmt.Fprintln(e.Out, "Uploaded symbol files.")
	return nil
}

func (d *downloadCmd) moveFiles(
	e *env,
	destination string,
	release *deployInfo) error {
	var wg errgroup.Group

	maxParallel := make(chan struct{}, maxOpenFD)
	numFiles := len(release.Versions.Cloud) + len(release.Versions.Public)
	wg.Add(numFiles)

	var numErrors int32
	moveFile := func(destination, kind, file, checksum string) {
		defer func() {
			wg.Done()
			<-maxParallel
		}()

		err := os.MkdirAll(
			filepath.Join(e.Root, kind, filepath.Dir(file)),
			0755,
		)
		if err != nil {
			atomic.AddInt32(&numErrors, 1)
			wg.Error(stackerr.Wrap(err))
			return
		}

		err = os.Rename(
			filepath.Join(destination, kind, file),
			filepath.Join(e.Root, kind, file),
		)
		if err != nil {
			atomic.AddInt32(&numErrors, 1)
			wg.Error(stackerr.Wrap(err))
			return
		}

		err = d.verifyChecksum(
			filepath.Join(e.Root, kind, file),
			checksum,
		)
		if err != nil {
			atomic.AddInt32(&numErrors, 1)
			wg.Error(err)
			return
		}
	}

	for file, checksum := range release.Checksums.Cloud {
		maxParallel <- struct{}{}
		go moveFile(
			destination,
			cloudDir,
			file,
			checksum,
		)
	}
	for file, checksum := range release.Checksums.Public {
		maxParallel <- struct{}{}
		go moveFile(
			destination,
			hostingDir,
			file,
			checksum,
		)
	}

	if err := wg.Wait(); err != nil {
		// could not move a single file so no corruption:w
		if int(numErrors) == numFiles {
			fmt.Fprintf(
				e.Out,
				`Failed to download Cloud Code to
 %q
Try "parse download" and manually move contents from
the temporary download location.
`,
				e.Root,
			)
			return nil
		}

		fmt.Fprintf(
			e.Out,
			`Failed to download Cloud Code to
 %q

It might have corrupted contents, due to partially moved files.

Try "parse download" and manually move contents from
the temporary download location.
`,
			e.Root,
		)
		return err
	}
	return nil
}

func (d *downloadCmd) download(e *env, destination string, release *deployInfo) error {
	var wg errgroup.Group
	maxParallel := make(chan struct{}, maxOpenFD)
	wg.Add(len(release.Versions.Cloud) + len(release.Versions.Public))

	downloadHosted := func(file, version, checksum string) {
		defer func() {
			wg.Done()
			<-maxParallel
		}()

		v := make(url.Values)
		v.Set("version", version)
		v.Set("checksum", checksum)
		u := &url.URL{
			Path:     path.Join("hosted_files", file),
			RawQuery: v.Encode(),
		}
		var content []byte
		_, err := e.ParseAPIClient.Get(u, &content)
		if err != nil {
			wg.Error(err)
			return
		}

		path := path.Join(destination, hostingDir, file)
		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
			wg.Error(stackerr.Wrap(err))
			return
		}
		if err := ioutil.WriteFile(path, content, 0644); err != nil {
			wg.Error(stackerr.Wrap(err))
			return
		}
		if err := d.verifyChecksum(path, checksum); err != nil {
			wg.Error(err)
			return
		}
	}

	downloadScript := func(file, version, checksum string) {
		defer func() {
			wg.Done()
			<-maxParallel
		}()

		v := make(url.Values)
		v.Set("version", version)
		v.Set("checksum", checksum)
		u := &url.URL{
			Path:     path.Join("scripts", file),
			RawQuery: v.Encode(),
		}
		var content string
		_, err := e.ParseAPIClient.Get(u, &content)
		if err != nil {
			wg.Error(err)
			return
		}

		path := path.Join(destination, cloudDir, file)
		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
			wg.Error(stackerr.Wrap(err))
			return
		}
		if err := ioutil.WriteFile(path, []byte(content), 0644); err != nil {
			wg.Error(stackerr.Wrap(err))
			return
		}
	}

	for file, version := range release.Versions.Public {
		checksum, ok := release.Checksums.Public[file]
		if !ok {
			continue
		}
		maxParallel <- struct{}{}
		go downloadHosted(file, version, checksum)
	}

	for file, version := range release.Versions.Cloud {
		checksum, ok := release.Checksums.Cloud[file]
		if !ok {
			continue
		}
		maxParallel <- struct{}{}
		go downloadScript(file, version, checksum)
	}

	return wg.Wait()
}

// Pair returns a TransformFunc that pairs all the elements in the table with another table, based
// on the given identifier functions and join type.
func Pair(rightTable optimus.Table, leftID, rightID RowIdentifier, filterFn func(optimus.Row) (bool, error)) optimus.TransformFunc {
	// Map of everything in the right table
	right := make(map[interface{}][]optimus.Row)
	// Track whether or not rows in the right table were joined against
	joined := make(map[interface{}]bool)

	// Start building the map right away, because it could be slow.
	mapResult := make(chan error)
	go func() {
		defer close(mapResult)
		for row := range rightTable.Rows() {
			id, err := rightID(row)
			if err != nil {
				mapResult <- err
				return
			}
			if val := right[id]; val == nil {
				right[id] = []optimus.Row{}
				joined[id] = false
			}
			right[id] = append(right[id], row)
		}
		mapResult <- rightTable.Err()
	}()

	return func(in <-chan optimus.Row, out chan<- optimus.Row) error {
		if err := <-mapResult; err != nil {
			return err
		}
		// The channel of paired rows from the left and right tables
		pairedRows := make(chan optimus.Row)

		wg := errgroup.Group{}
		// Pair the left table with the right table based on the ids
		wg.Add(1)
		go func() {
			defer close(pairedRows)
			defer wg.Done()

			for leftRow := range in {
				id, err := leftID(leftRow)
				if err != nil {
					wg.Error(err)
					return
				}
				if rightRows := right[id]; rightRows != nil && id != nil {
					joined[id] = true
					for _, rightRow := range rightRows {
						pairedRows <- optimus.Row{"left": leftRow, "right": rightRow}
					}
				} else {
					pairedRows <- optimus.Row{"left": leftRow}
				}
			}

			for id, joined := range joined {
				if joined {
					continue
				}
				for _, rightRow := range right[id] {
					pairedRows <- optimus.Row{"right": rightRow}
				}
			}
			return
		}()

		// Filter the paired rows based on our join type
		wg.Add(1)
		go func() {
			defer wg.Done()
			if err := Select(filterFn)(pairedRows, out); err != nil {
				wg.Error(err)
			}
		}()
		return wg.Wait()
	}
}

func (d *deployCmd) uploadSourceFiles(u *uploader) (map[string]string,
	map[string]string, error) {
	sourceFiles, ignoredFiles, err := d.getSourceFiles(filepath.Join(u.Env.Root, u.DirName), u.Suffixes, u.Env)
	if err != nil {
		return nil, nil, err
	}

	namePrefixLen := len(filepath.Join(u.Env.Root, u.DirName, "1")) - 1
	normalizeName := func(name string) string {
		name = filepath.ToSlash(filepath.Clean(name))
		return name[namePrefixLen:]
	}

	currentChecksums, err := d.computeChecksums(sourceFiles, normalizeName)
	if err != nil {
		return nil, nil, err
	}

	var mutex sync.Mutex
	maxParallel := make(chan struct{}, maxOpenFD)
	var wg errgroup.Group
	currentVersions := make(map[string]string)

	uploadFile := func(sourceFile string) {
		defer func() {
			wg.Done()
			<-maxParallel
		}()
		version, err := d.uploadFile(sourceFile, u.EndPoint, u.Env, normalizeName)
		if err != nil {
			wg.Error(err)
			return
		}
		mutex.Lock()
		currentVersions[normalizeName(sourceFile)] = version
		defer mutex.Unlock()
	}
	changed := false
	var changedFiles []string
	for _, sourceFile := range sourceFiles {
		if !d.Force { // if not forced, verify changed content using checksums
			name := normalizeName(sourceFile)
			var noUpload bool
			if prevChecksum, ok := u.PrevChecksums[name]; ok {
				noUpload = prevChecksum == currentChecksums[name]
			}
			if prevVersion, ok := u.PrevVersions[name]; ok && noUpload {
				currentVersions[name] = prevVersion
				continue
			}
		}
		changed = true
		wg.Add(1)

		changedFiles = append(changedFiles, sourceFile)

		maxParallel <- struct{}{}
		go uploadFile(sourceFile)
	}
	if changed && d.Verbose {
		var message string
		switch u.DirName {
		case "cloud":
			message = "scripts"
		case "public":
			message = "hosting"
		}
		fmt.Fprintf(u.Env.Out,
			`Uploading recent changes to %s...
The following files will be uploaded:
%s
`,
			message,
			strings.Join(changedFiles, "\n"),
		)
		if len(ignoredFiles) != 0 {
			fmt.Fprintln(u.Env.Out, "The following files will be ignored:")
			for _, file := range ignoredFiles {
				fmt.Fprintln(u.Env.Out, file)
			}
		}
	}
	if err := wg.Wait(); err != nil {
		return nil, nil, err
	}
	return currentChecksums, currentVersions, nil
}