Golang fileutils.CleanPatterns函数代码示例

// TarWithOptions creates an archive from the directory at `path`, only including files whose relative
// paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`.
func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) {

	patterns, patDirs, exceptions, err := fileutils.CleanPatterns(options.ExcludePatterns)

	if err != nil {
		return nil, err
	}

	pipeReader, pipeWriter := io.Pipe()

	compressWriter, err := CompressStream(pipeWriter, options.Compression)
	if err != nil {
		return nil, err
	}

	go func() {
		ta := &tarAppender{
			TarWriter: tar.NewWriter(compressWriter),
			Buffer:    pools.BufioWriter32KPool.Get(nil),
			SeenFiles: make(map[uint64]string),
		}
		// this buffer is needed for the duration of this piped stream
		defer pools.BufioWriter32KPool.Put(ta.Buffer)

		// In general we log errors here but ignore them because
		// during e.g. a diff operation the container can continue
		// mutating the filesystem and we can see transient errors
		// from this

		if options.IncludeFiles == nil {
			options.IncludeFiles = []string{"."}
		}

		seen := make(map[string]bool)

		var renamedRelFilePath string // For when tar.Options.Name is set
		for _, include := range options.IncludeFiles {
			filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error {
				if err != nil {
					logrus.Debugf("Tar: Can't stat file %s to tar: %s", srcPath, err)
					return nil
				}

				relFilePath, err := filepath.Rel(srcPath, filePath)
				if err != nil || (relFilePath == "." && f.IsDir()) {
					// Error getting relative path OR we are looking
					// at the root path. Skip in both situations.
					return nil
				}

				skip := false

				// If "include" is an exact match for the current file
				// then even if there's an "excludePatterns" pattern that
				// matches it, don't skip it. IOW, assume an explicit 'include'
				// is asking for that file no matter what - which is true
				// for some files, like .dockerignore and Dockerfile (sometimes)
				if include != relFilePath {
					skip, err = fileutils.OptimizedMatches(relFilePath, patterns, patDirs)
					if err != nil {
						logrus.Debugf("Error matching %s", relFilePath, err)
						return err
					}
				}

				if skip {
					if !exceptions && f.IsDir() {
						return filepath.SkipDir
					}
					return nil
				}

				if seen[relFilePath] {
					return nil
				}
				seen[relFilePath] = true

				// Rename the base resource
				if options.Name != "" && filePath == srcPath+"/"+filepath.Base(relFilePath) {
					renamedRelFilePath = relFilePath
				}
				// Set this to make sure the items underneath also get renamed
				if options.Name != "" {
					relFilePath = strings.Replace(relFilePath, renamedRelFilePath, options.Name, 1)
				}

				if err := ta.addTarFile(filePath, relFilePath); err != nil {
					logrus.Debugf("Can't add file %s to tar: %s", filePath, err)
				}
				return nil
			})
		}

		// Make sure to check the error on Close.
		if err := ta.TarWriter.Close(); err != nil {
			logrus.Debugf("Can't close tar writer: %s", err)
		}
		if err := compressWriter.Close(); err != nil {
//.........这里部分代码省略.........

// TarWithOptions creates an archive from the directory at `path`, only including files whose relative
// paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`.
func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) {

	// Fix the source path to work with long path names. This is a no-op
	// on platforms other than Windows.
	srcPath = fixVolumePathPrefix(srcPath)

	patterns, patDirs, exceptions, err := fileutils.CleanPatterns(options.ExcludePatterns)

	if err != nil {
		return nil, err
	}

	pipeReader, pipeWriter := io.Pipe()

	compressWriter, err := CompressStream(pipeWriter, options.Compression)
	if err != nil {
		return nil, err
	}

	go func() {
		ta := &tarAppender{
			TarWriter:         tar.NewWriter(compressWriter),
			Buffer:            pools.BufioWriter32KPool.Get(nil),
			SeenFiles:         make(map[uint64]string),
			UIDMaps:           options.UIDMaps,
			GIDMaps:           options.GIDMaps,
			WhiteoutConverter: getWhiteoutConverter(options.WhiteoutFormat),
		}

		defer func() {
			// Make sure to check the error on Close.
			if err := ta.TarWriter.Close(); err != nil {
				logrus.Errorf("Can't close tar writer: %s", err)
			}
			if err := compressWriter.Close(); err != nil {
				logrus.Errorf("Can't close compress writer: %s", err)
			}
			if err := pipeWriter.Close(); err != nil {
				logrus.Errorf("Can't close pipe writer: %s", err)
			}
		}()

		// this buffer is needed for the duration of this piped stream
		defer pools.BufioWriter32KPool.Put(ta.Buffer)

		// In general we log errors here but ignore them because
		// during e.g. a diff operation the container can continue
		// mutating the filesystem and we can see transient errors
		// from this

		stat, err := os.Lstat(srcPath)
		if err != nil {
			return
		}

		if !stat.IsDir() {
			// We can't later join a non-dir with any includes because the
			// 'walk' will error if "file/." is stat-ed and "file" is not a
			// directory. So, we must split the source path and use the
			// basename as the include.
			if len(options.IncludeFiles) > 0 {
				logrus.Warn("Tar: Can't archive a file with includes")
			}

			dir, base := SplitPathDirEntry(srcPath)
			srcPath = dir
			options.IncludeFiles = []string{base}
		}

		if len(options.IncludeFiles) == 0 {
			options.IncludeFiles = []string{"."}
		}

		seen := make(map[string]bool)

		for _, include := range options.IncludeFiles {
			rebaseName := options.RebaseNames[include]

			walkRoot := getWalkRoot(srcPath, include)
			filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error {
				if err != nil {
					logrus.Errorf("Tar: Can't stat file %s to tar: %s", srcPath, err)
					return nil
				}

				relFilePath, err := filepath.Rel(srcPath, filePath)
				if err != nil || (!options.IncludeSourceDir && relFilePath == "." && f.IsDir()) {
					// Error getting relative path OR we are looking
					// at the source directory path. Skip in both situations.
					return nil
				}

				if options.IncludeSourceDir && include == "." && relFilePath != "." {
					relFilePath = strings.Join([]string{".", relFilePath}, string(filepath.Separator))
				}

				skip := false

//.........这里部分代码省略.........

func listFiles(srcPath string, includes, excludes []string, cmdName string, urlFetcher URLFetcher) ([]*uploadFile, error) {

	log.Debugf("searching patterns, %# v\n", pretty.Formatter(includes))

	result := []*uploadFile{}
	seen := map[string]struct{}{}

	// TODO: support local archives (and maybe a remote archives as well)

	excludes, patDirs, exceptions, err := fileutils.CleanPatterns(excludes)
	if err != nil {
		return nil, err
	}

	// TODO: here we remove some exclude patterns, how about patDirs?
	excludes, nestedPatterns := findNestedPatterns(excludes)

	for _, pattern := range includes {

		if isURL(pattern) {
			if cmdName == "COPY" {
				return nil, fmt.Errorf("can't use url in COPY command: '%s'", pattern)
			}

			if urlFetcher == nil {
				return nil, fmt.Errorf("want to list a downloaded url '%s', but URLFetcher is not present", pattern)
			}

			ui, err := urlFetcher.GetInfo(pattern)
			if err != nil {
				return nil, err
			}

			result = append(result, &uploadFile{
				src:  ui.FileName,
				dest: ui.BaseName,
				size: ui.Size,
			})
			continue
		}

		matches, err := filepath.Glob(filepath.Join(srcPath, pattern))
		if err != nil {
			return result, err
		}

		for _, match := range matches {

			// We need to check if the current match is dir
			// to prefix files inside with it
			matchInfo, err := os.Stat(match)
			if err != nil {
				return result, err
			}

			// Walk through each match since it may be a directory
			err = filepath.Walk(match, func(path string, info os.FileInfo, err error) error {

				if err != nil {
					return err
				}

				relFilePath, err := filepath.Rel(srcPath, path)
				if err != nil {
					return err
				}

				// TODO: ensure ignoring works correctly, maybe improve .dockerignore to work more like .gitignore?

				skip := false
				skipNested := false

				// Here we want to keep files that are specified explicitly in the includes,
				// no matter what. For example, .dockerignore can have some wildcard items
				// specified, by in COPY we want explicitly add a file, that could be ignored
				// otherwise using a wildcard or directory COPY
				if pattern != relFilePath {
					if skip, err = fileutils.OptimizedMatches(relFilePath, excludes, patDirs); err != nil {
						return err
					}
					if skipNested, err = matchNested(relFilePath, nestedPatterns); err != nil {
						return err
					}
				}

				if skip || skipNested {
					if !exceptions && info.IsDir() {
						return filepath.SkipDir
					}
					return nil
				}

				// TODO: read links?

				// not interested in dirs, since we walk already
				if info.IsDir() {
					return nil
				}

				if _, ok := seen[relFilePath]; ok {
//.........这里部分代码省略.........

// TarWithOptions creates an archive from the directory at `path`, only including files whose relative
// paths are included in `options.IncludeFiles` (if non-nil) or not in `options.ExcludePatterns`.
func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) {

	patterns, patDirs, exceptions, err := fileutils.CleanPatterns(options.ExcludePatterns)

	if err != nil {
		return nil, err
	}

	pipeReader, pipeWriter := io.Pipe()

	compressWriter, err := CompressStream(pipeWriter, options.Compression)
	if err != nil {
		return nil, err
	}

	go func() {
		ta := &tarAppender{
			TarWriter: tar.NewWriter(compressWriter),
			Buffer:    pools.BufioWriter32KPool.Get(nil),
			SeenFiles: make(map[uint64]string),
		}

		defer func() {
			// Make sure to check the error on Close.
			if err := ta.TarWriter.Close(); err != nil {
				logrus.Debugf("Can't close tar writer: %s", err)
			}
			if err := compressWriter.Close(); err != nil {
				logrus.Debugf("Can't close compress writer: %s", err)
			}
			if err := pipeWriter.Close(); err != nil {
				logrus.Debugf("Can't close pipe writer: %s", err)
			}
		}()

		// this buffer is needed for the duration of this piped stream
		defer pools.BufioWriter32KPool.Put(ta.Buffer)

		// In general we log errors here but ignore them because
		// during e.g. a diff operation the container can continue
		// mutating the filesystem and we can see transient errors
		// from this

		stat, err := os.Lstat(srcPath)
		if err != nil {
			return
		}

		if !stat.IsDir() {
			// We can't later join a non-dir with any includes because the
			// 'walk' will error if "file/." is stat-ed and "file" is not a
			// directory. So, we must split the source path and use the
			// basename as the include.
			if len(options.IncludeFiles) > 0 {
				logrus.Warn("Tar: Can't archive a file with includes")
			}

			dir, base := SplitPathDirEntry(srcPath)
			srcPath = dir
			options.IncludeFiles = []string{base}
		}

		if len(options.IncludeFiles) == 0 {
			options.IncludeFiles = []string{"."}
		}

		seen := make(map[string]bool)

		for _, include := range options.IncludeFiles {
			rebaseName := options.RebaseNames[include]

			// We can't use filepath.Join(srcPath, include) because this will
			// clean away a trailing "." or "/" which may be important.
			walkRoot := strings.Join([]string{srcPath, include}, string(filepath.Separator))
			filepath.Walk(walkRoot, func(filePath string, f os.FileInfo, err error) error {
				if err != nil {
					logrus.Debugf("Tar: Can't stat file %s to tar: %s", srcPath, err)
					return nil
				}

				relFilePath, err := filepath.Rel(srcPath, filePath)
				if err != nil || (!options.IncludeSourceDir && relFilePath == "." && f.IsDir()) {
					// Error getting relative path OR we are looking
					// at the source directory path. Skip in both situations.
					return nil
				}

				if options.IncludeSourceDir && include == "." && relFilePath != "." {
					relFilePath = strings.Join([]string{".", relFilePath}, string(filepath.Separator))
				}

				skip := false

				// If "include" is an exact match for the current file
				// then even if there's an "excludePatterns" pattern that
				// matches it, don't skip it. IOW, assume an explicit 'include'
				// is asking for that file no matter what - which is true
				// for some files, like .dockerignore and Dockerfile (sometimes)
//.........这里部分代码省略.........

func listFiles(srcPath string, includes, excludes []string) ([]*uploadFile, error) {

	result := []*uploadFile{}
	seen := map[string]struct{}{}

	// TODO: support urls
	// TODO: support local archives (and maybe a remote archives as well)

	excludes, patDirs, exceptions, err := fileutils.CleanPatterns(excludes)
	if err != nil {
		return nil, err
	}

	// TODO: here we remove some exclude patterns, how about patDirs?
	excludes, nestedPatterns := findNestedPatterns(excludes)

	for _, pattern := range includes {

		matches, err := filepath.Glob(filepath.Join(srcPath, pattern))
		if err != nil {
			return result, err
		}

		for _, match := range matches {

			// We need to check if the current match is dir
			// to prefix files inside with it
			matchInfo, err := os.Stat(match)
			if err != nil {
				return result, err
			}

			// Walk through each match since it may be a directory
			err = filepath.Walk(match, func(path string, info os.FileInfo, err error) error {

				relFilePath, err := filepath.Rel(srcPath, path)
				if err != nil {
					return err
				}

				// TODO: ensure ignoring works correctly, maybe improve .dockerignore to work more like .gitignore?

				skip := false
				skipNested := false

				// Here we want to keep files that are specified explicitly in the includes,
				// no matter what. For example, .dockerignore can have some wildcard items
				// specified, by in COPY we want explicitly add a file, that could be ignored
				// otherwise using a wildcard or directory COPY
				if pattern != relFilePath {
					if skip, err = fileutils.OptimizedMatches(relFilePath, excludes, patDirs); err != nil {
						return err
					}
					if skipNested, err = matchNested(relFilePath, nestedPatterns); err != nil {
						return err
					}
				}

				if skip || skipNested {
					if !exceptions && info.IsDir() {
						return filepath.SkipDir
					}
					return nil
				}

				// TODO: read links?

				// not interested in dirs, since we walk already
				if info.IsDir() {
					return nil
				}

				// skip checking if symlinks point to non-existing file
				// also skip named pipes, because they hanging on open
				if info.Mode()&(os.ModeSymlink|os.ModeNamedPipe) != 0 {
					return nil
				}

				if _, ok := seen[relFilePath]; ok {
					return nil
				}
				seen[relFilePath] = struct{}{}

				// cut the wildcard path of the file or use base name

				var (
					resultFilePath string
					baseChunks     = splitPath(pattern)
					destChunks     = splitPath(relFilePath)
					lastChunk      = baseChunks[len(baseChunks)-1]
				)

				if containsWildcards(lastChunk) {
					// In case there is `foo/bar/*` source path we need to make a
					// destination files without `foo/bar/` prefix
					resultFilePath = filepath.Join(destChunks[len(baseChunks)-1:]...)
				} else if matchInfo.IsDir() {
					// If source is a directory, keep as is
					resultFilePath = relFilePath
				} else {
//.........这里部分代码省略.........