Golang archive.DecompressStream函数代码示例

// applyLayerHandler parses a diff in the standard layer format from `layer`, and
// applies it to the directory `dest`. Returns the size in bytes of the
// contents of the layer.
func applyLayerHandler(dest string, layer archive.Reader, decompress bool) (size int64, err error) {
	dest = filepath.Clean(dest)

	// Ensure it is a Windows-style volume path
	dest = longpath.AddPrefix(dest)

	if decompress {
		decompressed, err := archive.DecompressStream(layer)
		if err != nil {
			return 0, err
		}
		defer decompressed.Close()

		layer = decompressed
	}

	tmpDir, err := ioutil.TempDir(os.Getenv("temp"), "temp-docker-extract")
	if err != nil {
		return 0, fmt.Errorf("ApplyLayer failed to create temp-docker-extract under %s. %s", dest, err)
	}

	s, err := archive.UnpackLayer(dest, layer)
	os.RemoveAll(tmpDir)
	if err != nil {
		return 0, fmt.Errorf("ApplyLayer %s failed UnpackLayer to %s", err, dest)
	}

	return s, nil
}

func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, foreignSrc distribution.Descriptor, progressOutput progress.Output) (layer.Layer, error) {
	// We use system.OpenSequential to use sequential file access on Windows, avoiding
	// depleting the standby list. On Linux, this equates to a regular os.Open.
	rawTar, err := system.OpenSequential(filename)
	if err != nil {
		logrus.Debugf("Error reading embedded tar: %v", err)
		return nil, err
	}
	defer rawTar.Close()

	var r io.Reader
	if progressOutput != nil {
		fileInfo, err := rawTar.Stat()
		if err != nil {
			logrus.Debugf("Error statting file: %v", err)
			return nil, err
		}

		r = progress.NewProgressReader(rawTar, progressOutput, fileInfo.Size(), stringid.TruncateID(id), "Loading layer")
	} else {
		r = rawTar
	}

	inflatedLayerData, err := archive.DecompressStream(r)
	if err != nil {
		return nil, err
	}
	defer inflatedLayerData.Close()

	if ds, ok := l.ls.(layer.DescribableStore); ok {
		return ds.RegisterWithDescriptor(inflatedLayerData, rootFS.ChainID(), foreignSrc)
	}
	return l.ls.Register(inflatedLayerData, rootFS.ChainID())
}

// applyLayerHandler parses a diff in the standard layer format from `layer`, and
// applies it to the directory `dest`. Returns the size in bytes of the
// contents of the layer.
func applyLayerHandler(dest string, layer archive.ArchiveReader, decompress bool) (size int64, err error) {
	dest = filepath.Clean(dest)
	if decompress {
		decompressed, err := archive.DecompressStream(layer)
		if err != nil {
			return 0, err
		}
		defer decompressed.Close()

		layer = decompressed
	}

	cmd := reexec.Command("docker-applyLayer", dest)
	cmd.Stdin = layer

	outBuf, errBuf := new(bytes.Buffer), new(bytes.Buffer)
	cmd.Stdout, cmd.Stderr = outBuf, errBuf

	if err = cmd.Run(); err != nil {
		return 0, fmt.Errorf("ApplyLayer %s stdout: %s stderr: %s", err, outBuf, errBuf)
	}

	// Stdout should be a valid JSON struct representing an applyLayerResponse.
	response := applyLayerResponse{}
	decoder := json.NewDecoder(outBuf)
	if err = decoder.Decode(&response); err != nil {
		return 0, fmt.Errorf("unable to decode ApplyLayer JSON response: %s", err)
	}

	return response.LayerSize, nil
}

func (graph *Graph) disassembleAndApplyTarLayer(id, parent string, layerData archive.ArchiveReader, root string) (size int64, err error) {
	// this is saving the tar-split metadata
	mf, err := os.OpenFile(filepath.Join(root, tarDataFileName), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
	if err != nil {
		return 0, err
	}
	mfz := gzip.NewWriter(mf)
	metaPacker := storage.NewJSONPacker(mfz)
	defer mf.Close()
	defer mfz.Close()

	inflatedLayerData, err := archive.DecompressStream(layerData)
	if err != nil {
		return 0, err
	}

	// we're passing nil here for the file putter, because the ApplyDiff will
	// handle the extraction of the archive
	rdr, err := asm.NewInputTarStream(inflatedLayerData, metaPacker, nil)
	if err != nil {
		return 0, err
	}

	if size, err = graph.driver.ApplyDiff(id, parent, archive.ArchiveReader(rdr)); err != nil {
		return 0, err
	}

	return
}

func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, foreignSrc distribution.Descriptor, progressOutput progress.Output) (layer.Layer, error) {
	rawTar, err := os.Open(filename)
	if err != nil {
		logrus.Debugf("Error reading embedded tar: %v", err)
		return nil, err
	}
	defer rawTar.Close()

	var r io.Reader
	if progressOutput != nil {
		fileInfo, err := rawTar.Stat()
		if err != nil {
			logrus.Debugf("Error statting file: %v", err)
			return nil, err
		}

		r = progress.NewProgressReader(rawTar, progressOutput, fileInfo.Size(), stringid.TruncateID(id), "Loading layer")
	} else {
		r = rawTar
	}

	inflatedLayerData, err := archive.DecompressStream(r)
	if err != nil {
		return nil, err
	}
	defer inflatedLayerData.Close()

	if ds, ok := l.ls.(layer.DescribableStore); ok {
		return ds.RegisterWithDescriptor(inflatedLayerData, rootFS.ChainID(), foreignSrc)
	}
	return l.ls.Register(inflatedLayerData, rootFS.ChainID())
}

func (dm *downloadManager) Download(ctx context.Context, initialRootFS image.RootFS, layers []xfer.DownloadDescriptor, progressOutput progress.Output) (image.RootFS, func(), error) {
	for _, l := range layers {
		b, err := dm.blobStore.New()
		if err != nil {
			return initialRootFS, nil, err
		}
		defer b.Close()
		rc, _, err := l.Download(ctx, progressOutput)
		if err != nil {
			return initialRootFS, nil, errors.Wrap(err, "failed to download")
		}
		defer rc.Close()
		r := io.TeeReader(rc, b)
		inflatedLayerData, err := archive.DecompressStream(r)
		if err != nil {
			return initialRootFS, nil, err
		}
		digester := digest.Canonical.New()
		if _, err := archive.ApplyLayer(dm.tmpDir, io.TeeReader(inflatedLayerData, digester.Hash())); err != nil {
			return initialRootFS, nil, err
		}
		initialRootFS.Append(layer.DiffID(digester.Digest()))
		d, err := b.Commit()
		if err != nil {
			return initialRootFS, nil, err
		}
		dm.blobs = append(dm.blobs, d)
	}
	return initialRootFS, nil, nil
}

func (b *builder) readContext(context io.Reader) (err error) {
	tmpdirPath, err := ioutil.TempDir("", "docker-build")
	if err != nil {
		return
	}

	// Make sure we clean-up upon error.  In the happy case the caller
	// is expected to manage the clean-up
	defer func() {
		if err != nil {
			if e := os.RemoveAll(tmpdirPath); e != nil {
				logrus.Debugf("[BUILDER] failed to remove temporary context: %s", e)
			}
		}
	}()

	decompressedStream, err := archive.DecompressStream(context)
	if err != nil {
		return
	}

	if b.context, err = tarsum.NewTarSum(decompressedStream, true, tarsum.Version1); err != nil {
		return
	}

	if err = chrootarchive.Untar(b.context, tmpdirPath, nil); err != nil {
		return
	}

	b.contextPath = tmpdirPath
	return
}

func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS, id string, progressOutput progress.Output) (layer.Layer, error) {
	rawTar, err := os.Open(filename)
	if err != nil {
		logrus.Debugf("Error reading embedded tar: %v", err)
		return nil, err
	}
	defer rawTar.Close()

	inflatedLayerData, err := archive.DecompressStream(rawTar)
	if err != nil {
		return nil, err
	}
	defer inflatedLayerData.Close()

	if progressOutput != nil {
		fileInfo, err := os.Stat(filename)
		if err != nil {
			logrus.Debugf("Error statting file: %v", err)
			return nil, err
		}

		progressReader := progress.NewProgressReader(inflatedLayerData, progressOutput, fileInfo.Size(), stringid.TruncateID(id), "Loading layer")

		return l.ls.Register(progressReader, rootFS.ChainID())
	}
	return l.ls.Register(inflatedLayerData, rootFS.ChainID())
}

// Untar reads a stream of bytes from `archive`, parses it as a tar archive,
// and unpacks it into the directory at `dest`.
// The archive may be compressed with one of the following algorithms:
//  identity (uncompressed), gzip, bzip2, xz.
func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error {

	if tarArchive == nil {
		return fmt.Errorf("Empty archive")
	}
	if options == nil {
		options = &archive.TarOptions{}
	}
	if options.ExcludePatterns == nil {
		options.ExcludePatterns = []string{}
	}

	dest = filepath.Clean(dest)
	if _, err := os.Stat(dest); os.IsNotExist(err) {
		if err := system.MkdirAll(dest, 0777); err != nil {
			return err
		}
	}

	decompressedArchive, err := archive.DecompressStream(tarArchive)
	if err != nil {
		return err
	}
	defer decompressedArchive.Close()

	return invokeUnpack(decompressedArchive, dest, options)
}

// MakeTarSumContext returns a build Context from a tar stream.
//
// It extracts the tar stream to a temporary folder that is deleted as soon as
// the Context is closed.
// As the extraction happens, a tarsum is calculated for every file, and the set of
// all those sums then becomes the source of truth for all operations on this Context.
//
// Closing tarStream has to be done by the caller.
func MakeTarSumContext(tarStream io.Reader) (ModifiableContext, error) {
	root, err := ioutils.TempDir("", "docker-builder")
	if err != nil {
		return nil, err
	}

	tsc := &tarSumContext{root: root}

	// Make sure we clean-up upon error.  In the happy case the caller
	// is expected to manage the clean-up
	defer func() {
		if err != nil {
			tsc.Close()
		}
	}()

	decompressedStream, err := archive.DecompressStream(tarStream)
	if err != nil {
		return nil, err
	}

	sum, err := tarsum.NewTarSum(decompressedStream, true, tarsum.Version1)
	if err != nil {
		return nil, err
	}

	if err := chrootarchive.Untar(sum, root, nil); err != nil {
		return nil, err
	}

	tsc.sums = sum.GetSums()

	return tsc, nil
}

func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error {
	if tarArchive == nil {
		return fmt.Errorf("Empty archive")
	}
	if options == nil {
		options = &archive.TarOptions{}
	}
	if options.ExcludePatterns == nil {
		options.ExcludePatterns = []string{}
	}

	dest = filepath.Clean(dest)
	if _, err := os.Stat(dest); os.IsNotExist(err) {
		if err := os.MkdirAll(dest, 0777); err != nil {
			return err
		}
	}

	decompressedArchive, err := archive.DecompressStream(tarArchive)
	if err != nil {
		return err
	}
	defer decompressedArchive.Close()

	// We can't pass a potentially large exclude list directly via cmd line
	// because we easily overrun the kernel's max argument/environment size
	// when the full image list is passed (e.g. when this is used by
	// `docker load`). We will marshall the options via a pipe to the
	// child
	r, w, err := os.Pipe()
	if err != nil {
		return fmt.Errorf("Untar pipe failure: %v", err)
	}
	cmd := reexec.Command("docker-untar", dest)
	cmd.Stdin = decompressedArchive
	cmd.ExtraFiles = append(cmd.ExtraFiles, r)
	var output bytes.Buffer
	cmd.Stdout = &output
	cmd.Stderr = &output

	if err := cmd.Start(); err != nil {
		return fmt.Errorf("Untar error on re-exec cmd: %v", err)
	}
	//write the options to the pipe for the untar exec to read
	if err := json.NewEncoder(w).Encode(options); err != nil {
		return fmt.Errorf("Untar json encode to pipe failed: %v", err)
	}
	w.Close()

	if err := cmd.Wait(); err != nil {
		return fmt.Errorf("Untar re-exec error: %v: output: %s", err, output)
	}
	return nil
}

// StoreImage stores file system layer data for the given image to the
// image's registered storage driver. Image metadata is stored in a file
// at the specified root directory. This function also computes the TarSum
// of `layerData` (currently using tarsum.dev).
func StoreImage(img *Image, layerData archive.ArchiveReader, root string) error {
	// Store the layer
	var (
		size        int64
		err         error
		driver      = img.graph.Driver()
		layerTarSum tarsum.TarSum
	)

	// If layerData is not nil, unpack it into the new layer
	if layerData != nil {
		// If the image doesn't have a checksum, we should add it. The layer
		// checksums are verified when they are pulled from a remote, but when
		// a container is committed it should be added here.
		if img.Checksum == "" {
			layerDataDecompressed, err := archive.DecompressStream(layerData)
			if err != nil {
				return err
			}
			defer layerDataDecompressed.Close()

			if layerTarSum, err = tarsum.NewTarSum(layerDataDecompressed, true, tarsum.VersionDev); err != nil {
				return err
			}

			if size, err = driver.ApplyDiff(img.ID, img.Parent, layerTarSum); err != nil {
				return err
			}

			img.Checksum = layerTarSum.Sum(nil)
		} else if size, err = driver.ApplyDiff(img.ID, img.Parent, layerData); err != nil {
			return err
		}

	}

	img.Size = size
	if err := img.SaveSize(root); err != nil {
		return err
	}

	f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
	if err != nil {
		return err
	}

	defer f.Close()

	return json.NewEncoder(f).Encode(img)
}

// StoreImage stores file system layer data for the given image to the
// image's registered storage driver. Image metadata is stored in a file
// at the specified root directory. This function also computes the TarSum
// of `layerData` (currently using tarsum.dev).
func StoreImage(img *Image, layerData archive.ArchiveReader, root string) error {
	// Store the layer
	var (
		size        int64
		err         error
		driver      = img.graph.Driver()
		layerTarSum tarsum.TarSum
	)

	// If layerData is not nil, unpack it into the new layer
	if layerData != nil {
		layerDataDecompressed, err := archive.DecompressStream(layerData)
		if err != nil {
			return err
		}

		defer layerDataDecompressed.Close()

		if layerTarSum, err = tarsum.NewTarSum(layerDataDecompressed, true, tarsum.VersionDev); err != nil {
			return err
		}

		if size, err = driver.ApplyDiff(img.ID, img.Parent, layerTarSum); err != nil {
			return err
		}

		checksum := layerTarSum.Sum(nil)

		if img.Checksum != "" && img.Checksum != checksum {
			log.Warnf("image layer checksum mismatch: computed %q, expected %q", checksum, img.Checksum)
		}

		img.Checksum = checksum
	}

	img.Size = size
	if err := img.SaveSize(root); err != nil {
		return err
	}

	f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
	if err != nil {
		return err
	}

	defer f.Close()

	return json.NewEncoder(f).Encode(img)
}

func (l *tarexporter) loadLayer(filename string, rootFS image.RootFS) (layer.Layer, error) {
	rawTar, err := os.Open(filename)
	if err != nil {
		logrus.Debugf("Error reading embedded tar: %v", err)
		return nil, err
	}
	inflatedLayerData, err := archive.DecompressStream(rawTar)
	if err != nil {
		return nil, err
	}

	defer rawTar.Close()
	defer inflatedLayerData.Close()

	return l.ls.Register(inflatedLayerData, rootFS.ChainID())
}

func (b blobChanger) addFile(rw http.ResponseWriter, r *http.Request) {
	recorder := httptest.NewRecorder()

	b.Handler.ServeHTTP(recorder, r)

	inflated, err := archive.DecompressStream(bytes.NewReader(recorder.Body.Bytes()))
	if err != nil {
		logrus.Errorf("Error decompressing: %s", err)
		http.Error(rw, "Error handling tar stream in proxy", 500)
		return
	}

	copied := bytes.NewBuffer(nil)
	deflater, err := archive.CompressStream(writeCloser{copied}, archive.Gzip)
	if err != nil {
		logrus.Errorf("Error compressing: %s", err)
		http.Error(rw, "Error handling tar stream in proxy", 500)
		return
	}

	tw := tar.NewWriter(deflater)
	if err := addFile(tw, "/etc/malicious.txt", []byte("#Bad bad stuff")); err != nil {
		logrus.Errorf("Error adding file: %s", err)
		http.Error(rw, "Error handling tar stream in proxy", 500)
		return
	}

	if err := tarCopy(tw, tar.NewReader(inflated)); err != nil {
		logrus.Errorf("Error copying: %s", err)
		http.Error(rw, "Error handling tar stream in proxy", 500)
		return
	}

	recorder.Header().Set("Content-Length", strconv.Itoa(len(copied.Bytes())))
	copyHeader(rw.Header(), recorder.Header())
	rw.WriteHeader(recorder.Code)

	n, err := rw.Write(copied.Bytes())
	if err != nil {
		logrus.Errorf("Error writing: %s", err)
		return
	}
	if n != copied.Len() {
		logrus.Errorf("Short write: wrote %d, expected %d", n, copied.Len())
	}
}