Golang tarsum.NewTarSum函数代码示例

func writeReadTar(t *testing.T, tmpDir string, tarStream io.ReadCloser) string {
	data, err := ioutil.ReadAll(tarStream)
	if err != nil {
		t.Fatal(err)
	}
	defer tarStream.Close()

	tarSum, err := tarsum.NewTarSum(bytes.NewReader(data), true, tarsum.Version1)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := io.Copy(ioutil.Discard, tarSum); err != nil {
		t.Fatal(err)
	}
	t.Logf("tarsum: %s", tarSum.Sum(nil))

	if err := ioutil.WriteFile(tmpDir+"/archive.tar", data, 0644); err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpDir + "/archive.tar")

	cmd := exec.Command("tar", "-tf", tmpDir+"/archive.tar")
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatal(err)
	}

	return string(out)
}

func (b *builder) readContext(context io.Reader) (err error) {
	tmpdirPath, err := ioutil.TempDir("", "docker-build")
	if err != nil {
		return
	}

	// Make sure we clean-up upon error.  In the happy case the caller
	// is expected to manage the clean-up
	defer func() {
		if err != nil {
			if e := os.RemoveAll(tmpdirPath); e != nil {
				logrus.Debugf("[BUILDER] failed to remove temporary context: %s", e)
			}
		}
	}()

	decompressedStream, err := archive.DecompressStream(context)
	if err != nil {
		return
	}

	if b.context, err = tarsum.NewTarSum(decompressedStream, true, tarsum.Version1); err != nil {
		return
	}

	if err = chrootarchive.Untar(b.context, tmpdirPath, nil); err != nil {
		return
	}

	b.contextPath = tmpdirPath
	return
}

// MakeTarSumContext returns a build Context from a tar stream.
//
// It extracts the tar stream to a temporary folder that is deleted as soon as
// the Context is closed.
// As the extraction happens, a tarsum is calculated for every file, and the set of
// all those sums then becomes the source of truth for all operations on this Context.
//
// Closing tarStream has to be done by the caller.
func MakeTarSumContext(tarStream io.Reader) (ModifiableContext, error) {
	root, err := ioutils.TempDir("", "docker-builder")
	if err != nil {
		return nil, err
	}

	tsc := &tarSumContext{root: root}

	// Make sure we clean-up upon error.  In the happy case the caller
	// is expected to manage the clean-up
	defer func() {
		if err != nil {
			tsc.Close()
		}
	}()

	decompressedStream, err := archive.DecompressStream(tarStream)
	if err != nil {
		return nil, err
	}

	sum, err := tarsum.NewTarSum(decompressedStream, true, tarsum.Version1)
	if err != nil {
		return nil, err
	}

	if err := chrootarchive.Untar(sum, root, nil); err != nil {
		return nil, err
	}

	tsc.sums = sum.GetSums()

	return tsc, nil
}

// StoreImage stores file system layer data for the given image to the
// image's registered storage driver. Image metadata is stored in a file
// at the specified root directory. This function also computes the TarSum
// of `layerData` (currently using tarsum.dev).
func StoreImage(img *Image, layerData archive.ArchiveReader, root string) error {
	// Store the layer
	var (
		size        int64
		err         error
		driver      = img.graph.Driver()
		layerTarSum tarsum.TarSum
	)

	// If layerData is not nil, unpack it into the new layer
	if layerData != nil {
		// If the image doesn't have a checksum, we should add it. The layer
		// checksums are verified when they are pulled from a remote, but when
		// a container is committed it should be added here.
		if img.Checksum == "" {
			layerDataDecompressed, err := archive.DecompressStream(layerData)
			if err != nil {
				return err
			}
			defer layerDataDecompressed.Close()

			if layerTarSum, err = tarsum.NewTarSum(layerDataDecompressed, true, tarsum.VersionDev); err != nil {
				return err
			}

			if size, err = driver.ApplyDiff(img.ID, img.Parent, layerTarSum); err != nil {
				return err
			}

			img.Checksum = layerTarSum.Sum(nil)
		} else if size, err = driver.ApplyDiff(img.ID, img.Parent, layerData); err != nil {
			return err
		}

	}

	img.Size = size
	if err := img.SaveSize(root); err != nil {
		return err
	}

	f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
	if err != nil {
		return err
	}

	defer f.Close()

	return json.NewEncoder(f).Encode(img)
}

// StoreImage stores file system layer data for the given image to the
// image's registered storage driver. Image metadata is stored in a file
// at the specified root directory. This function also computes the TarSum
// of `layerData` (currently using tarsum.dev).
func StoreImage(img *Image, layerData archive.ArchiveReader, root string) error {
	// Store the layer
	var (
		size        int64
		err         error
		driver      = img.graph.Driver()
		layerTarSum tarsum.TarSum
	)

	// If layerData is not nil, unpack it into the new layer
	if layerData != nil {
		layerDataDecompressed, err := archive.DecompressStream(layerData)
		if err != nil {
			return err
		}

		defer layerDataDecompressed.Close()

		if layerTarSum, err = tarsum.NewTarSum(layerDataDecompressed, true, tarsum.VersionDev); err != nil {
			return err
		}

		if size, err = driver.ApplyDiff(img.ID, img.Parent, layerTarSum); err != nil {
			return err
		}

		checksum := layerTarSum.Sum(nil)

		if img.Checksum != "" && img.Checksum != checksum {
			log.Warnf("image layer checksum mismatch: computed %q, expected %q", checksum, img.Checksum)
		}

		img.Checksum = checksum
	}

	img.Size = size
	if err := img.SaveSize(root); err != nil {
		return err
	}

	f, err := os.OpenFile(jsonPath(root), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0600))
	if err != nil {
		return err
	}

	defer f.Close()

	return json.NewEncoder(f).Encode(img)
}

// NewDigestVerifier returns a verifier that compares the written bytes
// against a passed in digest.
func NewDigestVerifier(d Digest) (Verifier, error) {
	if err := d.Validate(); err != nil {
		return nil, err
	}

	alg := d.Algorithm()
	switch alg {
	case "sha256", "sha384", "sha512":
		return hashVerifier{
			hash:   alg.Hash(),
			digest: d,
		}, nil
	default:
		// Assume we have a tarsum.
		version, err := tarsum.GetVersionFromTarsum(string(d))
		if err != nil {
			return nil, err
		}

		pr, pw := io.Pipe()

		// TODO(stevvooe): We may actually want to ban the earlier versions of
		// tarsum. That decision may not be the place of the verifier.

		ts, err := tarsum.NewTarSum(pr, true, version)
		if err != nil {
			return nil, err
		}

		// TODO(sday): Ick! A goroutine per digest verification? We'll have to
		// get the tarsum library to export an io.Writer variant.
		go func() {
			if _, err := io.Copy(ioutil.Discard, ts); err != nil {
				pr.CloseWithError(err)
			} else {
				pr.Close()
			}
		}()

		return &tarsumVerifier{
			digest: d,
			ts:     ts,
			pr:     pr,
			pw:     pw,
		}, nil
	}
}

// FromTarArchive produces a tarsum digest from reader rd.
func FromTarArchive(rd io.Reader) (Digest, error) {
	ts, err := tarsum.NewTarSum(rd, true, tarsum.Version1)
	if err != nil {
		return "", err
	}

	if _, err := io.Copy(ioutil.Discard, ts); err != nil {
		return "", err
	}

	d, err := ParseDigest(ts.Sum(nil))
	if err != nil {
		return "", err
	}

	return d, nil
}

func (r *Session) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, jsonRaw []byte) (checksum string, checksumPayload string, err error) {

	u := registry + "images/" + imgID + "/layer"

	logrus.Debugf("[registry] Calling PUT %s", u)

	tarsumLayer, err := tarsum.NewTarSum(layer, false, tarsum.Version0)
	if err != nil {
		return "", "", err
	}
	h := sha256.New()
	h.Write(jsonRaw)
	h.Write([]byte{'\n'})
	checksumLayer := io.TeeReader(tarsumLayer, h)

	req, err := http.NewRequest("PUT", u, checksumLayer)
	if err != nil {
		return "", "", err
	}
	req.Header.Add("Content-Type", "application/octet-stream")
	req.ContentLength = -1
	req.TransferEncoding = []string{"chunked"}
	res, err := r.client.Do(req)
	if err != nil {
		return "", "", fmt.Errorf("Failed to upload layer: %v", err)
	}
	if rc, ok := layer.(io.Closer); ok {
		if err := rc.Close(); err != nil {
			return "", "", err
		}
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		errBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return "", "", httputils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res)
		}
		return "", "", httputils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %q", res.StatusCode, errBody), res)
	}

	checksumPayload = "sha256:" + hex.EncodeToString(h.Sum(nil))
	return tarsumLayer.Sum(jsonRaw), checksumPayload, nil
}

func (b *Builder) readContext(context io.Reader) error {
	tmpdirPath, err := ioutil.TempDir("", "docker-build")
	if err != nil {
		return err
	}

	decompressedStream, err := archive.DecompressStream(context)
	if err != nil {
		return err
	}

	if b.context, err = tarsum.NewTarSum(decompressedStream, true, tarsum.Version0); err != nil {
		return err
	}
	if err := archive.Untar(b.context, tmpdirPath, nil); err != nil {
		return err
	}

	b.contextPath = tmpdirPath
	return nil
}

// if out is not nil, then the tar input is written there instead
func SumTarLayerVersioned(tarReader io.Reader, json io.Reader, out io.Writer, v tarsum.Version) (string, error) {
	var writer io.Writer = ioutil.Discard
	if out != nil {
		writer = out
	}
	ts, err := tarsum.NewTarSum(tarReader, false, v)
	if err != nil {
		return "", err
	}
	_, err = io.Copy(writer, ts)
	if err != nil {
		return "", err
	}

	var buf []byte
	if json != nil {
		if buf, err = ioutil.ReadAll(json); err != nil {
			return "", err
		}
	}

	return ts.Sum(buf), nil
}

//.........这里部分代码省略.........
				times[1] = syscall.NsecToTimespec(mTime.UnixNano())
			}
		}

		if err := system.UtimesNano(tmpFileName, times); err != nil {
			return err
		}

		ci.origPath = path.Join(filepath.Base(tmpDirName), filepath.Base(tmpFileName))

		// If the destination is a directory, figure out the filename.
		if strings.HasSuffix(ci.destPath, "/") {
			u, err := url.Parse(origPath)
			if err != nil {
				return err
			}
			path := u.Path
			if strings.HasSuffix(path, "/") {
				path = path[:len(path)-1]
			}
			parts := strings.Split(path, "/")
			filename := parts[len(parts)-1]
			if filename == "" {
				return fmt.Errorf("cannot determine filename from url: %s", u)
			}
			ci.destPath = ci.destPath + filename
		}

		// Calc the checksum, even if we're using the cache
		r, err := archive.Tar(tmpFileName, archive.Uncompressed)
		if err != nil {
			return err
		}
		tarSum, err := tarsum.NewTarSum(r, true, tarsum.Version0)
		if err != nil {
			return err
		}
		if _, err := io.Copy(ioutil.Discard, tarSum); err != nil {
			return err
		}
		ci.hash = tarSum.Sum(nil)
		r.Close()

		return nil
	}

	// Deal with wildcards
	if ContainsWildcards(origPath) {
		for _, fileInfo := range b.context.GetSums() {
			if fileInfo.Name() == "" {
				continue
			}
			match, _ := path.Match(origPath, fileInfo.Name())
			if !match {
				continue
			}

			calcCopyInfo(b, cmdName, cInfos, fileInfo.Name(), destPath, allowRemote, allowDecompression)
		}
		return nil
	}

	// Must be a dir or a file

	if err := b.checkPathForAddition(origPath); err != nil {
		return err

func (b *Builder) download(srcURL string) (fi builder.FileInfo, err error) {
	// get filename from URL
	u, err := url.Parse(srcURL)
	if err != nil {
		return
	}
	path := filepath.FromSlash(u.Path) // Ensure in platform semantics
	if strings.HasSuffix(path, string(os.PathSeparator)) {
		path = path[:len(path)-1]
	}
	parts := strings.Split(path, string(os.PathSeparator))
	filename := parts[len(parts)-1]
	if filename == "" {
		err = fmt.Errorf("cannot determine filename from url: %s", u)
		return
	}

	// Initiate the download
	resp, err := httputils.Download(srcURL)
	if err != nil {
		return
	}

	// Prepare file in a tmp dir
	tmpDir, err := ioutils.TempDir("", "docker-remote")
	if err != nil {
		return
	}
	defer func() {
		if err != nil {
			os.RemoveAll(tmpDir)
		}
	}()
	tmpFileName := filepath.Join(tmpDir, filename)
	tmpFile, err := os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
	if err != nil {
		return
	}

	stdoutFormatter := b.Stdout.(*streamformatter.StdoutFormatter)
	progressOutput := stdoutFormatter.StreamFormatter.NewProgressOutput(stdoutFormatter.Writer, true)
	progressReader := progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Downloading")
	// Download and dump result to tmp file
	if _, err = io.Copy(tmpFile, progressReader); err != nil {
		tmpFile.Close()
		return
	}
	fmt.Fprintln(b.Stdout)
	// ignoring error because the file was already opened successfully
	tmpFileSt, err := tmpFile.Stat()
	if err != nil {
		return
	}
	tmpFile.Close()

	// Set the mtime to the Last-Modified header value if present
	// Otherwise just remove atime and mtime
	mTime := time.Time{}

	lastMod := resp.Header.Get("Last-Modified")
	if lastMod != "" {
		// If we can't parse it then just let it default to 'zero'
		// otherwise use the parsed time value
		if parsedMTime, err := http.ParseTime(lastMod); err == nil {
			mTime = parsedMTime
		}
	}

	if err = system.Chtimes(tmpFileName, mTime, mTime); err != nil {
		return
	}

	// Calc the checksum, even if we're using the cache
	r, err := archive.Tar(tmpFileName, archive.Uncompressed)
	if err != nil {
		return
	}
	tarSum, err := tarsum.NewTarSum(r, true, tarsum.Version1)
	if err != nil {
		return
	}
	if _, err = io.Copy(ioutil.Discard, tarSum); err != nil {
		return
	}
	hash := tarSum.Sum(nil)
	r.Close()
	return &builder.HashedFileInfo{FileInfo: builder.PathFileInfo{FileInfo: tmpFileSt, FilePath: tmpFileName}, FileHash: hash}, nil
}

func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error) {
	manifest := &registry.ManifestData{
		Name:          remoteName,
		Tag:           tag,
		SchemaVersion: 1,
	}
	localRepo, err := s.Get(localName)
	if err != nil {
		return nil, err
	}
	if localRepo == nil {
		return nil, fmt.Errorf("Repo does not exist: %s", localName)
	}

	// Get the top-most layer id which the tag points to
	layerId, exists := localRepo[tag]
	if !exists {
		return nil, fmt.Errorf("Tag does not exist for %s: %s", localName, tag)
	}
	layersSeen := make(map[string]bool)

	layer, err := s.graph.Get(layerId)
	if err != nil {
		return nil, err
	}
	manifest.Architecture = layer.Architecture
	manifest.FSLayers = make([]*registry.FSLayer, 0, 4)
	manifest.History = make([]*registry.ManifestHistory, 0, 4)
	var metadata runconfig.Config
	if layer.Config != nil {
		metadata = *layer.Config
	}

	for ; layer != nil; layer, err = layer.GetParent() {
		if err != nil {
			return nil, err
		}

		if layersSeen[layer.ID] {
			break
		}
		if layer.Config != nil && metadata.Image != layer.ID {
			err = runconfig.Merge(&metadata, layer.Config)
			if err != nil {
				return nil, err
			}
		}

		checksum, err := layer.GetCheckSum(s.graph.ImageRoot(layer.ID))
		if err != nil {
			return nil, fmt.Errorf("Error getting image checksum: %s", err)
		}
		if tarsum.VersionLabelForChecksum(checksum) != tarsum.Version1.String() {
			archive, err := layer.TarLayer()
			if err != nil {
				return nil, err
			}

			tarSum, err := tarsum.NewTarSum(archive, true, tarsum.Version1)
			if err != nil {
				return nil, err
			}
			if _, err := io.Copy(ioutil.Discard, tarSum); err != nil {
				return nil, err
			}

			checksum = tarSum.Sum(nil)

			// Save checksum value
			if err := layer.SaveCheckSum(s.graph.ImageRoot(layer.ID), checksum); err != nil {
				return nil, err
			}
		}

		jsonData, err := layer.RawJson()
		if err != nil {
			return nil, fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err)
		}

		manifest.FSLayers = append(manifest.FSLayers, &registry.FSLayer{BlobSum: checksum})

		layersSeen[layer.ID] = true

		manifest.History = append(manifest.History, &registry.ManifestHistory{V1Compatibility: string(jsonData)})
	}

	manifestBytes, err := json.MarshalIndent(manifest, "", "   ")
	if err != nil {
		return nil, err
	}

	return manifestBytes, nil
}

func calcCopyInfo(b *Builder, cmdName string, ci *copyInfo, allowRemote bool, allowDecompression bool) error {
	var (
		remoteHash string
		isRemote   bool
	)

	saveOrig := ci.origPath
	isRemote = utils.IsURL(ci.origPath)

	if isRemote && !allowRemote {
		return fmt.Errorf("Source can't be an URL for %s", cmdName)
	} else if isRemote {
		// Initiate the download
		resp, err := utils.Download(ci.origPath)
		if err != nil {
			return err
		}

		// Create a tmp dir
		tmpDirName, err := ioutil.TempDir(b.contextPath, "docker-remote")
		if err != nil {
			return err
		}
		ci.tmpDir = tmpDirName

		// Create a tmp file within our tmp dir
		tmpFileName := path.Join(tmpDirName, "tmp")
		tmpFile, err := os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			return err
		}

		// Download and dump result to tmp file
		if _, err := io.Copy(tmpFile, utils.ProgressReader(resp.Body, int(resp.ContentLength), b.OutOld, b.StreamFormatter, true, "", "Downloading")); err != nil {
			tmpFile.Close()
			return err
		}
		fmt.Fprintf(b.OutStream, "\n")
		tmpFile.Close()

		// Remove the mtime of the newly created tmp file
		if err := system.UtimesNano(tmpFileName, make([]syscall.Timespec, 2)); err != nil {
			return err
		}

		ci.origPath = path.Join(filepath.Base(tmpDirName), filepath.Base(tmpFileName))

		// Process the checksum
		r, err := archive.Tar(tmpFileName, archive.Uncompressed)
		if err != nil {
			return err
		}
		tarSum, err := tarsum.NewTarSum(r, true, tarsum.Version0)
		if err != nil {
			return err
		}
		if _, err := io.Copy(ioutil.Discard, tarSum); err != nil {
			return err
		}
		remoteHash = tarSum.Sum(nil)
		r.Close()

		// If the destination is a directory, figure out the filename.
		if strings.HasSuffix(ci.destPath, "/") {
			u, err := url.Parse(saveOrig)
			if err != nil {
				return err
			}
			path := u.Path
			if strings.HasSuffix(path, "/") {
				path = path[:len(path)-1]
			}
			parts := strings.Split(path, "/")
			filename := parts[len(parts)-1]
			if filename == "" {
				return fmt.Errorf("cannot determine filename from url: %s", u)
			}
			ci.destPath = ci.destPath + filename
		}
	}

	if err := b.checkPathForAddition(ci.origPath); err != nil {
		return err
	}

	// Hash path and check the cache
	if b.UtilizeCache {
		var (
			sums = b.context.GetSums()
		)

		if remoteHash != "" {
			ci.hashPath = remoteHash
		} else if fi, err := os.Stat(path.Join(b.contextPath, ci.origPath)); err != nil {
			return err
		} else if fi.IsDir() {
			var subfiles []string
			for _, fileInfo := range sums {
				absFile := path.Join(b.contextPath, fileInfo.Name())
				absOrigPath := path.Join(b.contextPath, ci.origPath)
//.........这里部分代码省略.........

// CreateRandomTarFile creates a random tarfile, returning it as an
// io.ReadSeeker along with its tarsum. An error is returned if there is a
// problem generating valid content.
func CreateRandomTarFile() (rs io.ReadSeeker, tarSum string, err error) {
	nFiles := mrand.Intn(10) + 10
	target := &bytes.Buffer{}
	wr := tar.NewWriter(target)

	// Perturb this on each iteration of the loop below.
	header := &tar.Header{
		Mode:       0644,
		ModTime:    time.Now(),
		Typeflag:   tar.TypeReg,
		Uname:      "randocalrissian",
		Gname:      "cloudcity",
		AccessTime: time.Now(),
		ChangeTime: time.Now(),
	}

	for fileNumber := 0; fileNumber < nFiles; fileNumber++ {
		fileSize := mrand.Int63n(1<<20) + 1<<20

		header.Name = fmt.Sprint(fileNumber)
		header.Size = fileSize

		if err := wr.WriteHeader(header); err != nil {
			return nil, "", err
		}

		randomData := make([]byte, fileSize)

		// Fill up the buffer with some random data.
		n, err := rand.Read(randomData)

		if n != len(randomData) {
			return nil, "", fmt.Errorf("short read creating random reader: %v bytes != %v bytes", n, len(randomData))
		}

		if err != nil {
			return nil, "", err
		}

		nn, err := io.Copy(wr, bytes.NewReader(randomData))
		if nn != fileSize {
			return nil, "", fmt.Errorf("short copy writing random file to tar")
		}

		if err != nil {
			return nil, "", err
		}

		if err := wr.Flush(); err != nil {
			return nil, "", err
		}
	}

	if err := wr.Close(); err != nil {
		return nil, "", err
	}

	reader := bytes.NewReader(target.Bytes())

	// A tar builder that supports tarsum inline calculation would be awesome
	// here.
	ts, err := tarsum.NewTarSum(reader, true, tarsum.Version1)
	if err != nil {
		return nil, "", err
	}

	nn, err := io.Copy(ioutil.Discard, ts)
	if nn != int64(len(target.Bytes())) {
		return nil, "", fmt.Errorf("short copy when getting tarsum of random layer: %v != %v", nn, len(target.Bytes()))
	}

	if err != nil {
		return nil, "", err
	}

	return bytes.NewReader(target.Bytes()), ts.Sum(nil), nil
}