Golang fs.NewMultiHasher函數代碼示例

func TestMultiHasher(t *testing.T) {
	for _, test := range hashTestSet {
		mh := fs.NewMultiHasher()
		n, err := io.Copy(mh, bytes.NewBuffer(test.input))
		if err != nil {
			t.Fatal(err)
		}
		if int(n) != len(test.input) {
			t.Fatalf("copy mismatch: %d != %d", n, len(test.input))
		}
		sums := mh.Sums()
		for k, v := range sums {
			expect, ok := test.output[k]
			if !ok {
				t.Errorf("Unknown hash type %v, sum: %q", k, v)
			}
			if expect != v {
				t.Errorf("hash %v mismatch %q != %q", k, v, expect)
			}
		}
		// Test that all are present
		for k, v := range test.output {
			expect, ok := sums[k]
			if !ok {
				t.Errorf("did not calculate hash type %v, sum: %q", k, v)
			}
			if expect != v {
				t.Errorf("hash %d mismatch %q != %q", k, v, expect)
			}
		}
	}
}

func testPut(t *testing.T, file *fstest.Item) {
again:
	buf := bytes.NewBufferString(fstest.RandomString(100))
	hash := fs.NewMultiHasher()
	in := io.TeeReader(buf, hash)

	tries := 1
	const maxTries = 10
	file.Size = int64(buf.Len())
	obji := fs.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
	obj, err := remote.Put(in, obji)
	if err != nil {
		// Retry if err returned a retry error
		if fs.IsRetryError(err) && tries < maxTries {
			t.Logf("Put error: %v - low level retry %d/%d", err, tries, maxTries)
			time.Sleep(2 * time.Second)

			tries++
			goto again
		}
		require.NoError(t, err, "Put error")
	}
	file.Hashes = hash.Sums()
	file.Check(t, obj, remote.Precision())
	// Re-read the object and check again
	obj = findObject(t, file.Path)
	file.Check(t, obj, remote.Precision())
}

// Update the object from in with modTime and size
func (o *Object) Update(in io.Reader, modTime time.Time, size int64) error {
	err := o.mkdirAll()
	if err != nil {
		return err
	}

	out, err := os.Create(o.path)
	if err != nil {
		return err
	}

	// Calculate the md5sum of the object we are reading as we go along
	hash := fs.NewMultiHasher()
	in = io.TeeReader(in, hash)

	_, err = io.Copy(out, in)
	outErr := out.Close()
	if err != nil {
		return err
	}
	if outErr != nil {
		return outErr
	}

	// All successful so update the hashes
	o.hashes = hash.Sums()

	// Set the mtime
	o.SetModTime(modTime)

	// ReRead info now that we have finished
	return o.lstat()
}

// Open an object for read
func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
	var offset int64
	for _, option := range options {
		switch x := option.(type) {
		case *fs.SeekOption:
			offset = x.Offset
		default:
			if option.Mandatory() {
				fs.Log(o, "Unsupported mandatory option: %v", option)
			}
		}
	}

	fd, err := os.Open(o.path)
	if err != nil {
		return
	}
	if offset != 0 {
		// seek the object
		_, err = fd.Seek(offset, 0)
		// don't attempt to make checksums
		return fd, err
	}
	// Update the md5sum as we go along
	in = &localOpenFile{
		o:    o,
		in:   fd,
		hash: fs.NewMultiHasher(),
	}
	return in, nil
}

// TestObjectOpen tests that Open works
func TestObjectOpen(t *testing.T) {
	skipIfNotOk(t)
	obj := findObject(t, file1.Path)
	in, err := obj.Open()
	if err != nil {
		t.Fatalf("Open() return error: %v", err)
	}
	hasher := fs.NewMultiHasher()
	n, err := io.Copy(hasher, in)
	if err != nil {
		t.Fatalf("io.Copy() return error: %v", err)
	}
	if n != file1.Size {
		t.Fatalf("Read wrong number of bytes %d != %d", n, file1.Size)
	}
	err = in.Close()
	if err != nil {
		t.Fatalf("in.Close() return error: %v", err)
	}
	// Check content of file by comparing the calculated hashes
	for hashType, got := range hasher.Sums() {
		want := file1.Hashes[hashType]
		if want != got {
			t.Errorf("%v is wrong %v != %v", hashType, want, got)
		}
	}

}

// TestObjectUpdate tests that Update works
func TestObjectUpdate(t *testing.T) {
	skipIfNotOk(t)
	contents := fstest.RandomString(200)
	buf := bytes.NewBufferString(contents)
	hash := fs.NewMultiHasher()
	in := io.TeeReader(buf, hash)

	file1.Size = int64(buf.Len())
	obj := findObject(t, file1.Path)
	obji := fs.NewStaticObjectInfo(file1.Path, file1.ModTime, int64(len(contents)), true, nil, obj.Fs())
	err := obj.Update(in, obji)
	require.NoError(t, err)
	file1.Hashes = hash.Sums()

	// check the object has been updated
	file1.Check(t, obj, remote.Precision())

	// Re-read the object and check again
	obj = findObject(t, file1.Path)
	file1.Check(t, obj, remote.Precision())

	// check contents correct
	assert.Equal(t, contents, readObject(t, obj), "contents of updated file1 differ")
	file1Contents = contents
}

// Open an object for read
func (o *Object) Open() (in io.ReadCloser, err error) {
	in, err = os.Open(o.path)
	if err != nil {
		return
	}
	// Update the md5sum as we go along
	in = &localOpenFile{
		o:    o,
		in:   in,
		hash: fs.NewMultiHasher(),
	}
	return
}

// NewItem creates an item from a string content
func NewItem(Path, Content string, modTime time.Time) Item {
	i := Item{
		Path:    Path,
		ModTime: modTime,
		Size:    int64(len(Content)),
	}
	hash := fs.NewMultiHasher()
	buf := bytes.NewBufferString(Content)
	_, err := io.Copy(hash, buf)
	if err != nil {
		log.Fatalf("Failed to create item: %v", err)
	}
	i.Hashes = hash.Sums()
	return i
}

func testPut(t *testing.T, file *fstest.Item) {
	buf := bytes.NewBufferString(fstest.RandomString(100))
	hash := fs.NewMultiHasher()
	in := io.TeeReader(buf, hash)

	file.Size = int64(buf.Len())
	obj, err := remote.Put(in, file.Path, file.ModTime, file.Size)
	if err != nil {
		t.Fatal("Put error", err)
	}
	file.Hashes = hash.Sums()
	file.Check(t, obj, remote.Precision())
	// Re-read the object and check again
	obj = findObject(t, file.Path)
	file.Check(t, obj, remote.Precision())
}

// TestObjectUpdate tests that Update works
func TestObjectUpdate(t *testing.T) {
	skipIfNotOk(t)
	buf := bytes.NewBufferString(fstest.RandomString(200))
	hash := fs.NewMultiHasher()
	in := io.TeeReader(buf, hash)

	file1.Size = int64(buf.Len())
	obj := findObject(t, file1.Path)
	obji := fs.NewStaticObjectInfo("", file1.ModTime, file1.Size, true, nil, obj.Fs())
	err := obj.Update(in, obji)
	require.NoError(t, err)
	file1.Hashes = hash.Sums()
	file1.Check(t, obj, remote.Precision())
	// Re-read the object and check again
	obj = findObject(t, file1.Path)
	file1.Check(t, obj, remote.Precision())
}

// TestObjectOpen tests that Open works
func TestObjectOpen(t *testing.T) {
	skipIfNotOk(t)
	obj := findObject(t, file1.Path)
	in, err := obj.Open()
	require.NoError(t, err)
	hasher := fs.NewMultiHasher()
	n, err := io.Copy(hasher, in)
	require.NoError(t, err)
	require.Equal(t, file1.Size, n, "Read wrong number of bytes")
	err = in.Close()
	require.NoError(t, err)
	// Check content of file by comparing the calculated hashes
	for hashType, got := range hasher.Sums() {
		assert.Equal(t, file1.Hashes[hashType], got)
	}

}

// TestObjectUpdate tests that Update works
func TestObjectUpdate(t *testing.T) {
	skipIfNotOk(t)
	buf := bytes.NewBufferString(fstest.RandomString(200))
	hash := fs.NewMultiHasher()
	in := io.TeeReader(buf, hash)

	file1.Size = int64(buf.Len())
	obj := findObject(t, file1.Path)
	err := obj.Update(in, file1.ModTime, file1.Size)
	if err != nil {
		t.Fatal("Update error", err)
	}
	file1.Hashes = hash.Sums()
	file1.Check(t, obj, remote.Precision())
	// Re-read the object and check again
	obj = findObject(t, file1.Path)
	file1.Check(t, obj, remote.Precision())
}

func TestMultiHasher(t *testing.T) {
	for _, test := range hashTestSet {
		mh := fs.NewMultiHasher()
		n, err := io.Copy(mh, bytes.NewBuffer(test.input))
		require.NoError(t, err)
		assert.Len(t, test.input, int(n))
		sums := mh.Sums()
		for k, v := range sums {
			expect, ok := test.output[k]
			require.True(t, ok)
			assert.Equal(t, v, expect)
		}
		// Test that all are present
		for k, v := range test.output {
			expect, ok := sums[k]
			require.True(t, ok)
			assert.Equal(t, v, expect)
		}
	}
}

// Update the object from in with modTime and size
func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
	err := o.mkdirAll()
	if err != nil {
		return err
	}

	out, err := os.Create(o.path)
	if err != nil {
		return err
	}

	// Calculate the hash of the object we are reading as we go along
	hash := fs.NewMultiHasher()
	in = io.TeeReader(in, hash)

	_, err = io.Copy(out, in)
	closeErr := out.Close()
	if err == nil {
		err = closeErr
	}
	if err != nil {
		fs.Debug(o, "Removing partially written file on error: %v", err)
		if removeErr := os.Remove(o.path); removeErr != nil {
			fs.ErrorLog(o, "Failed to remove partially written file: %v", removeErr)
		}
		return err
	}

	// All successful so update the hashes
	o.hashes = hash.Sums()

	// Set the mtime
	err = o.SetModTime(src.ModTime())
	if err != nil {
		return err
	}

	// ReRead info now that we have finished
	return o.lstat()
}