Golang os.Truncate函数代码示例

func TestDiskQueueWriterCorruption(t *testing.T) {
	l := newTestLogger(t)
	nsqLog.Logger = l
	dqName := "test_disk_queue_corruption" + strconv.Itoa(int(time.Now().Unix()))
	tmpDir, err := ioutil.TempDir("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano()))
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(tmpDir)
	// require a non-zero message length for the corrupt (len 0) test below
	dq, _ := newDiskQueueWriter(dqName, tmpDir, 1000, 10, 1<<10, 1)
	dqReader := newDiskQueueReader(dqName, dqName, tmpDir, 1000, 10, 1<<10, 5, 2*time.Second, nil, true)
	defer dqReader.Close()
	defer dq.Close()

	msg := make([]byte, 123) // 127 bytes per message, 8 (1016 bytes) messages per file
	var e BackendQueueEnd
	for i := 0; i < 25; i++ {
		dq.Put(msg)
	}
	dq.Flush()
	e = dq.GetQueueReadEnd()
	dqReader.UpdateQueueEnd(e, false)

	equal(t, dq.(*diskQueueWriter).diskWriteEnd.TotalMsgCnt(), int64(25))

	// corrupt the 2nd file
	dqFn := dq.(*diskQueueWriter).fileName(1)
	os.Truncate(dqFn, 500) // 3 valid messages, 5 corrupted

	for i := 0; i < 19; i++ { // 1 message leftover in 4th file
		m, _ := dqReader.TryReadOne()
		equal(t, m.Data, msg)
		equal(t, m.Err, nil)
	}

	// corrupt the 4th (current) file
	dqFn = dq.(*diskQueueWriter).fileName(3)
	os.Truncate(dqFn, 100)

	dq.Put(msg) // in 5th file
	dq.Flush()
	e = dq.GetQueueReadEnd()
	dqReader.UpdateQueueEnd(e, true)
	readResult, _ := dqReader.TryReadOne()
	equal(t, readResult.Data, msg)

	// write a corrupt (len 0) message at the 5th (current) file
	dq.(*diskQueueWriter).writeFile.Write([]byte{0, 0, 0, 0})

	// force a new 6th file - put into 5th, then readOne errors, then put into 6th
	dq.Put(msg)
	dq.Put(msg)
	dq.Flush()
	e = dq.GetQueueReadEnd()
	dqReader.UpdateQueueEnd(e, true)
	readResult, _ = dqReader.TryReadOne()

	equal(t, readResult.Data, msg)
}

func TestDiskQueueCorruption(t *testing.T) {
	log.SetOutput(ioutil.Discard)
	defer log.SetOutput(os.Stdout)

	dqName := "test_disk_queue_corruption" + strconv.Itoa(int(time.Now().Unix()))
	dq := NewDiskQueue(dqName, os.TempDir(), 1000, 5)

	msg := make([]byte, 123)
	for i := 0; i < 25; i++ {
		dq.Put(msg)
	}

	assert.Equal(t, dq.Depth(), int64(25))

	// corrupt the 2nd file
	dqFn := dq.(*DiskQueue).fileName(1)
	os.Truncate(dqFn, 500)

	for i := 0; i < 19; i++ {
		assert.Equal(t, <-dq.ReadChan(), msg)
	}

	// corrupt the 4th (current) file
	dqFn = dq.(*DiskQueue).fileName(3)
	os.Truncate(dqFn, 100)

	dq.Put(msg)

	assert.Equal(t, <-dq.ReadChan(), msg)
}

func TestDiskQueueCorruption(t *testing.T) {
	Convey("TestDiskQueueCorruption", t, func() {
		l := log.New(os.Stderr, "", log.LstdFlags)
		dqName := "test_disk_queue_corruption" + strconv.Itoa(int(time.Now().Unix()))
		dq := newDiskQueue(dqName, os.TempDir(), 1000, 5, 2*time.Second, l)

		msg := make([]byte, 123)
		for i := 0; i < 25; i++ {
			dq.Put(msg)
		}

		So(dq.Depth(), ShouldEqual, int64(25))

		// corrupt the 2nd file
		dqFn := dq.(*diskQueue).fileName(1)
		os.Truncate(dqFn, 500)

		for i := 0; i < 19; i++ {
			So(string(<-dq.ReadChan()), ShouldEqual, string(msg))
		}

		// corrupt the 4th (current) file
		dqFn = dq.(*diskQueue).fileName(3)
		os.Truncate(dqFn, 100)

		dq.Put(msg)

		So(string(<-dq.ReadChan()), ShouldEqual, string(msg))
		dq.Delete()
	})
}

func TestDiskQueueCorruption(t *testing.T) {
	l := newTestLogger(t)
	dqName := "test_disk_queue_corruption" + strconv.Itoa(int(time.Now().Unix()))
	tmpDir, err := ioutil.TempDir("", fmt.Sprintf("nsq-test-%d", time.Now().UnixNano()))
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(tmpDir)
	dq := newDiskQueue(dqName, tmpDir, 1000, 5, 2*time.Second, l)

	msg := make([]byte, 123)
	for i := 0; i < 25; i++ {
		dq.Put(msg)
	}

	equal(t, dq.Depth(), int64(25))

	// corrupt the 2nd file
	dqFn := dq.(*diskQueue).fileName(1)
	os.Truncate(dqFn, 500)

	for i := 0; i < 19; i++ {
		equal(t, <-dq.ReadChan(), msg)
	}

	// corrupt the 4th (current) file
	dqFn = dq.(*diskQueue).fileName(3)
	os.Truncate(dqFn, 100)

	dq.Put(msg)

	equal(t, <-dq.ReadChan(), msg)
}

func TestDiskQueueCorruption(t *testing.T) {
	l := newTestLogger(t)
	dqName := "test_disk_queue_corruption" + strconv.Itoa(int(time.Now().Unix()))
	dq := newDiskQueue(dqName, os.TempDir(), 1000, 5, 2*time.Second, l)

	msg := make([]byte, 123)
	for i := 0; i < 25; i++ {
		dq.Put(msg)
	}

	equal(t, dq.Depth(), int64(25))

	// corrupt the 2nd file
	dqFn := dq.(*diskQueue).fileName(1)
	os.Truncate(dqFn, 500)

	for i := 0; i < 19; i++ {
		equal(t, <-dq.ReadChan(), msg)
	}

	// corrupt the 4th (current) file
	dqFn = dq.(*diskQueue).fileName(3)
	os.Truncate(dqFn, 100)

	dq.Put(msg)

	equal(t, <-dq.ReadChan(), msg)
}

func TestDiskQueueCorruption(t *testing.T) {
	dqName := "test_disk_queue_corruption" + strconv.Itoa(int(time.Now().Unix()))
	tmpDir, err := ioutil.TempDir("", fmt.Sprintf("diskqueue-test-%d", time.Now().UnixNano()))
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(tmpDir)

	dq := newDiskQueue(dqName, tmpDir, 1000, 10, 1<<10, 5, 2*time.Second)
	defer dq.Close()

	//消息长度 123 + 4 字节head = 127
	// 每个文件最多可存放8条消息
	msg := make([]byte, 123) // 127 bytes per message, 8 (1016 bytes) message perfile
	for i := 0; i < 25; i++ {
		dq.Put(msg)
	}

	// 存放25条消息， 占用4个文件 8, 8, 8, 1
	Equal(t, dq.Depth(), int64(25))

	// corrupt the 2nd file
	// 第2个文件长度设置为500, 第二个文件后5个文件丢失,  目前总的可用消息数量为20
	dqFn := dq.(*diskQueue).fileName(1)
	// 跟新文件长度
	os.Truncate(dqFn, 500) // 3 valid messages, 5 corrupted

	// 8 + 3 + 8
	// 取19个消息，第4个文件耽搁消息暂不取
	for i := 0; i < 19; i++ {
		Equal(t, <-dq.ReadChan(), msg) // 1 message leftover in 4th file
	}

	// corrupt the 4th (current) file
	// 设置第四个文件长度为100, 读消息出错，会跳到下一个文件
	// 如果此事设置 time.sleep 超过一定时长， 会先读出数据，单不会报错
	dqFn = dq.(*diskQueue).fileName(3)
	os.Truncate(dqFn, 100)

	// 第四个文件损坏了，直接跳到第5个文件
	dq.Put(msg) // in 5th file

	Equal(t, <-dq.ReadChan(), msg)

	// 写入0长度到文件5
	dq.(*diskQueue).writeFile.Write([]byte{0, 0, 0, 0})

	// 写数据到文件5
	dq.Put(msg)
	// 写到文件6
	dq.Put(msg)

	Equal(t, <-dq.ReadChan(), msg)
}

// Clear the entire file and resize it to initial size.
func (file *DataFile) Clear() (err error) {
	if err = file.Buf.Unmap(); err != nil {
		return
	} else if err = os.Truncate(file.Path, 0); err != nil {
		return
	} else if err = os.Truncate(file.Path, int64(file.Growth)); err != nil {
		return
	} else if file.Buf, err = gommap.Map(file.Fh, gommap.RDWR, 0); err != nil {
		return
	}
	file.Used, file.Size = 0, file.Growth
	tdlog.Infof("%s cleared: %d of %d bytes in-use", file.Path, file.Used, file.Size)
	return
}

// saveVolumeIndex save volumes index info to disk.
func (s *Store) saveVolumeIndex() (err error) {
	var (
		tn, n int
		v     *volume.Volume
	)
	if _, err = s.vf.Seek(0, os.SEEK_SET); err != nil {
		log.Errorf("vf.Seek() error(%v)", err)
		return
	}
	for _, v = range s.Volumes {
		if n, err = s.vf.WriteString(fmt.Sprintf("%s\n", string(v.Meta()))); err != nil {
			log.Errorf("vf.WriteString() error(%v)", err)
			return
		}
		tn += n
	}
	if err = s.vf.Sync(); err != nil {
		log.Errorf("vf.Sync() error(%v)", err)
		return
	}
	if err = os.Truncate(s.conf.Store.VolumeIndex, int64(tn)); err != nil {
		log.Errorf("os.Truncate() error(%v)", err)
	}
	return
}

func (bl *Binlog) Archive(path string) error {
	bl.wrMutex.Lock()
	defer bl.wrMutex.Unlock()

	e := bl.f.Close()
	if e != nil {
		return e
	}

	srcFile, e := os.OpenFile(bl.f.Name(), os.O_RDONLY, 0777)
	if e != nil {
		return e
	}

	newFile, e := os.Create(path)
	if e != nil {
		return e
	}
	_, e = io.Copy(newFile, srcFile)
	if e != nil {
		return e
	}
	e = srcFile.Close()
	if e != nil {
		return e
	}
	e = os.Truncate(srcFile.Name(), 0)

	bl.f, e = os.OpenFile(srcFile.Name(), os.O_WRONLY|os.O_APPEND, 0777)
	if e != nil {
		return e
	}

	return nil
}

func testRepair(t *testing.T, name string, index int64, s int64, cutSize int64, useMmap bool) {
	var r *tableReader
	var err error

	if err := os.Truncate(name, s-cutSize); err != nil {
		t.Fatal(err)
	}

	if r, err = newTableReader(path.Dir(name), index, useMmap); err != nil {
		t.Fatal(err)
	}
	defer r.Close()

	var ll Log
	for i := 0; i < 10; i++ {
		if err := r.GetLog(uint64(i+1), &ll); err != nil {
			t.Fatal(err, i)
		} else if len(ll.Data) != 4096 {
			t.Fatal(len(ll.Data))
		} else if ll.Data[0] != byte(i+1) {
			t.Fatal(ll.Data[0])
		}
	}

	if err := r.GetLog(12, &ll); err == nil {
		t.Fatal("must nil")
	}

	if s != int64(r.data.Size()) {
		t.Fatalf("repair error size %d != %d", s, r.data.Size())
	}

}

func (bm *BackingStore) Create(id string, quota int64) (string, error) {
	log := bm.Logger.Session("create", lager.Data{"id": id, "quota": quota})

	path := bm.backingStorePath(id)
	f, err := os.Create(path)
	if err != nil {
		return "", fmt.Errorf("creating the backing store file: %s", err)
	}
	f.Close()

	if quota == 0 {
		return "", errors.New("cannot have zero sized quota")
	}

	if err := os.Truncate(path, quota); err != nil {
		return "", fmt.Errorf("truncating the file returned error: %s", err)
	}

	output, err := exec.Command("mkfs.ext4",
		"-O", "^has_journal",
		"-F", path,
	).CombinedOutput()
	if err != nil {
		log.Error("formatting-file", err, lager.Data{"path": path, "output": string(output)})
		return "", fmt.Errorf("formatting filesystem: %s", err)
	}

	return path, nil
}

func (db *Csv) DeleteActivity(id int64) (err error) {
	var pos int64
	var line []byte
	pos, line, err = db.findActivityLine(id)
	if err == io.EOF {
		return ErrNotFound
	}

	/* read data past the line */
	var data []byte
	data, err = db.readAll(pos + int64(len(line)))
	if err != nil {
		return
	}

	err = db.writeBytes(pos, data)
	if err != nil {
		return
	}

	db.Mutex.Lock()
	err = os.Truncate(db.Filename, pos+int64(len(data)))
	db.Mutex.Unlock()

	return
}

// copyFileSizeSparse copies file source to destination dest.
func copyFileSizeSparse(source string, dest string, allocatedSize int64) (err error) {
	df, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	err = os.Truncate(dest, allocatedSize)
	if err != nil {
		return err
	}
	err = df.Close()
	if err != nil {
		return err
	}

	if err != nil {
		return err
	}
	createWriterTruncator := func(file *os.File) (writer io.Writer) {
		return NewSparseFilesWriter(NewSeekBufferedWriter(file, ioBufferSize))
	}
	createBufReader := func(file2 *os.File) (writer io.Reader) {
		return bufio.NewReaderSize(file2, ioReadBufferSize)
	}
	return doCopy(source, dest, createBufReader, createWriterTruncator)
}

func fileWriter(t *testing.T, file *os.File, logs []string) {
	filename := file.Name()
	time.Sleep(1 * time.Second) // wait for start Tail...

	for _, line := range logs {
		if strings.Index(line, RotateMarker) != -1 {
			log.Println("fileWriter: rename file => file.old")
			os.Rename(filename, filename+".old")
			file.Close()
			file, _ = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0644)
			log.Println("fileWriter: re-opened file")
		} else if strings.Index(line, TruncateMarker) != -1 {
			time.Sleep(1 * time.Second)
			log.Println("fileWriter: truncate(file, 0)")
			os.Truncate(filename, 0)
			file.Seek(int64(0), os.SEEK_SET)
		}
		_, err := file.WriteString(line)
		log.Print("fileWriter: wrote ", line)
		if err != nil {
			log.Println("write failed", err)
		}
		time.Sleep(1 * time.Millisecond)
	}
	file.Close()
}

// PutContent stores the []byte content at a location designated by "path".
func (d *driver) PutContent(ctx context.Context, subPath string, contents []byte) error {
	if _, err := d.WriteStream(ctx, subPath, 0, bytes.NewReader(contents)); err != nil {
		return err
	}

	return os.Truncate(d.fullPath(subPath), int64(len(contents)))
}