Golang lldb.NewSimpleFileFiler函數代碼示例

func (s *file) CreateTemp(asc bool) (bt temp, err error) {
	f, err := ioutil.TempFile("", "ql-tmp-")
	if err != nil {
		return nil, err
	}

	fn := f.Name()
	filer := lldb.NewSimpleFileFiler(f)
	a, err := lldb.NewAllocator(filer, &lldb.Options{})
	if err != nil {
		f.Close()
		os.Remove(fn)
		return nil, err
	}

	t, _, err := lldb.CreateBTree(a, lldbCollators[asc])
	if err != nil {
		f.Close()
		os.Remove(fn)
		return nil, err
	}

	return &fileTemp{
		a: a,
		f: f,
		t: t,
	}, nil
}

// Open opens the named DB file for reading/writing. If successful, methods on
// the returned DB can be used for I/O; the associated file descriptor has mode
// os.O_RDWR. If there is an error, it will be of type *os.PathError.
//
// For the meaning of opts please see documentation of Options.
func Open(name string, opts *Options) (db *DB, err error) {
	defer func() {
		lock := opts.lock
		if err != nil && lock != nil {
			n := lock.Name()
			lock.Close()
			os.Remove(n)
			db = nil
		}
		if err != nil {
			if db != nil {
				db.Close()
				db = nil
			}
		}
	}()

	if err = opts.check(name, false, true); err != nil {
		return
	}

	f, err := os.OpenFile(name, os.O_RDWR, 0666)
	if err != nil {
		return
	}

	filer := lldb.Filer(lldb.NewSimpleFileFiler(f))
	sz, err := filer.Size()
	if err != nil {
		return
	}

	if sz%16 != 0 {
		return nil, &os.PathError{Op: "dbm.Open:", Path: name, Err: fmt.Errorf("file size %d(%#x) is not 0 (mod 16)", sz, sz)}
	}

	var b [16]byte
	if n, err := filer.ReadAt(b[:], 0); n != 16 || err != nil {
		return nil, &os.PathError{Op: "dbm.Open.ReadAt", Path: name, Err: err}
	}

	var h header
	if err = h.rd(b[:]); err != nil {
		return nil, &os.PathError{Op: "dbm.Open:validate header", Path: name, Err: err}
	}

	db = &DB{f: f, lock: opts.lock, closed: make(chan bool)}
	if filer, err = opts.acidFiler(db, filer); err != nil {
		return nil, err
	}

	db.filer = filer
	switch h.ver {
	default:
		return nil, &os.PathError{Op: "dbm.Open", Path: name, Err: fmt.Errorf("unknown dbm file format version %#x", h.ver)}
	case 0x00:
		return open00(name, db)
	}

}

func fillseq() {
	dbname := os.Args[0] + ".db"
	f, err := os.OpenFile(dbname, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
	if err != nil {
		log.Fatal(err)
	}

	defer func() {
		f.Close()
		os.Remove(f.Name())
	}()

	filer := lldb.NewSimpleFileFiler(f)
	a, err := lldb.NewAllocator(filer, &lldb.Options{})
	if err != nil {
		log.Println(err)
		return
	}

	a.Compress = true
	b, _, err := lldb.CreateBTree(a, nil)
	if err != nil {
		log.Println(err)
		return
	}

	var keys [N][16]byte
	for i := range keys {
		binary.BigEndian.PutUint32(keys[i][:], uint32(i))
	}

	debug.FreeOSMemory()
	t0 := time.Now()
	for _, key := range keys {
		if err = b.Set(key[:], value100); err != nil {
			log.Println(err)
			return
		}
	}
	if err := filer.Sync(); err != nil {
		log.Println(err)
		return
	}
	var ms runtime.MemStats
	runtime.ReadMemStats(&ms)

	d := time.Since(t0)
	fi, err := f.Stat()
	if err != nil {
		log.Println(err)
		return
	}

	secs := float64(d/time.Nanosecond) / float64(time.Second)
	sz := fi.Size()
	fmt.Printf("fillseq      :%19v/op;%7.1f MB/s (%g secs, %d bytes)\n", d/N, float64(sz)/secs/1e6, secs, sz)
	nn, bytes := bufs.GCache.Stats()
	fmt.Printf("%d %d\n", nn, bytes)
	fmt.Printf("%+v\n", ms)
}

// Verify attempts to find any structural errors in DB wrt the organization of
// it as defined by lldb.Allocator. 'bitmap' is a scratch pad for necessary
// bookkeeping and will grow to at most to DB size/128 (0,78%). Any problems
// found are reported to 'log' except non verify related errors like disk read
// fails etc. If 'log' returns false or the error doesn't allow to (reliably)
// continue, the verification process is stopped and an error is returned from
// the Verify function. Passing a nil log works like providing a log function
// always returning false. Any non-structural errors, like for instance Filer
// read errors, are NOT reported to 'log', but returned as the Verify's return
// value, because Verify cannot proceed in such cases. Verify returns nil only
// if it fully completed verifying DB without detecting any error.
//
// It is recommended to limit the number reported problems by returning false
// from 'log' after reaching some limit. Huge and corrupted DB can produce an
// overwhelming error report dataset.
//
// The verifying process will scan the whole DB at least 3 times (a trade
// between processing space and time consumed). It doesn't read the content of
// free blocks above the head/tail info bytes. If the 3rd phase detects lost
// free space, then a 4th scan (a faster one) is performed to precisely report
// all of them.
//
// Statistics are returned via 'stats' if non nil. The statistics are valid
// only if Verify succeeded, ie. it didn't reported anything to log and it
// returned a nil error.
func (db *DB) Verify(log func(error) bool, stats *lldb.AllocStats) (err error) {
	bitmapf, err := fileutil.TempFile(".", "verifier", ".tmp")
	if err != nil {
		return
	}

	defer func() {
		tn := bitmapf.Name()
		bitmapf.Close()
		os.Remove(tn)
	}()

	bitmap := lldb.NewSimpleFileFiler(bitmapf)

	if err = db.enter(); err != nil {
		return
	}

	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("%v", e)
		}
		db.leave(&err)
	}()

	return db.alloc.Verify(bitmap, log, stats)
}

// Create creates the named DB file mode 0666 (before umask). The file must not
// already exist. If successful, methods on the returned DB can be used for
// I/O; the associated file descriptor has mode os.O_RDWR. If there is an
// error, it will be of type *os.PathError.
//
// For the meaning of opts please see documentation of Options.
func Create(name string, opts *Options) (db *DB, err error) {
	f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
	if err != nil {
		return
	}

	return create(f, lldb.NewSimpleFileFiler(f), opts, false)
}

// Open opens the named DB file for reading/writing. If successful, methods on
// the returned DB can be used for I/O; the associated file descriptor has mode
// os.O_RDWR. If there is an error, it will be of type *os.PathError.
//
// Note: While a DB is opened, it is locked and cannot be simultaneously opened
// again.
//
// For the meaning of opts please see documentation of Options.
func Open(name string, opts *Options) (db *DB, err error) {
	f, err := os.OpenFile(name, os.O_RDWR, 0666)
	if err != nil {
		return nil, err
	}

	return OpenFromFiler(lldb.NewSimpleFileFiler(f), opts)
}

// CreateTemp creates a new temporary DB in the directory dir with a basename
// beginning with prefix and name ending in suffix. If dir is the empty string,
// CreateTemp uses the default directory for temporary files (see os.TempDir).
// Multiple programs calling CreateTemp simultaneously will not choose the same
// file name for the DB. The caller can use Name() to find the pathname of the
// DB file. It is the caller's responsibility to remove the file when no longer
// needed.
//
// For the meaning of opts please see documentation of Options.
func CreateTemp(dir, prefix, suffix string, opts *Options) (db *DB, err error) {
	f, err := fileutil.TempFile(dir, prefix, suffix)
	if err != nil {
		return
	}

	return create(f, lldb.NewSimpleFileFiler(f), opts, false)
}

// Create creates the named DB file mode 0666 (before umask). The file must not
// already exist. If successful, methods on the returned DB can be used for
// I/O; the associated file descriptor has mode os.O_RDWR. If there is an
// error, it will be of type *os.PathError.
//
// For the meaning of opts please see documentation of Options.
func Create(name string, opts *Options) (db *DB, err error) {
	opts = opts.clone()
	opts._ACID = _ACIDFull
	f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
	if err != nil {
		return
	}

	return CreateFromFiler(lldb.NewSimpleFileFiler(f), opts)
}

// CreateTemp creates a new temporary DB in the directory dir with a basename
// beginning with prefix and name ending in suffix. If dir is the empty string,
// CreateTemp uses the default directory for temporary files (see os.TempDir).
// Multiple programs calling CreateTemp simultaneously will not choose the same
// file name for the DB. The caller can use Name() to find the pathname of the
// DB file. It is the caller's responsibility to remove the file when no longer
// needed.
//
// For the meaning of opts please see documentation of Options.
func CreateTemp(dir, prefix, suffix string, opts *Options) (db *DB, err error) {
	opts = opts.clone()
	opts._ACID = _ACIDFull
	f, err := fileutil.TempFile(dir, prefix, suffix)
	if err != nil {
		return
	}

	return create(lldb.NewSimpleFileFiler(f), opts, false)
}

func verifyAllocator(a *lldb.Allocator) error {
	bits, err := ioutil.TempFile("", "kv-verify-")
	if err != nil {
		return err
	}

	defer func() {
		nm := bits.Name()
		bits.Close()
		os.Remove(nm)
	}()

	var lerr error
	if err = a.Verify(
		lldb.NewSimpleFileFiler(bits),
		func(err error) bool {
			lerr = err
			return false
		},
		nil,
	); err != nil {
		return err
	}

	if lerr != nil {
		return lerr
	}

	t, err := lldb.OpenBTree(a, nil, 1)
	if err != nil {
		return err
	}

	e, err := t.SeekFirst()
	if err != nil {
		if err == io.EOF {
			err = nil
		}
		return err
	}

	for {
		_, _, err := e.Next()
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return err
		}
	}
}

func TestProf(t *testing.T) {
	dbname := os.Args[0] + ".db"
	f, err := os.OpenFile(dbname, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
	if err != nil {
		t.Fatal(err)
	}

	defer func() {
		f.Close()
		os.Remove(f.Name())
	}()

	filer := lldb.NewSimpleFileFiler(f) // file
	//filer := lldb.NewMemFiler()         // mem
	a, err := lldb.NewAllocator(filer, &lldb.Options{})
	if err != nil {
		t.Error(err)
		return
	}

	a.Compress = true
	b, _, err := lldb.CreateBTree(a, nil)
	if err != nil {
		t.Error(err)
		return
	}

	var key [16]byte
	for i := uint32(0); int(i) < 1e6; i++ {
		binary.BigEndian.PutUint32(key[:], i)
		if err = b.Set(key[:], value100); err != nil {
			t.Error(err)
			return
		}
	}
	var ms runtime.MemStats
	runtime.ReadMemStats(&ms)
	bufsU, bufsT, bytesU, bytesT, h, m := a.CacheStats()
	const p = 100.0
	t.Logf(
		"cache: buffers %d/%d(%.1f%%), bytes %d/%d(%.1f%%), hits %d(%.1f%%), misses %d(%.1f%%)",
		bufsU, bufsT, p*float64(bufsU)/float64(bufsT),
		bytesU, bytesT, p*float64(bytesU)/float64(bytesT),
		h, p*float64(h)/float64(h+m),
		m, p*float64(m)/float64(h+m),
	)
	nn, bts := bufs.GCache.Stats()
	t.Logf("bufs.GCache.Stats() {%d, %d}", nn, bts)
	t.Logf("%+v\n", ms)
}

func verifyDbFile(fn string) error {
	f, err := os.Open(fn) // O_RDONLY
	if err != nil {
		return err
	}

	defer f.Close()

	a, err := lldb.NewAllocator(lldb.NewInnerFiler(lldb.NewSimpleFileFiler(f), 16), &lldb.Options{})
	if err != nil {
		return err
	}

	return verifyAllocator(a)
}

func BenchmarkMem(b *testing.B) {
	f, err := ioutil.TempFile("", "")
	if err != nil {
		b.Fatal(err)
	}

	defer func() {
		f.Close()
		os.Remove(f.Name())
	}()

	filer := lldb.NewSimpleFileFiler(f)
	a, err := lldb.NewAllocator(filer, &lldb.Options{})
	if err != nil {
		b.Error(err)
		return
	}

	a.Compress = true

	t, _, err := lldb.CreateBTree(a, nil)
	if err != nil {
		b.Error(err)
		return
	}

	b.ResetTimer()
	var key [16]byte
	for i := uint32(0); int(i) < b.N; i++ {
		binary.BigEndian.PutUint32(key[:], i)
		if err = t.Set(key[:], value100); err != nil {
			b.Error(err)
			return
		}
	}

	if err := filer.Sync(); err != nil {
		b.Error(err)
		return
	}
}

func main0(fn string, oMax int, w func(s string, a ...interface{}), oStat bool, first, last string, dump bool, oBuckets bool) error {
	f, err := os.Open(fn) // O_RDONLY
	if err != nil {
		return err
	}

	defer f.Close()

	bits, err := ioutil.TempFile("", "kvaudit-")
	if err != nil {
		return err
	}

	defer func() {
		nm := bits.Name()
		bits.Close()
		os.Remove(nm)
	}()

	a, err := lldb.NewAllocator(lldb.NewInnerFiler(lldb.NewSimpleFileFiler(f), 16), &lldb.Options{})
	if err != nil {
		return err
	}

	cnt := 0
	var stats lldb.AllocStats
	err = a.Verify(lldb.NewSimpleFileFiler(bits), func(err error) bool {
		cnt++
		w("%d: %v\n", cnt, err)
		return cnt < oMax
	}, &stats)
	if err != nil {
		return err
	}

	if oStat {
		w("Handles     %10d  // total valid handles in use\n", stats.Handles)
		w("Compression %10d  // number of compressed blocks\n", stats.Compression)
		w("TotalAtoms  %10d  // total number of atoms == AllocAtoms + FreeAtoms\n", stats.TotalAtoms)
		w("AllocBytes  %10d  // bytes allocated (after decompression, if/where used)\n", stats.AllocBytes)
		w("AllocAtoms  %10d  // atoms allocated/used, including relocation atoms\n", stats.AllocAtoms)
		w("Relocations %10d  // number of relocated used blocks\n", stats.Relocations)
		w("FreeAtoms   %10d  // atoms unused\n", stats.FreeAtoms)
	}
	if oBuckets {
		var alloc, free [14]int64
		sizes := [14]int64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 4112}
		for atoms, cnt := range stats.AllocMap {
			if atoms >= 4096 {
				alloc[13] += cnt
				continue
			}

			for i := range sizes {
				if sizes[i+1] > atoms {
					alloc[i] += cnt
					break
				}
			}
		}
		for atoms, cnt := range stats.FreeMap {
			if atoms > 4096 {
				free[13] += cnt
				continue
			}

			for i := range sizes {
				if sizes[i+1] > atoms {
					free[i] += cnt
					break
				}
			}
		}
		w("Alloc blocks\n")
		for i, v := range alloc {
			w("%4d: %10d\n", sizes[i], v)
		}
		w("Free blocks\n")
		for i, v := range free {
			w("%4d: %10d\n", sizes[i], v)
		}
	}

	if !(first != "" || last != "" || dump) {
		return nil
	}

	t, err := lldb.OpenBTree(a, nil, 1)
	if err != nil {
		return err
	}

	dw := bufio.NewWriter(os.Stdout)
	defer dw.Flush()

	var e *lldb.BTreeEnumerator
	switch {
	case first != "":
		e, _, err = t.Seek([]byte(first))
	default:
//.........這裏部分代碼省略.........

func newFileFromFile(f *os.File, simple bool) (fi *file, err error) {
	nm := lockName(f.Name())
	lck, err := lock.Lock(nm)
	if err != nil {
		if lck != nil {
			lck.Close()
		}
		return nil, err
	}

	close := true
	defer func() {
		if close && lck != nil {
			lck.Close()
		}
	}()

	var w *os.File
	closew := false
	if !simple {
		wn := walName(f.Name())
		w, err = os.OpenFile(wn, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
		closew = true
		defer func() {
			if closew {
				nm := w.Name()
				w.Close()
				os.Remove(nm)
				w = nil
			}
		}()

		if err != nil {
			if !os.IsExist(err) {
				return nil, err
			}

			closew = false
			w, err = os.OpenFile(wn, os.O_RDWR, 0666)
			if err != nil {
				return nil, err
			}

			closew = true
			st, err := w.Stat()
			if err != nil {
				return nil, err
			}

			if st.Size() != 0 {
				return nil, fmt.Errorf("non empty WAL file %s exists", wn)
			}
		}
	}

	info, err := f.Stat()
	if err != nil {
		return nil, err
	}

	switch sz := info.Size(); {
	case sz == 0:
		b := make([]byte, 16)
		copy(b, []byte(magic))
		if _, err := f.Write(b); err != nil {
			return nil, err
		}

		filer := lldb.Filer(lldb.NewSimpleFileFiler(f))
		filer = lldb.NewInnerFiler(filer, 16)
		switch simple {
		case true:
			f0 := filer
			if filer, err = lldb.NewRollbackFiler(
				filer,
				func(sz int64) error {
					return f0.Truncate(sz)
				},
				f0,
			); err != nil {
				return nil, err
			}
		default:
			if filer, err = lldb.NewACIDFiler(filer, w); err != nil {
				return nil, err
			}
		}

		a, err := lldb.NewAllocator(filer, &lldb.Options{})
		if err != nil {
			return nil, err
		}

		a.Compress = true
		s := &file{
			a:    a,
			f0:   f,
			f:    filer,
			lck:  lck,
			name: f.Name(),
//.........這裏部分代碼省略.........