Golang journal.NewReader函数代码示例

// Load the backup into real model
// backupFilePath - the backup file path
// dst - the struct model that represents datastore entity and the model you want to load the data of this backup
// onPreload - callback that will be called before loading each entity
// onResult - callback that will be called with already loaded entity in the model
func Load(backupFilePath string, dst interface{}, onPreload func(dst interface{}), onResult func(dst interface{})) {
	f, err := os.Open(backupFilePath)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	journals := journal.NewReader(f, nil, false, true)
	for {
		j, err := journals.Next()
		if err != nil {
			// log.Fatal(err)
			break
		}
		b, err := ioutil.ReadAll(j)
		if err != nil {
			// log.Fatal(err)
			break
		}
		pb := &pb.EntityProto{}
		if err := proto.Unmarshal(b, pb); err != nil {
			log.Fatal(err)
			break
		}
		if onPreload != nil {
			onPreload(dst)
		}
		LoadEntity(dst, pb)
		onResult(dst)
	}
}

func newJournalReader(file storage.File, checksum bool, dropf journal.DropFunc) (p *journalReader, err error) {
	r := new(journalReader)
	r.file = file
	r.reader, err = file.Open()
	if err != nil {
		return
	}
	r.journal = journal.NewReader(r.reader, checksum, dropf)
	return r, nil
}

func newJournalReader(file storage.File, checksum bool, dropf journal.DropFunc) (p *journalReader, err error) {
	r, err := file.Open()
	if err != nil {
		return nil, err
	}
	jr, err := journal.NewReader(r, 0, checksum, dropf)
	if err != nil {
		return nil, err
	}
	return &journalReader{
		file:    file,
		reader:  r,
		journal: jr,
	}, nil
}

func (db *DB) recoverJournal() error {
	// Get all tables and sort it by file number.
	journalFiles_, err := db.s.getFiles(storage.TypeJournal)
	if err != nil {
		return err
	}
	journalFiles := files(journalFiles_)
	journalFiles.sort()

	// Discard older journal.
	prev := -1
	for i, file := range journalFiles {
		if file.Num() >= db.s.stJournalNum {
			if prev >= 0 {
				i--
				journalFiles[i] = journalFiles[prev]
			}
			journalFiles = journalFiles[i:]
			break
		} else if file.Num() == db.s.stPrevJournalNum {
			prev = i
		}
	}

	var jr *journal.Reader
	var of storage.File
	var mem *memdb.DB
	batch := new(Batch)
	cm := newCMem(db.s)
	buf := new(util.Buffer)
	// Options.
	strict := db.s.o.GetStrict(opt.StrictJournal)
	checksum := db.s.o.GetStrict(opt.StrictJournalChecksum)
	writeBuffer := db.s.o.GetWriteBuffer()
	recoverJournal := func(file storage.File) error {
		db.logf("[email protected] recovering @%d", file.Num())
		reader, err := file.Open()
		if err != nil {
			return err
		}
		defer reader.Close()

		// Create/reset journal reader instance.
		if jr == nil {
			jr = journal.NewReader(reader, dropper{db.s, file}, strict, checksum)
		} else {
			jr.Reset(reader, dropper{db.s, file}, strict, checksum)
		}

		// Flush memdb and remove obsolete journal file.
		if of != nil {
			if mem.Len() > 0 {
				if err := cm.flush(mem, 0); err != nil {
					return err
				}
			}
			if err := cm.commit(file.Num(), db.seq); err != nil {
				return err
			}
			cm.reset()
			of.Remove()
			of = nil
		}

		// Replay journal to memdb.
		mem.Reset()
		for {
			r, err := jr.Next()
			if err != nil {
				if err == io.EOF {
					break
				}
				return err
			}

			buf.Reset()
			if _, err := buf.ReadFrom(r); err != nil {
				if err == io.ErrUnexpectedEOF {
					continue
				} else {
					return err
				}
			}
			if err := batch.decode(buf.Bytes()); err != nil {
				return err
			}
			if err := batch.memReplay(mem); err != nil {
				return err
			}

			// Save sequence number.
			db.seq = batch.seq + uint64(batch.len())

			// Flush it if large enough.
			if mem.Size() >= writeBuffer {
				if err := cm.flush(mem, 0); err != nil {
					return err
				}
				mem.Reset()
			}
//.........这里部分代码省略.........

// Recover a database session; need external synchronization.
func (s *session) recover() (err error) {
	defer func() {
		if os.IsNotExist(err) {
			// Don't return os.ErrNotExist if the underlying storage contains
			// other files that belong to LevelDB. So the DB won't get trashed.
			if files, _ := s.stor.GetFiles(storage.TypeAll); len(files) > 0 {
				err = ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest file missing")}
			}
		}
	}()

	file, err := s.stor.GetManifest()
	if err != nil {
		return
	}

	reader, err := file.Open()
	if err != nil {
		return
	}
	defer reader.Close()
	strict := s.o.GetStrict(opt.StrictManifest)
	jr := journal.NewReader(reader, dropper{s, file}, strict, true)

	staging := s.version_NB().newStaging()
	rec := &sessionRecord{}
	for {
		var r io.Reader
		r, err = jr.Next()
		if err != nil {
			if err == io.EOF {
				err = nil
				break
			}
			return
		}

		err = rec.decode(r)
		if err == nil {
			// save compact pointers
			for _, r := range rec.compactionPointers {
				s.stCptrs[r.level] = iKey(r.ikey)
			}
			// commit record to version staging
			staging.commit(rec)
		} else if strict {
			return ErrCorrupted{Type: CorruptedManifest, Err: err}
		} else {
			s.logf("manifest error: %v (skipped)", err)
		}
		rec.resetCompactionPointers()
		rec.resetAddedTables()
		rec.resetDeletedTables()
	}

	switch {
	case !rec.has(recComparer):
		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing comparer name")}
	case rec.comparer != s.icmp.uName():
		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: comparer mismatch, " + "want '" + s.icmp.uName() + "', " + "got '" + rec.comparer + "'")}
	case !rec.has(recNextNum):
		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing next file number")}
	case !rec.has(recJournalNum):
		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing journal file number")}
	case !rec.has(recSeq):
		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing seq number")}
	}

	s.manifestFile = file
	s.setVersion(staging.finish())
	s.setFileNum(rec.nextNum)
	s.recordCommited(rec)
	return nil
}

func (db *DB) recoverJournalRO() error {
	// Get all journals and sort it by file number.
	rawFds, err := db.s.stor.List(storage.TypeJournal)
	if err != nil {
		return err
	}
	sortFds(rawFds)

	// Journals that will be recovered.
	var fds []storage.FileDesc
	for _, fd := range rawFds {
		if fd.Num >= db.s.stJournalNum || fd.Num == db.s.stPrevJournalNum {
			fds = append(fds, fd)
		}
	}

	var (
		// Options.
		strict      = db.s.o.GetStrict(opt.StrictJournal)
		checksum    = db.s.o.GetStrict(opt.StrictJournalChecksum)
		writeBuffer = db.s.o.GetWriteBuffer()

		mdb = memdb.New(db.s.icmp, writeBuffer)
	)

	// Recover journals.
	if len(fds) > 0 {
		db.logf("[email protected] RO·Mode F·%d", len(fds))

		var (
			jr    *journal.Reader
			buf   = &util.Buffer{}
			batch = &Batch{}
		)

		for _, fd := range fds {
			db.logf("[email protected] recovering @%d", fd.Num)

			fr, err := db.s.stor.Open(fd)
			if err != nil {
				return err
			}

			// Create or reset journal reader instance.
			if jr == nil {
				jr = journal.NewReader(fr, dropper{db.s, fd}, strict, checksum)
			} else {
				jr.Reset(fr, dropper{db.s, fd}, strict, checksum)
			}

			// Replay journal to memdb.
			for {
				r, err := jr.Next()
				if err != nil {
					if err == io.EOF {
						break
					}

					fr.Close()
					return errors.SetFd(err, fd)
				}

				buf.Reset()
				if _, err := buf.ReadFrom(r); err != nil {
					if err == io.ErrUnexpectedEOF {
						// This is error returned due to corruption, with strict == false.
						continue
					}

					fr.Close()
					return errors.SetFd(err, fd)
				}
				if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mdb); err != nil {
					if !strict && errors.IsCorrupted(err) {
						db.s.logf("journal error: %v (skipped)", err)
						// We won't apply sequence number as it might be corrupted.
						continue
					}

					fr.Close()
					return errors.SetFd(err, fd)
				}

				// Save sequence number.
				db.seq = batch.seq + uint64(batch.Len())
			}

			fr.Close()
		}
	}

	// Set memDB.
	db.mem = &memDB{db: db, DB: mdb, ref: 1}

	return nil
}

func (db *DB) recoverJournal() error {
	// Get all journals and sort it by file number.
	rawFds, err := db.s.stor.List(storage.TypeJournal)
	if err != nil {
		return err
	}
	sortFds(rawFds)

	// Journals that will be recovered.
	var fds []storage.FileDesc
	for _, fd := range rawFds {
		if fd.Num >= db.s.stJournalNum || fd.Num == db.s.stPrevJournalNum {
			fds = append(fds, fd)
		}
	}

	var (
		ofd storage.FileDesc // Obsolete file.
		rec = &sessionRecord{}
	)

	// Recover journals.
	if len(fds) > 0 {
		db.logf("[email protected] F·%d", len(fds))

		// Mark file number as used.
		db.s.markFileNum(fds[len(fds)-1].Num)

		var (
			// Options.
			strict      = db.s.o.GetStrict(opt.StrictJournal)
			checksum    = db.s.o.GetStrict(opt.StrictJournalChecksum)
			writeBuffer = db.s.o.GetWriteBuffer()

			jr    *journal.Reader
			mdb   = memdb.New(db.s.icmp, writeBuffer)
			buf   = &util.Buffer{}
			batch = &Batch{}
		)

		for _, fd := range fds {
			db.logf("[email protected] recovering @%d", fd.Num)

			fr, err := db.s.stor.Open(fd)
			if err != nil {
				return err
			}

			// Create or reset journal reader instance.
			if jr == nil {
				jr = journal.NewReader(fr, dropper{db.s, fd}, strict, checksum)
			} else {
				jr.Reset(fr, dropper{db.s, fd}, strict, checksum)
			}

			// Flush memdb and remove obsolete journal file.
			if !ofd.Nil() {
				if mdb.Len() > 0 {
					if _, err := db.s.flushMemdb(rec, mdb, 0); err != nil {
						fr.Close()
						return err
					}
				}

				rec.setJournalNum(fd.Num)
				rec.setSeqNum(db.seq)
				if err := db.s.commit(rec); err != nil {
					fr.Close()
					return err
				}
				rec.resetAddedTables()

				db.s.stor.Remove(ofd)
				ofd = storage.FileDesc{}
			}

			// Replay journal to memdb.
			mdb.Reset()
			for {
				r, err := jr.Next()
				if err != nil {
					if err == io.EOF {
						break
					}

					fr.Close()
					return errors.SetFd(err, fd)
				}

				buf.Reset()
				if _, err := buf.ReadFrom(r); err != nil {
					if err == io.ErrUnexpectedEOF {
						// This is error returned due to corruption, with strict == false.
						continue
					}

					fr.Close()
					return errors.SetFd(err, fd)
				}
				if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mdb); err != nil {
//.........这里部分代码省略.........

// Recover a database session; need external synchronization.
func (s *session) recover() (err error) {
	defer func() {
		if os.IsNotExist(err) {
			// Don't return os.ErrNotExist if the underlying storage contains
			// other files that belong to LevelDB. So the DB won't get trashed.
			if fds, _ := s.stor.List(storage.TypeAll); len(fds) > 0 {
				err = &errors.ErrCorrupted{Fd: storage.FileDesc{Type: storage.TypeManifest}, Err: &errors.ErrMissingFiles{}}
			}
		}
	}()

	fd, err := s.stor.GetMeta()
	if err != nil {
		return
	}

	reader, err := s.stor.Open(fd)
	if err != nil {
		return
	}
	defer reader.Close()

	var (
		// Options.
		strict = s.o.GetStrict(opt.StrictManifest)

		jr      = journal.NewReader(reader, dropper{s, fd}, strict, true)
		rec     = &sessionRecord{}
		staging = s.stVersion.newStaging()
	)
	for {
		var r io.Reader
		r, err = jr.Next()
		if err != nil {
			if err == io.EOF {
				err = nil
				break
			}
			return errors.SetFd(err, fd)
		}

		err = rec.decode(r)
		if err == nil {
			// save compact pointers
			for _, r := range rec.compPtrs {
				s.setCompPtr(r.level, internalKey(r.ikey))
			}
			// commit record to version staging
			staging.commit(rec)
		} else {
			err = errors.SetFd(err, fd)
			if strict || !errors.IsCorrupted(err) {
				return
			}
			s.logf("manifest error: %v (skipped)", errors.SetFd(err, fd))
		}
		rec.resetCompPtrs()
		rec.resetAddedTables()
		rec.resetDeletedTables()
	}

	switch {
	case !rec.has(recComparer):
		return newErrManifestCorrupted(fd, "comparer", "missing")
	case rec.comparer != s.icmp.uName():
		return newErrManifestCorrupted(fd, "comparer", fmt.Sprintf("mismatch: want '%s', got '%s'", s.icmp.uName(), rec.comparer))
	case !rec.has(recNextFileNum):
		return newErrManifestCorrupted(fd, "next-file-num", "missing")
	case !rec.has(recJournalNum):
		return newErrManifestCorrupted(fd, "journal-file-num", "missing")
	case !rec.has(recSeqNum):
		return newErrManifestCorrupted(fd, "seq-num", "missing")
	}

	s.manifestFd = fd
	s.setVersion(staging.finish())
	s.setNextFileNum(rec.nextFileNum)
	s.recordCommited(rec)
	return nil
}

func (d *DB) recoverJournal() error {
	s := d.s

	ff0, err := s.getFiles(storage.TypeJournal)
	if err != nil {
		return err
	}
	ff1 := files(ff0)
	ff1.sort()
	ff2 := make([]storage.File, 0, len(ff1))
	for _, file := range ff1 {
		if file.Num() >= s.stJournalNum || file.Num() == s.stPrevJournalNum {
			s.markFileNum(file.Num())
			ff2 = append(ff2, file)
		}
	}

	var jr *journal.Reader
	var of storage.File
	var mem *memdb.DB
	batch := new(Batch)
	cm := newCMem(s)
	buf := new(util.Buffer)
	// Options.
	strict := s.o.GetStrict(opt.StrictJournal)
	checksum := s.o.GetStrict(opt.StrictJournalChecksum)
	writeBuffer := s.o.GetWriteBuffer()
	recoverJournal := func(file storage.File) error {
		s.logf("[email protected] recovering @%d", file.Num())
		reader, err := file.Open()
		if err != nil {
			return err
		}
		defer reader.Close()
		if jr == nil {
			jr = journal.NewReader(reader, dropper{s, file}, strict, checksum)
		} else {
			jr.Reset(reader, dropper{s, file}, strict, checksum)
		}
		if of != nil {
			if mem.Len() > 0 {
				if err := cm.flush(mem, 0); err != nil {
					return err
				}
			}
			if err := cm.commit(file.Num(), d.seq); err != nil {
				return err
			}
			cm.reset()
			of.Remove()
			of = nil
		}
		// Reset memdb.
		mem.Reset()
		for {
			r, err := jr.Next()
			if err != nil {
				if err == io.EOF {
					break
				}
				return err
			}
			buf.Reset()
			if _, err := buf.ReadFrom(r); err != nil {
				if strict {
					return err
				}
				continue
			}
			if err := batch.decode(buf.Bytes()); err != nil {
				return err
			}
			if err := batch.memReplay(mem); err != nil {
				return err
			}
			d.seq = batch.seq + uint64(batch.len())
			if mem.Size() >= writeBuffer {
				// Large enough, flush it.
				if err := cm.flush(mem, 0); err != nil {
					return err
				}
				// Reset memdb.
				mem.Reset()
			}
		}
		of = file
		return nil
	}
	// Recover all journals.
	if len(ff2) > 0 {
		s.logf("[email protected] F·%d", len(ff2))
		mem = memdb.New(s.icmp, writeBuffer)
		for _, file := range ff2 {
			if err := recoverJournal(file); err != nil {
				return err
			}
		}
		// Flush the last journal.
		if mem.Len() > 0 {
			if err := cm.flush(mem, 0); err != nil {
//.........这里部分代码省略.........

// Recover a database session; need external synchronization.
func (s *session) recover() (err error) {
	file, err := s.stor.GetManifest()
	if err != nil {
		return
	}

	reader, err := file.Open()
	if err != nil {
		return
	}
	defer reader.Close()
	strict := s.o.GetStrict(opt.StrictManifest)
	jr := journal.NewReader(reader, dropper{s, file}, strict, true)

	staging := s.version_NB().newStaging()
	rec := &sessionRecord{}
	for {
		var r io.Reader
		r, err = jr.Next()
		if err != nil {
			if err == io.EOF {
				err = nil
				break
			}
			return
		}

		err = rec.decode(r)
		if err == nil {
			// save compact pointers
			for _, rp := range rec.compactionPointers {
				s.stCPtrs[rp.level] = iKey(rp.key)
			}
			// commit record to version staging
			staging.commit(rec)
		} else if strict {
			return err
		} else {
			s.logf("manifest error: %v (skipped)", err)
		}
		rec.resetCompactionPointers()
		rec.resetAddedTables()
		rec.resetDeletedTables()
	}

	switch {
	case !rec.has(recComparer):
		return errors.New("leveldb: manifest missing comparer name")
	case rec.comparer != s.cmp.cmp.Name():
		return errors.New("leveldb: comparer mismatch, " + "want '" + s.cmp.cmp.Name() + "', " + "got '" + rec.comparer + "'")
	case !rec.has(recNextNum):
		return errors.New("leveldb: manifest missing next file number")
	case !rec.has(recJournalNum):
		return errors.New("leveldb: manifest missing journal file number")
	case !rec.has(recSeq):
		return errors.New("leveldb: manifest missing seq number")
	}

	s.manifestFile = file
	s.setVersion(staging.finish())
	s.setFileNum(rec.nextNum)
	s.recordCommited(rec)
	return nil
}