Golang log.Logvf函數代碼示例

// BuildWriteConcern takes a string and a NodeType indicating the type of node the write concern
// is intended to be used against, and converts the write concern string argument into an
// mgo.Safe object that's usable on sessions for that node type.
func BuildWriteConcern(writeConcern string, nodeType NodeType) (*mgo.Safe, error) {
	sessionSafety, err := constructWCObject(writeConcern)
	if err != nil {
		return nil, err
	}

	if sessionSafety == nil {
		log.Logvf(log.DebugLow, "using unacknowledged write concern")
		return nil, nil
	}

	// for standalone mongods, set the default write concern to 1
	if nodeType == Standalone {
		log.Logvf(log.DebugLow, "standalone server: setting write concern %v to 1", w)
		sessionSafety.W = 1
		sessionSafety.WMode = ""
	}

	var writeConcernStr interface{}

	if sessionSafety.WMode != "" {
		writeConcernStr = sessionSafety.WMode
	} else {
		writeConcernStr = sessionSafety.W
	}
	log.Logvf(log.Info, "using write concern: %v='%v', %v=%v, %v=%v, %v=%v",
		w, writeConcernStr,
		j, sessionSafety.J,
		fSync, sessionSafety.FSync,
		wTimeout, sessionSafety.WTimeout,
	)
	return sessionSafety, nil
}

// handleBSONInsteadOfDirectory updates -d and -c settings based on
// the path to the BSON file passed to mongorestore. This is only
// applicable if the target path points to a .bson file.
//
// As an example, when the user passes 'dump/mydb/col.bson', this method
// will infer that 'mydb' is the database and 'col' is the collection name.
func (restore *MongoRestore) handleBSONInsteadOfDirectory(path string) error {
	// we know we have been given a non-directory, so we should handle it
	// like a bson file and infer as much as we can
	if restore.NSOptions.Collection == "" {
		// if the user did not set -c, use the file name for the collection
		newCollectionName, fileType := restore.getInfoFromFilename(path)
		if fileType != BSONFileType {
			return fmt.Errorf("file %v does not have .bson extension", path)
		}
		restore.NSOptions.Collection = newCollectionName
		log.Logvf(log.DebugLow, "inferred collection '%v' from file", restore.NSOptions.Collection)
	}
	if restore.NSOptions.DB == "" {
		// if the user did not set -d, use the directory containing the target
		// file as the db name (as it would be in a dump directory). If
		// we cannot determine the directory name, use "test"
		dirForFile := filepath.Base(filepath.Dir(path))
		if dirForFile == "." || dirForFile == ".." {
			dirForFile = "test"
		}
		restore.NSOptions.DB = dirForFile
		log.Logvf(log.DebugLow, "inferred db '%v' from the file's directory", restore.NSOptions.DB)
	}
	return nil
}

func (dump *MongoDump) createIntentFromOptions(dbName string, ci *collectionInfo) error {
	if dump.shouldSkipCollection(ci.Name) {
		log.Logvf(log.DebugLow, "skipping dump of %v.%v, it is excluded", dbName, ci.Name)
		return nil
	}

	if dump.OutputOptions.ViewsAsCollections && !ci.IsView() {
		log.Logvf(log.DebugLow, "skipping dump of %v.%v because it is not a view", dbName, ci.Name)
		return nil
	}

	intent, err := dump.NewIntent(dbName, ci.Name)
	if err != nil {
		return err
	}
	if dump.OutputOptions.ViewsAsCollections {
		log.Logvf(log.DebugLow, "not dumping metadata for %v.%v because it is a view", dbName, ci.Name)
		intent.MetadataFile = nil
	} else if ci.IsView() {
		log.Logvf(log.DebugLow, "not dumping data for %v.%v because it is a view", dbName, ci.Name)
		// only write a bson file if using archive
		if dump.OutputOptions.Archive == "" {
			intent.BSONFile = nil
		}
	}
	intent.Options = ci.Options
	dump.manager.Put(intent)
	log.Logvf(log.DebugLow, "enqueued collection '%v'", intent.Namespace())
	return nil
}

func handleSignals(finalizer func(), finishedChan chan struct{}) {
	// explicitly ignore SIGPIPE; the tools should deal with write errors
	noopChan := make(chan os.Signal)
	signal.Notify(noopChan, syscall.SIGPIPE)

	log.Logv(log.DebugLow, "will listen for SIGTERM, SIGINT, and SIGKILL")
	sigChan := make(chan os.Signal, 2)
	signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL)
	defer signal.Stop(sigChan)
	if finalizer != nil {
		select {
		case sig := <-sigChan:
			// first signal use finalizer to terminate cleanly
			log.Logvf(log.Always, "signal '%s' received; attempting to shut down", sig)
			finalizer()
		case <-finishedChan:
			return
		}
	}
	select {
	case sig := <-sigChan:
		// second signal exits immediately
		log.Logvf(log.Always, "signal '%s' received; forcefully terminating", sig)
		os.Exit(util.ExitKill)
	case <-finishedChan:
		return
	}
}

// CreateIntentForCollection builds an intent for the given database and collection name
// along with a path to a .bson collection file. It searches the file's parent directory
// for a matching metadata file.
//
// This method is not called by CreateIntentsForDB,
// it is only used in the case where --db and --collection flags are set.
func (restore *MongoRestore) CreateIntentForCollection(db string, collection string, dir archive.DirLike) error {
	log.Logvf(log.DebugLow, "reading collection %v for database %v from %v",
		collection, db, dir.Path())
	// first make sure the bson file exists and is valid
	_, err := dir.Stat()
	if err != nil {
		return err
	}
	if dir.IsDir() {
		return fmt.Errorf("file %v is a directory, not a bson file", dir.Path())
	}

	baseName, fileType := restore.getInfoFromFilename(dir.Name())
	if fileType != BSONFileType {
		return fmt.Errorf("file %v does not have .bson extension", dir.Path())
	}

	// then create its intent
	intent := &intents.Intent{
		DB:       db,
		C:        collection,
		Size:     dir.Size(),
		Location: dir.Path(),
	}
	intent.BSONFile = &realBSONFile{path: dir.Path(), intent: intent, gzip: restore.InputOptions.Gzip}

	// finally, check if it has a .metadata.json file in its folder
	log.Logvf(log.DebugLow, "scanning directory %v for metadata", dir.Name())
	entries, err := dir.Parent().ReadDir()
	if err != nil {
		// try and carry on if we can
		log.Logvf(log.Info, "error attempting to locate metadata for file: %v", err)
		log.Logv(log.Info, "restoring collection without metadata")
		restore.manager.Put(intent)
		return nil
	}
	metadataName := baseName + ".metadata.json"
	if restore.InputOptions.Gzip {
		metadataName += ".gz"
	}
	for _, entry := range entries {
		if entry.Name() == metadataName {
			metadataPath := entry.Path()
			log.Logvf(log.Info, "found metadata for collection at %v", metadataPath)
			intent.MetadataLocation = metadataPath
			intent.MetadataFile = &realMetadataFile{path: metadataPath, intent: intent, gzip: restore.InputOptions.Gzip}
			break
		}
	}

	if intent.MetadataFile == nil {
		log.Logv(log.Info, "restoring collection without metadata")
	}

	restore.manager.Put(intent)

	return nil
}

// Run creates and runs a parser with the Demultiplexer as a consumer
func (demux *Demultiplexer) Run() error {
	parser := Parser{In: demux.In}
	err := parser.ReadAllBlocks(demux)
	if len(demux.outs) > 0 {
		log.Logvf(log.Always, "demux finishing when there are still outs (%v)", len(demux.outs))
	}
	log.Logvf(log.DebugLow, "demux finishing (err:%v)", err)
	return err
}

// Report collects the stat info for a single node and sends found hostnames on
// the "discover" channel if checkShards is true.
func (node *NodeMonitor) Poll(discover chan string, checkShards bool) (*status.ServerStatus, error) {
	stat := &status.ServerStatus{}
	log.Logvf(log.DebugHigh, "getting session on server: %v", node.host)
	s, err := node.sessionProvider.GetSession()
	if err != nil {
		log.Logvf(log.DebugLow, "got error getting session to server %v", node.host)
		return nil, err
	}
	log.Logvf(log.DebugHigh, "got session on server: %v", node.host)

	// The read pref for the session must be set to 'secondary' to enable using
	// the driver with 'direct' connections, which disables the built-in
	// replset discovery mechanism since we do our own node discovery here.
	s.SetMode(mgo.Eventual, true)

	// Disable the socket timeout - otherwise if db.serverStatus() takes a long time on the server
	// side, the client will close the connection early and report an error.
	s.SetSocketTimeout(0)
	defer s.Close()

	err = s.DB("admin").Run(bson.D{{"serverStatus", 1}, {"recordStats", 0}}, stat)
	if err != nil {
		log.Logvf(log.DebugLow, "got error calling serverStatus against server %v", node.host)
		return nil, err
	}
	statMap := make(map[string]interface{})
	s.DB("admin").Run(bson.D{{"serverStatus", 1}, {"recordStats", 0}}, statMap)
	stat.Flattened = status.Flatten(statMap)

	node.Err = nil
	stat.SampleTime = time.Now()

	if stat.Repl != nil && discover != nil {
		for _, host := range stat.Repl.Hosts {
			discover <- host
		}
		for _, host := range stat.Repl.Passives {
			discover <- host
		}
	}
	node.alias = stat.Host
	stat.Host = node.host
	if discover != nil && stat != nil && status.IsMongos(stat) && checkShards {
		log.Logvf(log.DebugLow, "checking config database to discover shards")
		shardCursor := s.DB("config").C("shards").Find(bson.M{}).Iter()
		shard := ConfigShard{}
		for shardCursor.Next(&shard) {
			shardHosts := strings.Split(shard.Host, ",")
			for _, shardHost := range shardHosts {
				discover <- shardHost
			}
		}
		shardCursor.Close()
	}

	return stat, nil
}

// dumpQueryToIntent takes an mgo Query, its intent, and a writer, performs the query,
// and writes the raw bson results to the writer. Returns a final count of documents
// dumped, and any errors that occured.
func (dump *MongoDump) dumpQueryToIntent(
	query *mgo.Query, intent *intents.Intent, buffer resettableOutputBuffer) (dumpCount int64, err error) {

	// restore of views from archives require an empty collection as the trigger to create the view
	// so, we open here before the early return if IsView so that we write an empty collection to the archive
	err = intent.BSONFile.Open()
	if err != nil {
		return 0, err
	}
	defer func() {
		closeErr := intent.BSONFile.Close()
		if err == nil && closeErr != nil {
			err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), closeErr)
		}
	}()
	// don't dump any data for views being dumped as views
	if intent.IsView() && !dump.OutputOptions.ViewsAsCollections {
		return 0, nil
	}
	var total int
	if len(dump.query) == 0 {
		total, err = query.Count()
		if err != nil {
			return int64(0), fmt.Errorf("error reading from db: %v", err)
		}
		log.Logvf(log.DebugLow, "counted %v %v in %v", total, docPlural(int64(total)), intent.Namespace())
	} else {
		log.Logvf(log.DebugLow, "not counting query on %v", intent.Namespace())
	}

	dumpProgressor := progress.NewCounter(int64(total))
	if dump.ProgressManager != nil {
		dump.ProgressManager.Attach(intent.Namespace(), dumpProgressor)
		defer dump.ProgressManager.Detach(intent.Namespace())
	}

	var f io.Writer
	f = intent.BSONFile
	if buffer != nil {
		buffer.Reset(f)
		f = buffer
		defer func() {
			closeErr := buffer.Close()
			if err == nil && closeErr != nil {
				err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), closeErr)
			}
		}()
	}

	err = dump.dumpIterToWriter(query.Iter(), f, dumpProgressor)
	dumpCount, _ = dumpProgressor.Progress()
	if err != nil {
		err = fmt.Errorf("error writing data for collection `%v` to disk: %v", intent.Namespace(), err)
	}
	return
}

// HeaderBSON is part of the ParserConsumer interface and receives headers from parser.
// Its main role is to implement opens and EOFs of the embedded stream.
func (demux *Demultiplexer) HeaderBSON(buf []byte) error {
	colHeader := NamespaceHeader{}
	err := bson.Unmarshal(buf, &colHeader)
	if err != nil {
		return newWrappedError("header bson doesn't unmarshal as a collection header", err)
	}
	log.Logvf(log.DebugHigh, "demux namespaceHeader: %v", colHeader)
	if colHeader.Collection == "" {
		return newError("collection header is missing a Collection")
	}
	demux.currentNamespace = colHeader.Database + "." + colHeader.Collection
	if _, ok := demux.outs[demux.currentNamespace]; !ok {
		if demux.NamespaceChan != nil {
			demux.NamespaceChan <- demux.currentNamespace
			err := <-demux.NamespaceErrorChan
			if err == io.EOF {
				// if the Prioritizer sends us back an io.EOF then it's telling us that
				// it's finishing and doesn't need any more namespace announcements.
				close(demux.NamespaceChan)
				demux.NamespaceChan = nil
				return nil
			}
			if err != nil {
				return newWrappedError("failed arranging a consumer for new namespace", err)
			}
		}
	}
	if colHeader.EOF {
		demux.outs[demux.currentNamespace].Close()
		length := int64(demux.lengths[demux.currentNamespace])
		crcUInt64, ok := demux.outs[demux.currentNamespace].Sum64()
		if ok {
			crc := int64(crcUInt64)
			if crc != colHeader.CRC {
				return fmt.Errorf("CRC mismatch for namespace %v, %v!=%v",
					demux.currentNamespace,
					crc,
					colHeader.CRC,
				)
			}
			log.Logvf(log.DebugHigh,
				"demux checksum for namespace %v is correct (%v), %v bytes",
				demux.currentNamespace, crc, length)
		} else {
			log.Logvf(log.DebugHigh,
				"demux checksum for namespace %v was not calculated.",
				demux.currentNamespace)
		}
		delete(demux.outs, demux.currentNamespace)
		delete(demux.lengths, demux.currentNamespace)
		// in case we get a BSONBody with this block,
		// we want to ensure that that causes an error
		demux.currentNamespace = ""
	}
	return nil
}

// validateReaderFields is a helper to validate fields for input readers
func validateReaderFields(fields []string) error {
	if err := validateFields(fields); err != nil {
		return err
	}
	if len(fields) == 1 {
		log.Logvf(log.Info, "using field: %v", fields[0])
	} else {
		log.Logvf(log.Info, "using fields: %v", strings.Join(fields, ","))
	}
	return nil
}

// handle logic for 'put' command.
func (mf *MongoFiles) handlePut(gfs *mgo.GridFS) (output string, err error) {
	localFileName := mf.getLocalFileName(nil)

	// check if --replace flag turned on
	if mf.StorageOptions.Replace {
		err := gfs.Remove(mf.FileName)
		if err != nil {
			return "", err
		}
		output = fmt.Sprintf("removed all instances of '%v' from GridFS\n", mf.FileName)
	}

	var localFile io.ReadCloser

	if localFileName == "-" {
		localFile = os.Stdin
	} else {
		localFile, err = os.Open(localFileName)
		if err != nil {
			return "", fmt.Errorf("error while opening local file '%v' : %v\n", localFileName, err)
		}
		defer localFile.Close()
		log.Logvf(log.DebugLow, "creating GridFS file '%v' from local file '%v'", mf.FileName, localFileName)
	}

	gFile, err := gfs.Create(mf.FileName)
	if err != nil {
		return "", fmt.Errorf("error while creating '%v' in GridFS: %v\n", mf.FileName, err)
	}
	defer func() {
		// GridFS files flush a buffer on Close(), so it's important we
		// capture any errors that occur as this function exits and
		// overwrite the error if earlier writes executed successfully
		if closeErr := gFile.Close(); err == nil && closeErr != nil {
			log.Logvf(log.DebugHigh, "error occurred while closing GridFS file handler")
			err = fmt.Errorf("error while storing '%v' into GridFS: %v\n", localFileName, closeErr)
		}
	}()

	// set optional mime type
	if mf.StorageOptions.ContentType != "" {
		gFile.SetContentType(mf.StorageOptions.ContentType)
	}

	n, err := io.Copy(gFile, localFile)
	if err != nil {
		return "", fmt.Errorf("error while storing '%v' into GridFS: %v\n", localFileName, err)
	}
	log.Logvf(log.DebugLow, "copied %v bytes to server", n)

	output += fmt.Sprintf("added file: %v\n", gFile.Name())
	return output, nil
}

// Convert implements the Converter interface for JSON input. It converts a
// JSONConverter struct to a BSON document.
func (c JSONConverter) Convert() (bson.D, error) {
	document, err := json.UnmarshalBsonD(c.data)
	if err != nil {
		return nil, fmt.Errorf("error unmarshaling bytes on document #%v: %v", c.index, err)
	}
	log.Logvf(log.DebugHigh, "got line: %v", document)

	bsonD, err := bsonutil.GetExtendedBsonD(document)
	if err != nil {
		return nil, fmt.Errorf("error getting extended BSON for document #%v: %v", c.index, err)
	}
	log.Logvf(log.DebugHigh, "got extended line: %#v", bsonD)
	return bsonD, nil
}

func (dump *MongoDump) createIntentFromOptions(dbName string, ci *collectionInfo) error {
	if dump.shouldSkipCollection(ci.Name) {
		log.Logvf(log.DebugLow, "skipping dump of %v.%v, it is excluded", dbName, ci.Name)
		return nil
	}
	intent, err := dump.NewIntent(dbName, ci.Name)
	if err != nil {
		return err
	}
	intent.Options = ci.Options
	dump.manager.Put(intent)
	log.Logvf(log.DebugLow, "enqueued collection '%v'", intent.Namespace())
	return nil
}

// DumpIntents iterates through the previously-created intents and
// dumps all of the found collections.
func (dump *MongoDump) DumpIntents() error {
	resultChan := make(chan error)

	jobs := dump.OutputOptions.NumParallelCollections
	if numIntents := len(dump.manager.Intents()); jobs > numIntents {
		jobs = numIntents
	}

	if jobs > 1 {
		dump.manager.Finalize(intents.LongestTaskFirst)
	} else {
		dump.manager.Finalize(intents.Legacy)
	}

	log.Logvf(log.Info, "dumping up to %v collections in parallel", jobs)

	// start a goroutine for each job thread
	for i := 0; i < jobs; i++ {
		go func(id int) {
			buffer := dump.getResettableOutputBuffer()
			log.Logvf(log.DebugHigh, "starting dump routine with id=%v", id)
			for {
				intent := dump.manager.Pop()
				if intent == nil {
					log.Logvf(log.DebugHigh, "ending dump routine with id=%v, no more work to do", id)
					resultChan <- nil
					return
				}
				if intent.BSONFile != nil {
					err := dump.DumpIntent(intent, buffer)
					if err != nil {
						resultChan <- err
						return
					}
				}
				dump.manager.Finish(intent)
			}
		}(i)
	}

	// wait until all goroutines are done or one of them errors out
	for i := 0; i < jobs; i++ {
		if err := <-resultChan; err != nil {
			return err
		}
	}

	return nil
}

// checkOplogTimestampExists checks to make sure the oplog hasn't rolled over
// since mongodump started. It does this by checking the oldest oplog entry
// still in the database and making sure it happened at or before the timestamp
// captured at the start of the dump.
func (dump *MongoDump) checkOplogTimestampExists(ts bson.MongoTimestamp) (bool, error) {
	oldestOplogEntry := db.Oplog{}
	err := dump.sessionProvider.FindOne("local", dump.oplogCollection, 0, nil, []string{"+$natural"}, &oldestOplogEntry, 0)
	if err != nil {
		return false, fmt.Errorf("unable to read entry from oplog: %v", err)
	}

	log.Logvf(log.DebugHigh, "oldest oplog entry has timestamp %v", oldestOplogEntry.Timestamp)
	if oldestOplogEntry.Timestamp > ts {
		log.Logvf(log.Info, "oldest oplog entry of timestamp %v is older than %v",
			oldestOplogEntry.Timestamp, ts)
		return false, nil
	}
	return true, nil
}