C++ HostAndPort::toString方法代码示例

        bool tryHeartbeat(BSONObj* info, int* theirConfigVersion) {
            bool ok = false;

            try {
                ok = requestHeartbeat(theReplSet->name(), theReplSet->selfFullName(),
                                      h.toString(), *info, theReplSet->config().version,
                                      *theirConfigVersion);
            }
            catch (DBException&) {
                // don't do anything, ok is already false
            }

            return ok;
        }

    bool SyncSourceFeedback::_connect(OperationContext* txn, const HostAndPort& host) {
        if (hasConnection()) {
            return true;
        }
        log() << "setting syncSourceFeedback to " << host.toString();
        _connection.reset(new DBClientConnection(false, 0, OplogReader::tcp_timeout));
        string errmsg;
        try {
            if (!_connection->connect(host, errmsg) ||
                (getGlobalAuthorizationManager()->isAuthEnabled() && !replAuthenticate())) {
                _resetConnection();
                log() << errmsg << endl;
                return false;
            }
        }
        catch (const DBException& e) {
            error() << "Error connecting to " << host.toString() << ": " << e.what();
            _resetConnection();
            return false;
        }

        return hasConnection();
    }

        void up(const BSONObj& info, HeartbeatInfo& mem) {
            HeartbeatInfo::numPings++;
            mem.authIssue = false;

            if( mem.upSince == 0 ) {
                log() << "replSet member " << h.toString() << " is up" << rsLog;
                mem.upSince = mem.lastHeartbeat;
            }
            mem.health = 1.0;
            mem.lastHeartbeatMsg = info["hbmsg"].String();
            if (info.hasElement("syncingTo")) {
                mem.syncingTo = info["syncingTo"].String();
            }

            if( info.hasElement("opTime") )
                mem.opTime = info["opTime"].Date();

            // see if this member is in the electable set
            if( info["e"].eoo() ) {
                // for backwards compatibility
                const Member *member = theReplSet->findById(mem.id());
                if (member && member->config().potentiallyHot()) {
                    theReplSet->addToElectable(mem.id());
                }
                else {
                    theReplSet->rmFromElectable(mem.id());
                }
            }
            // add this server to the electable set if it is within 10
            // seconds of the latest optime we know of
            else if( info["e"].trueValue() &&
                     mem.opTime >= theReplSet->lastOpTimeWritten.getSecs() - 10) {
                unsigned lastOp = theReplSet->lastOtherOpTime().getSecs();
                if (lastOp > 0 && mem.opTime >= lastOp - 10) {
                    theReplSet->addToElectable(mem.id());
                }
            }
            else {
                theReplSet->rmFromElectable(mem.id());
            }

            be cfg = info["config"];
            if( cfg.ok() ) {
                // received a new config
                boost::function<void()> f =
                    boost::bind(&Manager::msgReceivedNewConfig, theReplSet->mgr, cfg.Obj().copy());
                theReplSet->mgr->send(f);
            }
        }

StatusWith<boost::optional<executor::RemoteCommandRequest>>
ShardingNetworkConnectionHook::makeRequest(const HostAndPort& remoteHost) {
    auto shard = grid.shardRegistry()->getShardNoReload(remoteHost.toString());
    if (!shard) {
        return {ErrorCodes::ShardNotFound,
                str::stream() << "No shard found for host: " << remoteHost.toString()};
    }
    if (shard->isConfig()) {
        // No need to initialize sharding metadata if talking to a config server
        return {boost::none};
    }

    SetShardVersionRequest ssv = SetShardVersionRequest::makeForInitNoPersist(
        grid.shardRegistry()->getConfigServerConnectionString(),
        shard->getId(),
        shard->getConnString());
    executor::RemoteCommandRequest request;
    request.dbname = "admin";
    request.target = remoteHost;
    request.timeout = stdx::chrono::seconds{30};
    request.cmdObj = ssv.toBSON();

    return {request};
}

 bool ReplSetImpl::shouldChangeSyncTarget(const HostAndPort& currentTarget) {
     lock lk(this);
     OpTime targetOpTime = findByName(currentTarget.toString())->hbinfo().opTime;
     for (Member *m = _members.head(); m; m = m->next()) {
         if (m->syncable() &&
             targetOpTime.getSecs()+maxSyncSourceLagSecs < m->hbinfo().opTime.getSecs()) {
             log() << "changing sync target because current sync target's most recent OpTime is "
                   << targetOpTime.toStringPretty() << " which is more than "
                   << maxSyncSourceLagSecs << " seconds behind member " << m->fullName()
                   << " whose most recent OpTime is " << m->hbinfo().opTime.getSecs();
             return true;
         }
     }
     if (gotForceSync()) {
         return true;
     }
     return false;
 }

    Status DBClientShardResolver::findMaster( const std::string connString,
                                              ConnectionString* resolvedHost ) {
        std::string errMsg;

        ConnectionString rawHost = ConnectionString::parse( connString, errMsg );
        dassert( errMsg == "" );
        dassert( rawHost.type() == ConnectionString::SET
                 || rawHost.type() == ConnectionString::MASTER );

        if ( rawHost.type() == ConnectionString::MASTER ) {
            *resolvedHost = rawHost;
            return Status::OK();
        }

        //
        // If we need to, then get the particular node we're targeting in the replica set
        //

        // Does not reload the monitor if it doesn't currently exist
        ReplicaSetMonitorPtr replMonitor = ReplicaSetMonitor::get( rawHost.getSetName(),
                                                                   false );
        if ( !replMonitor ) {
            return Status( ErrorCodes::ReplicaSetNotFound,
                           string("unknown replica set ") + rawHost.getSetName() );
        }

        try {
            // This can throw when we don't find a master!
            HostAndPort masterHostAndPort = replMonitor->getMasterOrUassert();
            *resolvedHost = ConnectionString::parse( masterHostAndPort.toString( true ), errMsg );
            dassert( errMsg == "" );
            return Status::OK();
        }
        catch ( const DBException& ) {
            return Status( ErrorCodes::HostNotFound,
                           string("could not contact primary for replica set ")
                           + replMonitor->getName() );
        }

        // Unreachable
        dassert( false );
        return Status( ErrorCodes::UnknownError, "" );
    }

void ClientInfo::newRequest( AbstractMessagingPort* p ) {

    if ( p ) {
        HostAndPort r = p->remote();
        if ( ! _remote.hasPort() )
            _remote = r;
        else if ( _remote != r ) {
            stringstream ss;
            ss << "remotes don't match old [" << _remote.toString() << "] new [" << r.toString() << "]";
            throw UserException( 13134 , ss.str() );
        }
    }

    _lastAccess = (int) time(0);

    set<string> * temp = _cur;
    _cur = _prev;
    _prev = temp;
    _cur->clear();
}

        bool _requestHeartbeat(HeartbeatInfo& mem, BSONObj& info, int& theirConfigVersion) {
            {
                ScopedConn conn(h.toString());
                conn.setTimeout(_timeout);
                if (tries++ % threshold == (threshold - 1)) {
                    conn.reconnect();
                }
            }

            Timer timer;
            time_t before = curTimeMicros64() / 1000000;

            bool ok = tryHeartbeat(&info, &theirConfigVersion);

            mem.ping = static_cast<unsigned int>(timer.millis());
            time_t totalSecs = mem.ping / 1000;

            // if that didn't work and we have more time, lower timeout and try again
            if (!ok && totalSecs < _timeout) {
                log() << "replset info " << h.toString() << " heartbeat failed, retrying" << rsLog;

                // lower timeout to remaining ping time
                {
                    ScopedConn conn(h.toString());
                    conn.setTimeout(_timeout - totalSecs);
                }

                int checkpoint = timer.millis();
                timer.reset();
                ok = tryHeartbeat(&info, &theirConfigVersion);
                mem.ping = static_cast<unsigned int>(timer.millis());
                totalSecs = (checkpoint + mem.ping)/1000;

                // set timeout back to default
                {
                    ScopedConn conn(h.toString());
                    conn.setTimeout(_timeout);
                }
            }

            // we set this on any response - we don't get this far if
            // couldn't connect because exception is thrown
            time_t after = mem.lastHeartbeat = before + totalSecs;

            if ( info["time"].isNumber() ) {
                long long t = info["time"].numberLong();
                if( t > after )
                    mem.skew = (int) (t - after);
                else if( t < before )
                    mem.skew = (int) (t - before); // negative
            }
            else {
                // it won't be there if remote hasn't initialized yet
                if( info.hasElement("time") )
                    warning() << "heatbeat.time isn't a number: " << info << endl;
                mem.skew = INT_MIN;
            }

            {
                be state = info["state"];
                if( state.ok() )
                    mem.hbstate = MemberState(state.Int());
            }

            if (info.hasField("stateDisagreement") && info["stateDisagreement"].trueValue()) {
                log() << "replset info " << h.toString() << " thinks that we are down" << endl;
            }

            return ok;
        }

        void doWork() {
            if ( !theReplSet ) {
                log(2) << "theReplSet not initialized yet, skipping health poll this round" << rsLog;
                return;
            }

            HeartbeatInfo mem = m;
            HeartbeatInfo old = mem;
            try {
                BSONObj info;
                int theirConfigVersion = -10000;

                Timer timer;

                bool ok = requestHeartbeat(theReplSet->name(), theReplSet->selfFullName(), h.toString(), info, theReplSet->config().version, theirConfigVersion);
                
                mem.ping = (unsigned int)timer.micros();

                time_t before = timer.startTime() / 1000000;
                // we set this on any response - we don't get this far if
                // couldn't connect because exception is thrown
                time_t after = mem.lastHeartbeat = before + (mem.ping / 1000000);
                
                if ( info["time"].isNumber() ) {
                    long long t = info["time"].numberLong();
                    if( t > after )
                        mem.skew = (int) (t - after);
                    else if( t < before )
                        mem.skew = (int) (t - before); // negative
                }
                else {
                    // it won't be there if remote hasn't initialized yet
                    if( info.hasElement("time") )
                        warning() << "heatbeat.time isn't a number: " << info << endl;
                    mem.skew = INT_MIN;
                }

                {
                    be state = info["state"];
                    if( state.ok() )
                        mem.hbstate = MemberState(state.Int());
                }
                if( ok ) {
                    if( mem.upSince == 0 ) {
                        log() << "replSet info " << h.toString() << " is up" << rsLog;
                        mem.upSince = mem.lastHeartbeat;
                    }
                    mem.health = 1.0;
                    mem.lastHeartbeatMsg = info["hbmsg"].String();
                    if( info.hasElement("opTime") )
                        mem.opTime = info["opTime"].Date();

                    be cfg = info["config"];
                    if( cfg.ok() ) {
                        // received a new config
                        boost::function<void()> f =
                            boost::bind(&Manager::msgReceivedNewConfig, theReplSet->mgr, cfg.Obj().copy());
                        theReplSet->mgr->send(f);
                    }
                }
                else {
                    down(mem, info.getStringField("errmsg"));
                }
            }
            catch(DBException& e) {
                down(mem, e.what());
            }
            catch(...) {
                down(mem, "something unusual went wrong");
            }
            m = mem;

            theReplSet->mgr->send( boost::bind(&ReplSet::msgUpdateHBInfo, theReplSet, mem) );

            static time_t last = 0;
            time_t now = time(0);
            bool changed = mem.changed(old);
            if( changed ) {
                if( old.hbstate != mem.hbstate )
                    log() << "replSet member " << h.toString() << ' ' << mem.hbstate.toString() << rsLog;
            }
            if( changed || now-last>4 ) {
                last = now;
                theReplSet->mgr->send( boost::bind(&Manager::msgCheckNewState, theReplSet->mgr) );
            }
        }

void OplogReader::connectToSyncSource(OperationContext* txn,
                                      const OpTime& lastOpTimeFetched,
                                      ReplicationCoordinator* replCoord) {
    const Timestamp sentinelTimestamp(duration_cast<Seconds>(Milliseconds(curTimeMillis64())), 0);
    const OpTime sentinel(sentinelTimestamp, std::numeric_limits<long long>::max());
    OpTime oldestOpTimeSeen = sentinel;

    invariant(conn() == NULL);

    while (true) {
        HostAndPort candidate = replCoord->chooseNewSyncSource(lastOpTimeFetched.getTimestamp());

        if (candidate.empty()) {
            if (oldestOpTimeSeen == sentinel) {
                // If, in this invocation of connectToSyncSource(), we did not successfully
                // connect to any node ahead of us,
                // we apparently have no sync sources to connect to.
                // This situation is common; e.g. if there are no writes to the primary at
                // the moment.
                return;
            }

            // Connected to at least one member, but in all cases we were too stale to use them
            // as a sync source.
            error() << "too stale to catch up";
            log() << "our last optime : " << lastOpTimeFetched;
            log() << "oldest available is " << oldestOpTimeSeen;
            log() << "See http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember";
            setMinValid(txn, oldestOpTimeSeen);
            bool worked = replCoord->setFollowerMode(MemberState::RS_RECOVERING);
            if (!worked) {
                warning() << "Failed to transition into " << MemberState(MemberState::RS_RECOVERING)
                          << ". Current state: " << replCoord->getMemberState();
            }
            return;
        }

        if (!connect(candidate)) {
            LOG(2) << "can't connect to " << candidate.toString() << " to read operations";
            resetConnection();
            replCoord->blacklistSyncSource(candidate, Date_t::now() + Seconds(10));
            continue;
        }
        // Read the first (oldest) op and confirm that it's not newer than our last
        // fetched op. Otherwise, we have fallen off the back of that source's oplog.
        BSONObj remoteOldestOp(findOne(rsOplogName.c_str(), Query()));
        OpTime remoteOldOpTime = fassertStatusOK(28776, OpTime::parseFromBSON(remoteOldestOp));

        // remoteOldOpTime may come from a very old config, so we cannot compare their terms.
        if (!lastOpTimeFetched.isNull() &&
            lastOpTimeFetched.getTimestamp() < remoteOldOpTime.getTimestamp()) {
            // We're too stale to use this sync source.
            resetConnection();
            replCoord->blacklistSyncSource(candidate, Date_t::now() + Minutes(1));
            if (oldestOpTimeSeen.getTimestamp() > remoteOldOpTime.getTimestamp()) {
                warning() << "we are too stale to use " << candidate.toString()
                          << " as a sync source";
                oldestOpTimeSeen = remoteOldOpTime;
            }
            continue;
        }

        // Got a valid sync source.
        return;
    }  // while (true)
}

    ConnectionPool::ConnectionList::iterator ConnectionPool::acquireConnection(
                                                                const HostAndPort& target,
                                                                Date_t now,
                                                                Milliseconds timeout) {
        boost::unique_lock<boost::mutex> lk(_mutex);

        // Clean up connections on stale/unused hosts
        _cleanUpStaleHosts_inlock(now);

        for (HostConnectionMap::iterator hostConns;
             ((hostConns = _connections.find(target)) != _connections.end());) {

            // Clean up the requested host to remove stale/unused connections
            _cleanUpOlderThan_inlock(now, &hostConns->second);
            if (hostConns->second.empty()) {
                // prevent host from causing unnecessary cleanups
                _lastUsedHosts[hostConns->first] = kNeverTooStale;
                break;
            }

            _inUseConnections.splice(_inUseConnections.begin(),
                                     hostConns->second,
                                     hostConns->second.begin());

            const ConnectionList::iterator candidate = _inUseConnections.begin();
            lk.unlock();
            try {
                if (candidate->conn->isStillConnected()) {
                    // setSoTimeout takes a double representing the number of seconds for send and
                    // receive timeouts.  Thus, we must take count() and divide by
                    // 1000.0 to get the number of seconds with a fractional part.
                    candidate->conn->setSoTimeout(timeout.count() / 1000.0);
                    return candidate;
                }
            }
            catch (...) {
                lk.lock();
                _destroyConnection_inlock(&_inUseConnections, candidate);
                throw;
            }

            lk.lock();
            _destroyConnection_inlock(&_inUseConnections, candidate);
        }

        // No idle connection in the pool; make a new one.
        lk.unlock();
        std::auto_ptr<DBClientConnection> conn(new DBClientConnection);

        // setSoTimeout takes a double representing the number of seconds for send and receive
        // timeouts.  Thus, we must take count() and divide by 1000.0 to get the number
        // of seconds with a fractional part.
        conn->setSoTimeout(timeout.count() / 1000.0);
        std::string errmsg;
        uassert(28640,
                str::stream() << "Failed attempt to connect to "
                              << target.toString() << "; " << errmsg,
                conn->connect(target, errmsg));

        conn->port().tag |= _messagingPortTags;

        if (getGlobalAuthorizationManager()->isAuthEnabled()) {
            uassert(ErrorCodes::AuthenticationFailed,
                    "Missing credentials for authenticating as internal user",
                    isInternalAuthSet());
            conn->auth(getInternalUserAuthParamsWithFallback());
        }

        lk.lock();
        return _inUseConnections.insert(_inUseConnections.begin(),
                                        ConnectionInfo(conn.release(), now));
    }

//.........这里部分代码省略.........
                return Status(ErrorCodes::OplogStartMissing, "remote oplog start missing");
            }
            return *(firstDocToApply++);
        };

        *returnStatus = checkRemoteOplogStart(getNextOperation, lastOpTimeFetched, lastFetchedHash);
        if (!returnStatus->isOK()) {
            // Stop fetcher and execute rollback.
            return;
        }

        // If this is the first batch and no rollback is needed, we should have advanced
        // the document iterator.
        invariant(firstDocToApply != documents.cbegin());
    }

    // No work to do if we are draining/primary.
    if (_replCoord->isWaitingForApplierToDrain() || _replCoord->getMemberState().primary()) {
        LOG(2) << "Interrupted by waiting for applier to drain "
               << "or becoming primary while querying the oplog. 1";  // 1st instance.
        return;
    }

    // The count of the bytes of the documents read off the network.
    int networkDocumentBytes = 0;
    Timestamp lastTS;
    {
        stdx::unique_lock<stdx::mutex> lock(_mutex);
        // If we are stopped then return without queueing this batch to apply.
        if (_stopped) {
            LOG(2) << "Interrupted by stop request while querying the oplog. 2";  // 2nd instance.
            return;
        }
        lastTS = _lastOpTimeFetched.getTimestamp();
    }
    int count = 0;
    for (auto&& doc : documents) {
        networkDocumentBytes += doc.objsize();
        ++count;

        // If this is the first response (to the $gte query) then we already applied the first doc.
        if (queryResponse.first && count == 1) {
            continue;
        }

        // Check to see if the oplog entry goes back in time for this document.
        const auto docOpTime = OpTime::parseFromOplogEntry(doc);
        fassertStatusOK(34362, docOpTime.getStatus());  // entries must have a "ts" field.
        const auto docTS = docOpTime.getValue().getTimestamp();

        if (lastTS >= docTS) {
            *returnStatus = Status(
                ErrorCodes::OplogOutOfOrder,
                str::stream() << "Reading the oplog from" << source.toString()
                              << " returned out of order entries. lastTS: " << lastTS.toString()
                              << " outOfOrderTS:" << docTS.toString() << " at count:" << count);
            return;
        }
        lastTS = docTS;
    }

    // These numbers are for the documents we will apply.
    auto toApplyDocumentCount = documents.size();
    auto toApplyDocumentBytes = networkDocumentBytes;
    if (queryResponse.first) {
        // The count is one less since the first document found was already applied ($gte $ts query)
        // and we will not apply it again. We just needed to check it so we didn't rollback, or
        // error above.
        --toApplyDocumentCount;
        const auto alreadyAppliedDocument = documents.cbegin();
        toApplyDocumentBytes -= alreadyAppliedDocument->objsize();
    }

    if (toApplyDocumentBytes > 0) {
        // Wait for enough space.
        _buffer.waitForSpace(toApplyDocumentBytes);

        OCCASIONALLY {
            LOG(2) << "bgsync buffer has " << _buffer.size() << " bytes";
        }

        // Buffer docs for later application.
        std::vector<BSONObj> objs{firstDocToApply, lastDocToApply};
        _buffer.pushAllNonBlocking(objs);

        // Inc stats.
        opsReadStats.increment(documents.size());  // we read all of the docs in the query.
        networkByteStats.increment(networkDocumentBytes);
        bufferCountGauge.increment(toApplyDocumentCount);
        bufferSizeGauge.increment(toApplyDocumentBytes);

        // Update last fetched info.
        auto lastDoc = objs.back();
        {
            stdx::unique_lock<stdx::mutex> lock(_mutex);
            _lastFetchedHash = lastDoc["h"].numberLong();
            _lastOpTimeFetched = fassertStatusOK(28770, OpTime::parseFromOplogEntry(lastDoc));
            LOG(3) << "batch resetting _lastOpTimeFetched: " << _lastOpTimeFetched;
        }
    }

        void up(const BSONObj& info, HeartbeatInfo& mem, bool* needsNewStateChecked) {
            HeartbeatInfo::numPings++;
            mem.authIssue = false;

            if( mem.upSince == 0 ) {
                log() << "replSet member " << h.toString() << " is up" << rsLog;
                mem.upSince = mem.lastHeartbeat;
            }
            mem.health = 1.0;
            mem.lastHeartbeatMsg = info["hbmsg"].String();
            if (info.hasElement("syncingTo")) {
                mem.syncingTo = info["syncingTo"].String();
            }
            if( info.hasElement("opTime") ) {
                mem.opTime = info["opTime"].Date();
            }
            if ( info.hasElement("GTID")) {
                mem.gtid = getGTIDFromBSON("GTID", info);
            }
            if ( info.hasElement("lastUnappliedGTID")) {
                mem.lastUnappliedGTID = getGTIDFromBSON("lastUnappliedGTID", info);
            }
            if ( info.hasElement("minLiveGTID")) {
                mem.minLiveGTID= getGTIDFromBSON("minLiveGTID", info);
            }
            if ( info.hasElement("minUnappliedGTID")) {
                mem.minUnappliedGTID= getGTIDFromBSON("minUnappliedGTID", info);
            }
            if ( info.hasElement("oplogVersion")) {
                mem.oplogVersion = info["oplogVersion"].numberLong();
            }
            else {
                mem.oplogVersion = 0;
            }
            // for "highest known primary"
            if ( info.hasElement("hkp")) {
                mem.highestKnownPrimaryInSet = info["hkp"].numberLong();
                // if the highest known primary across the replica set has changed,
                // communicate that to the caller so that Manager::msgCheckNewState
                // eventually gets called
                *needsNewStateChecked = theReplSet->handleHighestKnownPrimaryOfMember(mem.highestKnownPrimaryInSet);
            }
            else {
                mem.highestKnownPrimaryInSet = 0;
            }
            // see if this member is in the electable set
            if( info["e"].eoo() ) {
                // for backwards compatibility
                const Member *member = theReplSet->findById(mem.id());
                if (member && member->config().potentiallyHot()) {
                    theReplSet->addToElectable(mem.id());
                }
                else {
                    theReplSet->rmFromElectable(mem.id());
                }
            }
            // add this server to the electable set if it is within 10
            // seconds of the latest optime we know of
            else if( info["e"].trueValue() &&
                     mem.opTime + 10000 >= (theReplSet->gtidManager ? theReplSet->gtidManager->getCurrTimestamp() : 0)) 
            {
                unsigned lastOp = theReplSet->lastOtherOpTime();
                if (lastOp > 0 && mem.opTime + 10000 >= lastOp) {
                    theReplSet->addToElectable(mem.id());
                }
            }
            else {
                theReplSet->rmFromElectable(mem.id());
            }

            be cfg = info["config"];
            if( cfg.ok() ) {
                // received a new config
                boost::function<void()> f =
                    boost::bind(&Manager::msgReceivedNewConfig, theReplSet->mgr, cfg.Obj().copy());
                theReplSet->mgr->send(f);
            }
        }

void OplogReader::connectToSyncSource(OperationContext* txn,
                                      const OpTime& lastOpTimeFetched,
                                      const OpTime& requiredOpTime,
                                      ReplicationCoordinator* replCoord) {
    const Timestamp sentinelTimestamp(duration_cast<Seconds>(Date_t::now().toDurationSinceEpoch()),
                                      0);
    const OpTime sentinel(sentinelTimestamp, std::numeric_limits<long long>::max());
    OpTime oldestOpTimeSeen = sentinel;

    invariant(conn() == NULL);

    while (true) {
        HostAndPort candidate = replCoord->chooseNewSyncSource(lastOpTimeFetched);

        if (candidate.empty()) {
            if (oldestOpTimeSeen == sentinel) {
                // If, in this invocation of connectToSyncSource(), we did not successfully
                // connect to any node ahead of us,
                // we apparently have no sync sources to connect to.
                // This situation is common; e.g. if there are no writes to the primary at
                // the moment.
                return;
            }

            // Connected to at least one member, but in all cases we were too stale to use them
            // as a sync source.
            error() << "too stale to catch up -- entering maintenance mode";
            log() << "our last optime : " << lastOpTimeFetched;
            log() << "oldest available is " << oldestOpTimeSeen;
            log() << "See http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember";
            auto status = replCoord->setMaintenanceMode(true);
            if (!status.isOK()) {
                warning() << "Failed to transition into maintenance mode: " << status;
            }
            bool worked = replCoord->setFollowerMode(MemberState::RS_RECOVERING);
            if (!worked) {
                warning() << "Failed to transition into " << MemberState(MemberState::RS_RECOVERING)
                          << ". Current state: " << replCoord->getMemberState();
            }
            return;
        }

        if (!connect(candidate)) {
            LOG(2) << "can't connect to " << candidate.toString() << " to read operations";
            resetConnection();
            replCoord->blacklistSyncSource(candidate, Date_t::now() + Seconds(10));
            continue;
        }
        // Read the first (oldest) op and confirm that it's not newer than our last
        // fetched op. Otherwise, we have fallen off the back of that source's oplog.
        BSONObj remoteOldestOp(findOne(rsOplogName.c_str(), Query()));
        OpTime remoteOldOpTime =
            fassertStatusOK(28776, OpTime::parseFromOplogEntry(remoteOldestOp));

        // remoteOldOpTime may come from a very old config, so we cannot compare their terms.
        if (!lastOpTimeFetched.isNull() &&
            lastOpTimeFetched.getTimestamp() < remoteOldOpTime.getTimestamp()) {
            // We're too stale to use this sync source.
            resetConnection();
            replCoord->blacklistSyncSource(candidate, Date_t::now() + Minutes(1));
            if (oldestOpTimeSeen.getTimestamp() > remoteOldOpTime.getTimestamp()) {
                warning() << "we are too stale to use " << candidate.toString()
                          << " as a sync source";
                oldestOpTimeSeen = remoteOldOpTime;
            }
            continue;
        }

        // Check if sync source contains required optime.
        if (!requiredOpTime.isNull()) {
            // This query is structured so that it is executed on the sync source using the oplog
            // start hack (oplogReplay=true and $gt/$gte predicate over "ts").
            auto ts = requiredOpTime.getTimestamp();
            tailingQuery(rsOplogName.c_str(), BSON("ts" << BSON("$gte" << ts << "$lte" << ts)));
            auto status = _compareRequiredOpTimeWithQueryResponse(requiredOpTime);
            if (!status.isOK()) {
                const auto blacklistDuration = Seconds(60);
                const auto until = Date_t::now() + blacklistDuration;
                warning() << "We cannot use " << candidate.toString()
                          << " as a sync source because it does not contain the necessary "
                             "operations for us to reach a consistent state: "
                          << status << " last fetched optime: " << lastOpTimeFetched
                          << ". required optime: " << requiredOpTime
                          << ". Blacklisting this sync source for " << blacklistDuration
                          << " until: " << until;
                resetConnection();
                replCoord->blacklistSyncSource(candidate, until);
                continue;
            }
            resetCursor();
        }

        // TODO: If we were too stale (recovering with maintenance mode on), then turn it off, to
        //       allow becoming secondary/etc.

        // Got a valid sync source.
        return;
    }  // while (true)
}

        void doWork() { 
            if ( !theReplSet ) {
                log(2) << "theReplSet not initialized yet, skipping health poll this round" << rsLog;
                return;
            }

            HeartbeatInfo mem = m;
            HeartbeatInfo old = mem;
            try { 
                BSONObj info;
                int theirConfigVersion = -10000;

                time_t before = time(0);

                bool ok = requestHeartbeat(theReplSet->name(), theReplSet->selfFullName(), h.toString(), info, theReplSet->config().version, theirConfigVersion);

                time_t after = mem.lastHeartbeat = time(0); // we set this on any response - we don't get this far if couldn't connect because exception is thrown

                try {
                    mem.skew = 0;
                    long long t = info["time"].Long();
                    if( t > after ) 
                        mem.skew = (int) (t - after);
                    else if( t < before ) 
                        mem.skew = (int) (t - before); // negative
                }
                catch(...) { 
                    mem.skew = INT_MIN;
                }

                {
                    be state = info["state"];
                    if( state.ok() )
                        mem.hbstate = MemberState(state.Int());
                }
                if( ok ) {
                    if( mem.upSince == 0 ) {
                        log() << "replSet info " << h.toString() << " is now up" << rsLog;
                        mem.upSince = mem.lastHeartbeat;
                    }
                    mem.health = 1.0;
                    mem.lastHeartbeatMsg = info["hbmsg"].String();
                    if( info.hasElement("opTime") )
                        mem.opTime = info["opTime"].Date();

                    be cfg = info["config"];
                    if( cfg.ok() ) {
                        // received a new config
                        boost::function<void()> f = 
                            boost::bind(&Manager::msgReceivedNewConfig, theReplSet->mgr, cfg.Obj().copy());
                        theReplSet->mgr->send(f);
                    }
                }
                else { 
                    down(mem, info.getStringField("errmsg"));
                }
            }
            catch(...) { 
                down(mem, "connect/transport error");             
            }
            m = mem;

            theReplSet->mgr->send( boost::bind(&ReplSet::msgUpdateHBInfo, theReplSet, mem) );

            static time_t last = 0;
            time_t now = time(0);
            bool changed = mem.changed(old);
            if( changed ) { 
                if( old.hbstate != mem.hbstate ) 
                    log() << "replSet " << h.toString() << ' ' << mem.hbstate.toString() << rsLog;
            }
            if( changed || now-last>4 ) {
                last = now;
                theReplSet->mgr->send( boost::bind(&Manager::msgCheckNewState, theReplSet->mgr) );
            }
        }