MongoDB 节点宕机引发的思考

2019-05-06T16:51:11.766+0800 I REPL     [ReplicationExecutor] Starting an election, since we've seen no PRIMARY in the past 10000ms
2019-05-06T16:51:11.766+0800 I REPL     [ReplicationExecutor] conducting a dry run election to see if we could be elected
2019-05-06T16:51:11.766+0800 I ASIO     [NetworkInterfaceASIO-Replication-0] Connecting to 172.30.129.78:30071
2019-05-06T16:51:11.767+0800 I REPL     [ReplicationExecutor] VoteRequester(term 3 dry run) received a yes vote from 172.30.129.7:30071; response message: { term: 3, voteGranted: true, reason: "", ok: 1.0 }
2019-05-06T16:51:11.767+0800 I REPL     [ReplicationExecutor] dry election run succeeded, running for election
2019-05-06T16:51:11.768+0800 I ASIO     [NetworkInterfaceASIO-Replication-0] Connecting to 172.30.129.78:30071
2019-05-06T16:51:11.771+0800 I REPL     [ReplicationExecutor] VoteRequester(term 4) received a yes vote from 172.30.129.7:30071; response message: { term: 4, voteGranted: true, reason: "", ok: 1.0 }
2019-05-06T16:51:11.771+0800 I REPL     [ReplicationExecutor] election succeeded, assuming primary role in term 4
2019-05-06T16:51:11.771+0800 I REPL     [ReplicationExecutor] transition to PRIMARY
2019-05-06T16:51:11.771+0800 I REPL     [ReplicationExecutor] Entering primary catch-up mode.
2019-05-06T16:51:11.771+0800 I ASIO     [NetworkInterfaceASIO-Replication-0] Ending connection to host 172.30.129.78:30071 due to bad connection status; 2 connections to that host remain open
2019-05-06T16:51:11.771+0800 I ASIO     [NetworkInterfaceASIO-Replication-0] Connecting to 172.30.129.78:30071
2019-05-06T16:51:13.350+0800 I REPL     [ReplicationExecutor] Error in heartbeat request to 172.30.129.78:30071; ExceededTimeLimit: Couldn't get a connection within the time limit

2019-05-06T16:51:12.816+0800 I ASIO     [NetworkInterfaceASIO-Replication-0] Ending connection to host 172.30.129.78:30071 due to bad connection status; 0 connections to that host remain open
2019-05-06T16:51:12.816+0800 I REPL     [ReplicationExecutor] Error in heartbeat request to 172.30.129.78:30071; ExceededTimeLimit: Operation timed out, request was RemoteCommand 72553 -- target:172.30.129.78:30071 db:admin expDate:2019-05-06T16:51:12.816+0800 cmd:{ replSetHeartbeat: "shard0", configVersion: 96911, from: "172.30.129.7:30071", fromId: 1, term: 3 }
2019-05-06T16:51:12.821+0800 I REPL     [ReplicationExecutor] Member 172.30.129.160:30071 is now in state PRIMARY

void ReplicationCoordinatorImpl::_startHeartbeats_inlock() {
const Date_t now = _replExecutor.now();
    _seedList.clear();

//获取副本集成员
for (int i = 0; i restartHeartbeats();

//使用V1的选举协议(3.2之后)
if (isV1ElectionProtocol()) {
for (auto&& slaveInfo : _slaveInfo) {
            slaveInfo.lastUpdate = _replExecutor.now();
            slaveInfo.down = false;
        }

//调度保活状态检查定时器
        _scheduleNextLivenessUpdate_inlock();
    }
}

void ReplicationCoordinatorImpl::_scheduleHeartbeatToTarget(const HostAndPort& target,
                                                            int targetIndex,
                                                            Date_t when) {
//执行调度，在某个时间点调用_doMemberHeartbeat
    _trackHeartbeatHandle(
        _replExecutor.scheduleWorkAt(when,
                                     stdx::bind(&ReplicationCoordinatorImpl::_doMemberHeartbeat,
                                                this,
                                                stdx::placeholders::_1,
                                                target,
                                                targetIndex)));
}

void ReplicationCoordinatorImpl::_doMemberHeartbeat(ReplicationExecutor::CallbackArgs cbData,
const HostAndPort& target,
int targetIndex) {
LockGuard topoLock(_topoMutex);

//取消callback 跟踪
    _untrackHeartbeatHandle(cbData.myHandle);
if (cbData.status == ErrorCodes::CallbackCanceled) {
return;
    }

const Date_t now = _replExecutor.now();
    BSONObj heartbeatObj;
Milliseconds timeout(0);

//3.2 以后的版本
if (isV1ElectionProtocol()) {
const std::pair hbRequest =
            _topCoord->prepareHeartbeatRequestV1(now, _settings.ourSetName(), target);
//构造请求，设置一个timeout
        heartbeatObj = hbRequest.first.toBSON();
        timeout = hbRequest.second;
    } else {
        ...
    }

//构造远程命令
const RemoteCommandRequest request(
        target, "admin", heartbeatObj, BSON(rpc::kReplSetMetadataFieldName <getTerm()) {

//取消并重新调度 electionTimeout定时器
            cancelAndRescheduleElectionTimeout();
        }
    }
    ...
//调用topCoord的processHeartbeatResponse方法处理心跳响应状态，并返回下一步执行的Action
    HeartbeatResponseAction action = _topCoord->processHeartbeatResponse(
        now, networkTime, target, hbStatusResponse, lastApplied);
    ...
//调度下一次心跳，时间间隔采用action提供的信息
    _scheduleHeartbeatToTarget(
        target, targetIndex, std::max(now, action.getNextHeartbeatStartDate()));

//根据Action 执行处理
    _handleHeartbeatResponseAction(action, hbStatusResponse, false);
}

HeartbeatResponseAction TopologyCoordinatorImpl::processHeartbeatResponse(...) {

    ...

    const Milliseconds alreadyElapsed = now - hbStats.getLastHeartbeatStartDate();
    Date_t nextHeartbeatStartDate;

    // 计算下一次 心跳启动时间
    // numFailuresSinceLastStart 对应连续失败的次数(2次以内)
    if (hbStats.getNumFailuresSinceLastStart() <= kMaxHeartbeatRetries &&
        alreadyElapsed = _rsConfig.getElectionTimeoutPeriod()) {
            ...
            //在保活周期后仍然未更新节点，置为down状态
            slaveInfo.down = true;

            //如果当前节点是主，且检测到某个备节点为down的状态，进入memberdown流程
            if (_memberState.primary()) {

                //调用_topCoord的setMemberAsDown方法，记录某个备节点不可达，并获得下一步的指示
               //当大多数节点不可见时，这里会获得让自身降备的指示
                HeartbeatResponseAction action =
                    _topCoord->setMemberAsDown(now, memberIndex, _getMyLastDurableOpTime_inlock());
                //执行指示
                _handleHeartbeatResponseAction(action,
                                               makeStatusWith(),
                                               true);
            }
        }
    }
    //继续调度下一个周期
    _scheduleNextLivenessUpdate_inlock();
}

void ReplicationCoordinatorImpl::_cancelAndRescheduleElectionTimeout_inlock() {

    //如果上一个定时器已经启用了，则直接取消
    if (_handleElectionTimeoutCbh.isValid()) {
        LOG(4) << "Canceling election timeout callback at " << _handleElectionTimeoutWhen;
        _replExecutor.cancel(_handleElectionTimeoutCbh);
        _handleElectionTimeoutCbh = CallbackHandle();
        _handleElectionTimeoutWhen = Date_t();
    }

    //仅支持3.2后的V1版本
    if (!isV1ElectionProtocol()) {
        return;
    }
    //仅备节点可执行
    if (!_memberState.secondary()) {
        return;
    }
    ...
    //是否可以选举
    if (!_rsConfig.getMemberAt(_selfIndex).isElectable()) {
        return;
    }

    //检测周期，由 electionTimeout + randomOffset
    //randomOffset是随机偏移量，默认为 0~0.15*ElectionTimeoutPeriod = 0~1.5s
    Milliseconds randomOffset = _getRandomizedElectionOffset();
    auto now = _replExecutor.now();
    auto when = now + _rsConfig.getElectionTimeoutPeriod() + randomOffset;

    LOG(4) << "Scheduling election timeout callback at " << when;
    _handleElectionTimeoutWhen = when;

    //触发调度，时间为 now + ElectionTimeoutPeriod + randomOffset
    _handleElectionTimeoutCbh =
        _scheduleWorkAt(when,
                        stdx::bind(&ReplicationCoordinatorImpl::_startElectSelfIfEligibleV1,
                                   this,
                                   StartElectionV1Reason::kElectionTimeout));
}