Java State.SERVICE_HEALTHY属性代码示例

/**
 * Wait until one of the following events:
 * <ul>
 * <li>Another thread publishes the results of an attempt to become active
 * using {@link #recordActiveAttempt(ActiveAttemptRecord)}</li>
 * <li>The node enters bad health status</li>
 * <li>The specified timeout elapses</li>
 * </ul>
 * 
 * @param timeoutMillis number of millis to wait
 * @return the published record, or null if the timeout elapses or the
 * service becomes unhealthy 
 * @throws InterruptedException if the thread is interrupted.
 */
private ActiveAttemptRecord waitForActiveAttempt(int timeoutMillis)
    throws InterruptedException {
  long st = System.nanoTime();
  long waitUntil = st + TimeUnit.NANOSECONDS.convert(
      timeoutMillis, TimeUnit.MILLISECONDS);
  
  do {
    // periodically check health state, because entering an
    // unhealthy state could prevent us from ever attempting to
    // become active. We can detect this and respond to the user
    // immediately.
    synchronized (this) {
      if (lastHealthState != State.SERVICE_HEALTHY) {
        // early out if service became unhealthy
        return null;
      }
    }

    synchronized (activeAttemptRecordLock) {
      if ((lastActiveAttemptRecord != null &&
          lastActiveAttemptRecord.nanoTime >= st)) {
        return lastActiveAttemptRecord;
      }
      // Only wait 1sec so that we periodically recheck the health state
      // above.
      activeAttemptRecordLock.wait(1000);
    }
  } while (System.nanoTime() < waitUntil);
  
  // Timeout elapsed.
  LOG.warn(timeoutMillis + "ms timeout elapsed waiting for an attempt " +
      "to become active");
  return null;
}
 

/**
 * Ensure that the local node is in a healthy state, and thus
 * eligible for graceful failover.
 * @throws ServiceFailedException if the node is unhealthy
 */
private synchronized void checkEligibleForFailover()
    throws ServiceFailedException {
  // Check health
  if (this.getLastHealthState() != State.SERVICE_HEALTHY) {
    throw new ServiceFailedException(
        localTarget + " is not currently healthy. " +
        "Cannot be failover target");
  }
}
 

/**
 * Check the current state of the service, and join the election
 * if it should be in the election.
 */
private void recheckElectability() {
  // Maintain lock ordering of elector -> ZKFC
  synchronized (elector) {
    synchronized (this) {
      boolean healthy = lastHealthState == State.SERVICE_HEALTHY;
  
      long remainingDelay = delayJoiningUntilNanotime - System.nanoTime(); 
      if (remainingDelay > 0) {
        if (healthy) {
          LOG.info("Would have joined master election, but this node is " +
              "prohibited from doing so for " +
              TimeUnit.NANOSECONDS.toMillis(remainingDelay) + " more ms");
        }
        scheduleRecheck(remainingDelay);
        return;
      }
  
      switch (lastHealthState) {
      case SERVICE_HEALTHY:
        elector.joinElection(targetToData(localTarget));
        if (quitElectionOnBadState) {
          quitElectionOnBadState = false;
        }
        break;
        
      case INITIALIZING:
        LOG.info("Ensuring that " + localTarget + " does not " +
            "participate in active master election");
        elector.quitElection(false);
        serviceState = HAServiceState.INITIALIZING;
        break;
  
      case SERVICE_UNHEALTHY:
      case SERVICE_NOT_RESPONDING:
        LOG.info("Quitting master election for " + localTarget +
            " and marking that fencing is necessary");
        elector.quitElection(true);
        serviceState = HAServiceState.INITIALIZING;
        break;
        
      case HEALTH_MONITOR_FAILED:
        fatalError("Health monitor failed!");
        break;
        
      default:
        throw new IllegalArgumentException("Unhandled state:"
                                             + lastHealthState);
      }
    }
  }
}
 

/**
 * Coordinate a graceful failover. This proceeds in several phases:
 * 1) Pre-flight checks: ensure that the local node is healthy, and
 * thus a candidate for failover.
 * 2) Determine the current active node. If it is the local node, no
 * need to failover - return success.
 * 3) Ask that node to yield from the election for a number of seconds.
 * 4) Allow the normal election path to run in other threads. Wait until
 * we either become unhealthy or we see an election attempt recorded by
 * the normal code path.
 * 5) Allow the old active to rejoin the election, so a future
 * failback is possible.
 */
private void doGracefulFailover()
    throws ServiceFailedException, IOException, InterruptedException {
  int timeout = FailoverController.getGracefulFenceTimeout(conf) * 2;
  
  // Phase 1: pre-flight checks
  checkEligibleForFailover();
  
  // Phase 2: determine old/current active node. Check that we're not
  // ourselves active, etc.
  HAServiceTarget oldActive = getCurrentActive();
  if (oldActive == null) {
    // No node is currently active. So, if we aren't already
    // active ourselves by means of a normal election, then there's
    // probably something preventing us from becoming active.
    throw new ServiceFailedException(
        "No other node is currently active.");
  }
  
  if (oldActive.getAddress().equals(localTarget.getAddress())) {
    LOG.info("Local node " + localTarget + " is already active. " +
        "No need to failover. Returning success.");
    return;
  }
  
  // Phase 3: ask the old active to yield from the election.
  LOG.info("Asking " + oldActive + " to cede its active state for " +
      timeout + "ms");
  ZKFCProtocol oldZkfc = oldActive.getZKFCProxy(conf, timeout);
  oldZkfc.cedeActive(timeout);

  // Phase 4: wait for the normal election to make the local node
  // active.
  ActiveAttemptRecord attempt = waitForActiveAttempt(timeout + 60000);
  
  if (attempt == null) {
    // We didn't even make an attempt to become active.
    synchronized(this) {
      if (lastHealthState != State.SERVICE_HEALTHY) {
        throw new ServiceFailedException("Unable to become active. " +
          "Service became unhealthy while trying to failover.");          
      }
    }
    
    throw new ServiceFailedException("Unable to become active. " +
        "Local node did not get an opportunity to do so from ZooKeeper, " +
        "or the local node took too long to transition to active.");
  }

  // Phase 5. At this point, we made some attempt to become active. So we
  // can tell the old active to rejoin if it wants. This allows a quick
  // fail-back if we immediately crash.
  oldZkfc.cedeActive(-1);
  
  if (attempt.succeeded) {
    LOG.info("Successfully became active. " + attempt.status);
  } else {
    // Propagate failure
    String msg = "Failed to become active. " + attempt.status;
    throw new ServiceFailedException(msg);
  }
}
 

/**
 * Check the current state of the service, and join the election
 * if it should be in the election.
 */
private void recheckElectability() {
  // Maintain lock ordering of elector -> ZKFC
  synchronized (elector) {
    synchronized (this) {
      boolean healthy = lastHealthState == State.SERVICE_HEALTHY;
  
      long remainingDelay = delayJoiningUntilNanotime - System.nanoTime(); 
      if (remainingDelay > 0) {
        if (healthy) {
          LOG.info("Would have joined master election, but this node is " +
              "prohibited from doing so for " +
              TimeUnit.NANOSECONDS.toMillis(remainingDelay) + " more ms");
        }
        scheduleRecheck(remainingDelay);
        return;
      }
  
      switch (lastHealthState) {
      case SERVICE_HEALTHY:
        elector.joinElection(targetToData(localTarget));
        if (quitElectionOnBadState) {
          quitElectionOnBadState = false;
        }
        break;
        
      case INITIALIZING:
        LOG.info("Ensuring that " + localTarget + " does not " +
            "participate in active master election");
        elector.quitElection(false);
        serviceState = HAServiceState.INITIALIZING;
        break;
  
      case SERVICE_UNHEALTHY:
      case SERVICE_NOT_RESPONDING:
        LOG.info("Quitting master election for " + localTarget +
            " and marking that fencing is necessary");
        elector.quitElection(true);
        serviceState = HAServiceState.INITIALIZING;
        break;
        
      case HEALTH_MONITOR_FAILED:
        fatalError("Health monitor failed!");
        break;
        
      default:
        throw new IllegalArgumentException("Unhandled state:" + lastHealthState);
      }
    }
  }
}
 

/**
 * Check the current state of the service, and join the election
 * if it should be in the election.
 */
private void recheckElectability() {
  // Maintain lock ordering of elector -> ZKFC
  synchronized (elector) {
    synchronized (this) {
      boolean healthy = lastHealthState == State.SERVICE_HEALTHY;
  
      long remainingDelay = delayJoiningUntilNanotime - System.nanoTime(); 
      if (remainingDelay > 0) {
        if (healthy) {
          LOG.info("Would have joined master election, but this node is " +
              "prohibited from doing so for " +
              TimeUnit.NANOSECONDS.toMillis(remainingDelay) + " more ms");
        }
        scheduleRecheck(remainingDelay);
        return;
      }
  
      switch (lastHealthState) {
      case SERVICE_HEALTHY:
        elector.joinElection(targetToData(localTarget));
        break;
        
      case INITIALIZING:
        LOG.info("Ensuring that " + localTarget + " does not " +
            "participate in active master election");
        elector.quitElection(false);
        break;
  
      case SERVICE_UNHEALTHY:
      case SERVICE_NOT_RESPONDING:
        LOG.info("Quitting master election for " + localTarget +
            " and marking that fencing is necessary");
        elector.quitElection(true);
        break;
        
      case HEALTH_MONITOR_FAILED:
        fatalError("Health monitor failed!");
        break;
        
      default:
        throw new IllegalArgumentException("Unhandled state:" + lastHealthState);
      }
    }
  }
}