Java LeaderAndIsr类代码示例

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public static LeaderIsrAndControllerEpoch leaderIsrAndEpochForPartition(
		ZKConnector<?> zkClient, String topic, int partition) {
	
	try{
		String leaderAndIsrPath = topicPartitionLeaderAndIsrPath(topic, partition);
		Pair<String,Stat> leaderAndIsrInfo =zkClient.readDataMaybeNullWithStat(leaderAndIsrPath);
		Map<String, Object> jsonObj = mapper.reader(typeMap).readValue(leaderAndIsrInfo.lhs);
		Integer leader=(Integer)jsonObj.get("leader");
		Integer epoch=(Integer)jsonObj.get("leader_epoch");
		List<Object> isr=(List<Object>)jsonObj.get("isr");
		Integer controllerEpoch=(Integer)jsonObj.get("controller_epoch");
		return new LeaderIsrAndControllerEpoch(new LeaderAndIsr(leader,epoch,(scala.collection.immutable.List<Object>) isr,leaderAndIsrInfo.rhs.getVersion()),controllerEpoch);
	}catch(Exception e){
		
	}
	return  null;
}
 

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
@Override
public Tuple2<LeaderAndIsr, List<Integer>> selectLeader(TopicAndPartition topicAndPartition, LeaderAndIsr currentLeaderAndIsr) {
    List<Integer> assignedReplicas = Lists.newArrayList(controllerContext.partitionReplicaAssignment.get(topicAndPartition));
    Integer preferredReplica = Utils.head(assignedReplicas);
    // check if preferred replica is the current leader
    int currentLeader = controllerContext.partitionLeadershipInfo.get(topicAndPartition).leaderAndIsr.leader;
    if (currentLeader == preferredReplica) {
        throw new LeaderElectionNotNeededException("Preferred replica %d is already the current leader for partition %s",
                preferredReplica, topicAndPartition);
    }

    logger.info("Current leader {} for partition {} is not the preferred replica. Trigerring preferred replica leader election", currentLeader, topicAndPartition);

    // check if preferred replica is not the current leader and is alive and in the isr
    if (controllerContext.liveBrokerIds().contains(preferredReplica) && currentLeaderAndIsr.isr.contains(preferredReplica)) {
        return Tuple2.make(new LeaderAndIsr(preferredReplica, currentLeaderAndIsr.leaderEpoch + 1, currentLeaderAndIsr.isr,
                currentLeaderAndIsr.zkVersion + 1), assignedReplicas);
    }
    throw new StateChangeFailedException("Preferred replica %d for partition %s is either not alive or not in the isr. Current leader and ISR: [%s]",
            preferredReplica, topicAndPartition, currentLeaderAndIsr);
}
 

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
public static LeaderAndIsr leaderAndIsrForPartition(ZKConnector<?> zkClient,
		String topic, int partition) {
	LeaderIsrAndControllerEpoch entity = leaderIsrAndEpochForPartition(
			zkClient, topic, partition);
	if (entity != null)
		return entity.leaderAndIsr();
	return null;
}
 

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
/**
 * This API should read the epoch in the ISR path. It is sufficient to read
 * the epoch in the ISR path, since if the leader fails after updating epoch
 * in the leader path and before updating epoch in the ISR path, effectively
 * some other broker will retry becoming leader with the same new epoch
 * value.
 * @throws Exception 
 */
public static Integer epochForPartition(ZKConnector<?> zkClient, String topic,
		int partition) {
	LeaderAndIsr isr = leaderAndIsrForPartition(zkClient, topic, partition);
	if (isr != null)
		return isr.leaderEpoch();
	else
		throw new KafkaZKException(String.format(
				"No epoch, ISR path for partition [%s,%d] is empty", topic,
				partition));
}
 

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public static List<Integer> isrForPartition(ZKConnector<?> zkClient,
		String topic, int partition) {
	LeaderAndIsr isr = leaderAndIsrForPartition(zkClient, topic, partition);
	if (isr == null)
		throw new KafkaZKException(
				String.format(
						"No epoch, leaderAndISR data for partition [%s,%d] is invalid",
						topic, partition));
	return (List<Integer>) isr.isr();
}
 

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
public static String leaderAndIsrZkData(LeaderAndIsr leaderAndIsr,
		int controllerEpoch) {
	return Json.encode(ImmutableMap.of("version", 1, "leader",
			leaderAndIsr.leader(), "leader_epoch",
			leaderAndIsr.leaderEpoch(), "controller_epoch",
			controllerEpoch, "isr", leaderAndIsr.isr()));
}
 

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
public static LeaderIsrAndControllerEpoch parseLeaderAndIsr(String leaderAndIsrStr, String topic, int partition, Stat stat) {
    JSONObject leaderIsrAndEpochInfo = Json.parseFull(leaderAndIsrStr);
    if (leaderIsrAndEpochInfo == null) return null;

    int leader = leaderIsrAndEpochInfo.getIntValue("leader");
    int epoch = leaderIsrAndEpochInfo.getIntValue("leader_epoch");

    List<Integer> isr = (List<Integer>) leaderIsrAndEpochInfo.get("isr");
    int controllerEpoch = leaderIsrAndEpochInfo.getIntValue("controller_epoch");
    int zkPathVersion = stat.getVersion();
    logger.debug("Leader {}, Epoch {}, Isr {}, Zk path version {} for partition [{},{}]", leader, epoch,
            isr.toString(), zkPathVersion, topic, partition);
    return new LeaderIsrAndControllerEpoch(new LeaderAndIsr(leader, epoch, isr, zkPathVersion), controllerEpoch);

}
 

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
@Override
public Tuple2<LeaderAndIsr, List<Integer>> selectLeader(TopicAndPartition topicAndPartition, LeaderAndIsr currentLeaderAndIsr) {
    int currentLeaderEpoch = currentLeaderAndIsr.leaderEpoch;
    int currentLeaderIsrZkPathVersion = currentLeaderAndIsr.zkVersion;

    final int currentLeader = currentLeaderAndIsr.leader;

    Collection<Integer> assignedReplicas = controllerContext.partitionReplicaAssignment.get(topicAndPartition);
    final Set<Integer> liveOrShuttingDownBrokerIds = controllerContext.liveOrShuttingDownBrokerIds();
    List<Integer> liveAssignedReplicas = Utils.filter(assignedReplicas, new Predicate<Integer>() {
        @Override
        public boolean apply(Integer r) {
            return liveOrShuttingDownBrokerIds.contains(r);
        }
    });

    List<Integer> newIsr = Utils.filter(currentLeaderAndIsr.isr, new Predicate<Integer>() {
        @Override
        public boolean apply(Integer brokerId) {
            return brokerId != currentLeader &&
                    !controllerContext.shuttingDownBrokerIds.contains(brokerId);
        }
    });

    Integer newLeader = Utils.head(newIsr);
    if (newLeader != null) {
        logger.debug("Partition {} : current leader = {}, new leader = {}",
                topicAndPartition, currentLeader, newLeader);
        return Tuple2.make(new LeaderAndIsr(newLeader, currentLeaderEpoch + 1, newIsr, currentLeaderIsrZkPathVersion + 1),
                liveAssignedReplicas);
    }

    throw new StateChangeFailedException("No other replicas in ISR %s for %s besides current leader %d and" +
            " shutting down brokers %s", currentLeaderAndIsr.isr, topicAndPartition, currentLeader, controllerContext.shuttingDownBrokerIds);
}
 

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
/**
 * The reassigned replicas are already in the ISR when selectLeader is called.
 */
@Override
public Tuple2<LeaderAndIsr, List<Integer>> selectLeader(TopicAndPartition topicAndPartition, LeaderAndIsr currentLeaderAndIsr) {
    Collection<Integer> reassignedInSyncReplicas = controllerContext.partitionsBeingReassigned.get(topicAndPartition).newReplicas;
    int currentLeaderEpoch = currentLeaderAndIsr.leaderEpoch;
    int currentLeaderIsrZkPathVersion = currentLeaderAndIsr.zkVersion;
    List<Integer> aliveReassignedInSyncReplicas = Utils.filter(reassignedInSyncReplicas, new Predicate<Integer>() {
        @Override
        public boolean apply(Integer r) {
            return controllerContext.liveBrokerIds().contains(r);
        }
    });
    Integer newLeader = Utils.head(aliveReassignedInSyncReplicas);
    if (newLeader != null) {
        List<Integer> t2 = Lists.newArrayList(reassignedInSyncReplicas);
        return Tuple2.make(new LeaderAndIsr(newLeader, currentLeaderEpoch + 1, currentLeaderAndIsr.isr,
                currentLeaderIsrZkPathVersion + 1), t2);
    }

    switch (reassignedInSyncReplicas.size()) {
        case 0:
            throw new StateChangeFailedException("List of reassigned replicas for partition " +
                    " %s is empty. Current leader and ISR: [%s]", topicAndPartition, currentLeaderAndIsr);
        default:
            throw new StateChangeFailedException("None of the reassigned replicas for partition " +
                    "%s are alive. Current leader and ISR: [%s]", topicAndPartition, currentLeaderAndIsr);
    }
}
 

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
private boolean areReplicasInIsr(String topic, int partition, Collection<Integer> replicas) {
    final LeaderAndIsr leaderAndIsr = getLeaderAndIsrForPartition(zkClient, topic, partition);
    if (leaderAndIsr != null) {
        List<Integer> replicasNotInIsr = Utils.filter(replicas, new Predicate<Integer>() {
            @Override
            public boolean apply(Integer r) {
                return !leaderAndIsr.isr.contains(r);
            }
        });
        return replicasNotInIsr.isEmpty();
    } else {
        return false;
    }
}
 

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
@Override
public void handleDataChange(String dataPath, Object data) throws Exception {
    try {
        synchronized (controllerContext.controllerLock) {
            logger.debug("Reassigned partitions isr change listener fired for path %s with children {}", dataPath, data);
            // check if this partition is still being reassigned or not
            TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
            ReassignedPartitionsContext reassignedPartitionContext = controllerContext.partitionsBeingReassigned.get(topicAndPartition);
            if (reassignedPartitionContext != null) {
                // need to re-read leader and isr from zookeeper since the zkclient callback doesn't return the Stat object
                LeaderAndIsr leaderAndIsr = ZkUtils.getLeaderAndIsrForPartition(zkClient, topic, partition);
                if (leaderAndIsr != null) { // check if new replicas have joined ISR
                    Set<Integer> caughtUpReplicas = Utils.and(reassignedReplicas, leaderAndIsr.isr);
                    if (caughtUpReplicas.equals(reassignedReplicas)) {
                        // resume the partition reassignment process
                        logger.info("{}/{} replicas have caught up with the leader for partition {} being reassigned.Resuming partition reassignment",
                                caughtUpReplicas.size(), reassignedReplicas.size(), topicAndPartition);
                        controller.onPartitionReassignment(topicAndPartition, reassignedPartitionContext);
                    } else {
                        logger.info("{}/{} replicas have caught up with the leader for partition {} being reassigned. Replica(s) {} still need to catch up",
                                caughtUpReplicas.size(), reassignedReplicas.size(), topicAndPartition, Utils.minus(reassignedReplicas, leaderAndIsr.isr));
                    }
                } else {
                    logger.error("Error handling reassignment of partition {} to replicas {} as it was never created",
                            topicAndPartition, reassignedReplicas);
                }
            }
        }
    } catch (Throwable e) {
        logger.error("Error while handling partition reassignment", e);
    }
}
 

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
public static LeaderAndIsr getLeaderAndIsrForPartition(ZkClient zkClient, String topic, int partition) {
    return getLeaderIsrAndEpochForPartition(zkClient, topic, partition).leaderAndIsr;
}
 

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
public static String leaderAndIsrZkData(LeaderAndIsr leaderAndIsr, int controllerEpoch) {
    return Json.encode(ImmutableMap.of("version", 1, "leader", leaderAndIsr.leader, "leader_epoch", leaderAndIsr.leaderEpoch,
            "controller_epoch", controllerEpoch, "isr", leaderAndIsr.isr));
}
 

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
/**
 * Invoked on the NewPartition->OnlinePartition state change. When a partition is in the New state, it does not have
 * a leader and isr path in zookeeper. Once the partition moves to the OnlinePartition state, it's leader and isr
 * path gets initialized and it never goes back to the NewPartition state. From here, it can only go to the
 * OfflinePartition state.
 *
 * @param topicAndPartition The topic/partition whose leader and isr path is to be initialized
 */
private void initializeLeaderAndIsrForPartition(TopicAndPartition topicAndPartition) {
    Collection<Integer> replicaAssignment = controllerContext.partitionReplicaAssignment.get(topicAndPartition);
    List<Integer> liveAssignedReplicas = Utils.filter(replicaAssignment, new Predicate<Integer>() {
        @Override
        public boolean apply(Integer r) {
            return controllerContext.liveBrokerIds().contains(r);
        }
    });
    switch (liveAssignedReplicas.size()) {
        case 0:
            String failMsg = String.format("encountered error during state change of partition %s from New to Online, assigned replicas are [%s], " +
                    "live brokers are [%s]. No assigned replica is alive.",
                    topicAndPartition, replicaAssignment.toString(), controllerContext.liveBrokerIds());
            stateChangeLogger.error("Controller {} epoch {} ", controllerId, controller.epoch() + failMsg);
            throw new StateChangeFailedException(failMsg);
        default:
            logger.debug("Live assigned replicas for partition {} are: [{}]", topicAndPartition, liveAssignedReplicas);
            // make the first replica in the list of assigned replicas, the leader
            Integer leader = Utils.head(liveAssignedReplicas);
            LeaderIsrAndControllerEpoch leaderIsrAndControllerEpoch = new LeaderIsrAndControllerEpoch(new LeaderAndIsr(leader, liveAssignedReplicas),
                    controller.epoch());
            logger.debug("Initializing leader and isr for partition {} to {}", topicAndPartition, leaderIsrAndControllerEpoch);
            try {
                ZkUtils.createPersistentPath(controllerContext.zkClient,
                        ZkUtils.getTopicPartitionLeaderAndIsrPath(topicAndPartition.topic, topicAndPartition.partition),
                        ZkUtils.leaderAndIsrZkData(leaderIsrAndControllerEpoch.leaderAndIsr, controller.epoch()));
                // NOTE: the above write can fail only if the current controller lost its zk session and the new controller
                // took over and initialized this partition. This can happen if the current controller went into a long
                // GC pause
                controllerContext.partitionLeadershipInfo.put(topicAndPartition, leaderIsrAndControllerEpoch);
                brokerRequestBatch.addLeaderAndIsrRequestForBrokers(liveAssignedReplicas, topicAndPartition.topic,
                        topicAndPartition.partition, leaderIsrAndControllerEpoch, Sets.newHashSet(replicaAssignment));
            } catch (ZkNodeExistsException e) {
                // read the controller epoch
                LeaderIsrAndControllerEpoch leaderIsrAndEpoch = ZkUtils.getLeaderIsrAndEpochForPartition(zkClient, topicAndPartition.topic,
                        topicAndPartition.partition);
                failMsg = String.format("encountered error while changing partition %s's state from New to Online since LeaderAndIsr path already " +
                        "exists with value %s and controller epoch %d",
                        topicAndPartition, leaderIsrAndEpoch.leaderAndIsr.toString(), leaderIsrAndEpoch.controllerEpoch);
                stateChangeLogger.error(String.format("Controller %d epoch %d ", controllerId, controller.epoch()) + failMsg);
                throw new StateChangeFailedException(failMsg);
            }
    }
}
 

import kafka.api.LeaderAndIsr; //导入依赖的package包/类
@Override
public Tuple2<LeaderAndIsr, List<Integer>> selectLeader(TopicAndPartition topicAndPartition, LeaderAndIsr currentLeaderAndIsr) {
    logger.warn("I should never have been asked to perform leader election, returning the current LeaderAndIsr and replica assignment.");
    List<Integer> t2 = Lists.newArrayList(controllerContext.partitionReplicaAssignment.get(topicAndPartition));
    return Tuple2.make(currentLeaderAndIsr, t2);
}