Java ZNRecord.setListField方法代码示例

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
@Test
public void testLegacyEnableDisablePartition() {
  String instanceName = "TestInstanceLegacy";
  String testResourcePrefix = "TestResourceLegacy";
  ZNRecord record = new ZNRecord(instanceName);
  List<String> disabledPartitions =
      new ArrayList<String>(Arrays.asList(new String[] { "1", "2", "3" }));
  record.setListField(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_PARTITION.name(),
      disabledPartitions);
  InstanceConfig instanceConfig = new InstanceConfig(record);
  instanceConfig.setInstanceEnabledForPartition(testResourcePrefix, "2", false);
  Assert.assertEquals(instanceConfig.getDisabledPartitions(testResourcePrefix).size(), 3);
  Assert.assertEquals(instanceConfig.getRecord()
          .getListField(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_PARTITION.name()).size(),
      3);
  instanceConfig.setInstanceEnabledForPartition(testResourcePrefix, "2", true);
  Assert.assertEquals(instanceConfig.getDisabledPartitions(testResourcePrefix).size(), 2);
  Assert.assertEquals(instanceConfig.getRecord()
          .getListField(InstanceConfig.InstanceConfigProperty.HELIX_DISABLED_PARTITION.name()).size(),
      2);
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
private ZNRecord getTestPartitionToReplicaGroupMappingZNRecord() {
  String tableName = "testTable";
  ZNRecord record = new ZNRecord(tableName);

  List<String> replicaGroupOne = new ArrayList<>();
  replicaGroupOne.add("instance1");
  replicaGroupOne.add("instance2");

  List<String> replicaGroupTwo = new ArrayList<>();
  replicaGroupTwo.add("instance3");
  replicaGroupTwo.add("instance4");

  record.setListField(generateKeyForPartitionMappingTable(0, 0), replicaGroupOne);
  record.setListField(generateKeyForPartitionMappingTable(0, 1), replicaGroupTwo);
  record.setListField(generateKeyForPartitionMappingTable(1, 0), replicaGroupOne);
  record.setListField(generateKeyForPartitionMappingTable(1, 1), replicaGroupTwo);

  return record;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
private Map<Integer, List<String>> buildKafkaPartitionMapping(String tableName, FakePropertyStore propertyStore,
    List<InstanceConfig> instanceConfigs) {
  // Create partition assignment mapping table.
  Map<Integer, List<String>> partitionToServers = new HashMap<>();
  ZNRecord kafkaPartitionMapping = new ZNRecord(REALTIME_TABLE_NAME);

  int serverIndex = 0;
  for (int partitionId = 0; partitionId < NUM_PARTITION; partitionId++) {
    List<String> assignedServers = new ArrayList<>();
    for (int replicaId = 0; replicaId < NUM_REPLICA; replicaId++) {
      assignedServers.add(instanceConfigs.get(serverIndex % NUM_SERVERS).getInstanceName());
      serverIndex++;
    }
    partitionToServers.put(partitionId, assignedServers);
    kafkaPartitionMapping.setListField(Integer.toString(partitionId), assignedServers);
  }

  String path = ZKMetadataProvider.constructPropertyStorePathForKafkaPartitions(tableName);
  propertyStore.set(path, kafkaPartitionMapping, AccessOption.PERSISTENT);

  return partitionToServers;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
StringRepresentation getStateModelsRepresentation() throws JsonGenerationException,
    JsonMappingException, IOException {
  String clusterName = (String) getRequest().getAttributes().get("clusterName");
  ZkClient zkClient = (ZkClient) getContext().getAttributes().get(RestAdminApplication.ZKCLIENT);
  ClusterSetup setupTool = new ClusterSetup(zkClient);

  List<String> models = setupTool.getClusterManagementTool().getStateModelDefs(clusterName);

  ZNRecord modelDefinitions = new ZNRecord("modelDefinitions");
  modelDefinitions.setListField("models", models);

  StringRepresentation representation =
      new StringRepresentation(ClusterRepresentationUtil.ZNRecordToJson(modelDefinitions),
          MediaType.APPLICATION_JSON);

  return representation;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
private ZNRecord getFinalDelayedMapping(IdealState idealState, ZNRecord newIdealMapping,
    ZNRecord newActiveMapping, Set<String> liveInstances, int numReplica, int minActiveReplica) {
  if (minActiveReplica >= numReplica) {
    return newIdealMapping;
  }
  ZNRecord finalMapping = new ZNRecord(idealState.getResourceName());
  for (String partition : newIdealMapping.getListFields().keySet()) {
    List<String> idealList = newIdealMapping.getListField(partition);
    List<String> activeList = newActiveMapping.getListField(partition);

    List<String> liveList = new ArrayList<>();
    int activeReplica = 0;
    for (String ins : activeList) {
      if (liveInstances.contains(ins)) {
        activeReplica++;
        liveList.add(ins);
      }
    }

    if (activeReplica >= minActiveReplica) {
      finalMapping.setListField(partition, activeList);
    } else {
      List<String> candidates = new ArrayList<String>(idealList);
      candidates.removeAll(activeList);
      for (String liveIns : candidates) {
        liveList.add(liveIns);
        if (liveList.size() >= minActiveReplica) {
          break;
        }
      }
      finalMapping.setListField(partition, liveList);
    }
  }
  return finalMapping;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
/**
 * Calculate the initial ideal state given a batch of storage instances, the replication factor
 * and
 * number of partitions
 * 1. Calculate the master assignment by random shuffling
 * 2. for each storage instance, calculate the 1st slave assignment map, by another random
 * shuffling
 * 3. for each storage instance, calculate the i-th slave assignment map
 * 4. Combine the i-th slave assignment maps together
 * @param instanceNames
 *          list of storage node instances
 * @param partitions
 *          number of partitions
 * @param replicas
 *          The number of replicas (slave partitions) per master partition
 * @param masterStateValue
 *          master state value: e.g. "MASTER" or "LEADER"
 * @param slaveStateValue
 *          slave state value: e.g. "SLAVE" or "STANDBY"
 * @param resourceName
 * @return a ZNRecord that contain the idealstate info
 */
public static ZNRecord calculateIdealState(List<String> instanceNames, int partitions,
    int replicas, String resourceName, String masterStateValue, String slaveStateValue) {
  Collections.sort(instanceNames);
  if (instanceNames.size() < replicas + 1) {
    throw new HelixException("Number of instances must not be less than replicas + 1. "
        + "instanceNr:" + instanceNames.size() + ", replicas:" + replicas);
  } else if (partitions < instanceNames.size()) {
    ZNRecord idealState =
        IdealStateCalculatorByShuffling.calculateIdealState(instanceNames, partitions, replicas,
            resourceName, 12345, masterStateValue, slaveStateValue);
    int i = 0;
    for (String partitionId : idealState.getMapFields().keySet()) {
      Map<String, String> partitionAssignmentMap = idealState.getMapField(partitionId);
      List<String> partitionAssignmentPriorityList = new ArrayList<String>();
      String masterInstance = "";
      for (String instanceName : partitionAssignmentMap.keySet()) {
        if (partitionAssignmentMap.get(instanceName).equalsIgnoreCase(masterStateValue)
            && masterInstance.equals("")) {
          masterInstance = instanceName;
        } else {
          partitionAssignmentPriorityList.add(instanceName);
        }
      }
      Collections.shuffle(partitionAssignmentPriorityList, new Random(i++));
      partitionAssignmentPriorityList.add(0, masterInstance);
      idealState.setListField(partitionId, partitionAssignmentPriorityList);
    }
    return idealState;
  }

  Map<String, Object> result = calculateInitialIdealState(instanceNames, partitions, replicas);

  return convertToZNRecord(result, resourceName, masterStateValue, slaveStateValue);
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
StringRepresentation getHostedEntitiesRepresentation(String clusterName)
    throws JsonGenerationException, JsonMappingException, IOException {
  // Get all resources
  ZkClient zkClient =
      ResourceUtil.getAttributeFromCtx(getContext(), ResourceUtil.ContextKey.ZKCLIENT);
  HelixDataAccessor accessor =
      ClusterRepresentationUtil.getClusterDataAccessor(zkClient, clusterName);
  PropertyKey.Builder keyBuilder = accessor.keyBuilder();
  Map<String, HelixProperty> resourceConfigMap =
      accessor.getChildValuesMap(keyBuilder.resourceConfigs());

  // Create the result
  ZNRecord hostedEntitiesRecord = new ZNRecord("JobQueues");

  // Filter out non-workflow resources
  Iterator<Map.Entry<String, HelixProperty>> it = resourceConfigMap.entrySet().iterator();
  while (it.hasNext()) {
    Map.Entry<String, HelixProperty> e = it.next();
    HelixProperty resource = e.getValue();
    Map<String, String> simpleFields = resource.getRecord().getSimpleFields();
    boolean isTerminable = resource.getRecord()
        .getBooleanField(WorkflowConfig.WorkflowConfigProperty.Terminable.name(), true);
    if (!simpleFields.containsKey(WorkflowConfig.WorkflowConfigProperty.TargetState.name())
        || !simpleFields.containsKey(WorkflowConfig.WorkflowConfigProperty.Dag.name())
        || isTerminable) {
      it.remove();
    }
  }

  // Populate the result
  List<String> allResources = Lists.newArrayList(resourceConfigMap.keySet());
  hostedEntitiesRecord.setListField("JobQueues", allResources);

  StringRepresentation representation =
      new StringRepresentation(ClusterRepresentationUtil.ZNRecordToJson(hostedEntitiesRecord),
          MediaType.APPLICATION_JSON);

  return representation;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
StringRepresentation getHostedEntitiesRepresentation(String clusterName)
    throws JsonGenerationException, JsonMappingException, IOException {
  // Get all resources
  ZkClient zkClient = (ZkClient) getContext().getAttributes().get(RestAdminApplication.ZKCLIENT);
  HelixDataAccessor accessor =
      ClusterRepresentationUtil.getClusterDataAccessor(zkClient, clusterName);
  PropertyKey.Builder keyBuilder = accessor.keyBuilder();
  Map<String, HelixProperty> resourceConfigMap =
      accessor.getChildValuesMap(keyBuilder.resourceConfigs());

  // Create the result
  ZNRecord hostedEntitiesRecord = new ZNRecord("Workflows");

  // Filter out non-workflow resources
  Iterator<Map.Entry<String, HelixProperty>> it = resourceConfigMap.entrySet().iterator();
  while (it.hasNext()) {
    Map.Entry<String, HelixProperty> e = it.next();
    HelixProperty resource = e.getValue();
    Map<String, String> simpleFields = resource.getRecord().getSimpleFields();
    if (!simpleFields.containsKey(WorkflowConfig.WorkflowConfigProperty.TargetState.name())
        || !simpleFields.containsKey(WorkflowConfig.WorkflowConfigProperty.Dag.name())) {
      it.remove();
    }
  }

  // Populate the result
  List<String> allResources = Lists.newArrayList(resourceConfigMap.keySet());
  hostedEntitiesRecord.setListField("WorkflowList", allResources);

  StringRepresentation representation =
      new StringRepresentation(ClusterRepresentationUtil.ZNRecordToJson(hostedEntitiesRecord),
          MediaType.APPLICATION_JSON);

  return representation;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
void startAdmin() throws Exception {
  HelixAdmin admin = new ZKHelixAdmin(ZK_ADDR);

  // create cluster
  System.out.println("Creating cluster: " + clusterName);
  admin.addCluster(clusterName, true);

  // add MasterSlave state mode definition
  admin.addStateModelDef(clusterName, "MasterSlave", new StateModelDefinition(
      generateConfigForMasterSlave()));

  // ideal-state znrecord
  ZNRecord record = new ZNRecord(resourceName);
  record.setSimpleField("IDEAL_STATE_MODE", "AUTO");
  record.setSimpleField("NUM_PARTITIONS", "1");
  record.setSimpleField("REPLICAS", "2");
  record.setSimpleField("STATE_MODEL_DEF_REF", "MasterSlave");
  record.setListField(resourceName, Arrays.asList("node1", "node2"));

  admin.setResourceIdealState(clusterName, resourceName, new IdealState(record));

  ConstraintItemBuilder builder = new ConstraintItemBuilder();

  // limit one transition message at a time across the entire cluster
  builder.addConstraintAttribute("MESSAGE_TYPE", "STATE_TRANSITION")
  // .addConstraintAttribute("INSTANCE", ".*") // un-comment this line if using instance-level
  // constraint
      .addConstraintAttribute("CONSTRAINT_VALUE", "1");
  admin.setConstraint(clusterName, ClusterConstraints.ConstraintType.MESSAGE_CONSTRAINT,
      "constraint1", builder.build());
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
/**
 * Test the normal case of serialize/deserialize where ZNRecord is well-formed
 */
@Test
public void basicTest() {
  ZNRecord record = new ZNRecord("testId");
  record.setMapField("k1", ImmutableMap.of("a", "b", "c", "d"));
  record.setMapField("k2", ImmutableMap.of("e", "f", "g", "h"));
  record.setListField("k3", ImmutableList.of("a", "b", "c", "d"));
  record.setListField("k4", ImmutableList.of("d", "e", "f", "g"));
  record.setSimpleField("k5", "a");
  record.setSimpleField("k5", "b");
  ZNRecordSerializer serializer = new ZNRecordSerializer();
  ZNRecord result = (ZNRecord) serializer.deserialize(serializer.serialize(record));
  Assert.assertEquals(result, record);
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
/**
 * Test the normal case of serialize/deserialize where ZNRecord is well-formed
 */
@Test
public void basicTest() {
  ZNRecord record = new ZNRecord("testId");
  record.setMapField("k1", ImmutableMap.of("a", "b", "c", "d"));
  record.setMapField("k2", ImmutableMap.of("e", "f", "g", "h"));
  record.setListField("k3", ImmutableList.of("a", "b", "c", "d"));
  record.setListField("k4", ImmutableList.of("d", "e", "f", "g"));
  record.setSimpleField("k5", "a");
  record.setSimpleField("k5", "b");
  ZNRecordStreamingSerializer serializer = new ZNRecordStreamingSerializer();
  ZNRecord result = (ZNRecord) serializer.deserialize(serializer.serialize(record));
  Assert.assertEquals(result, record);
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
private ZNRecord getExampleZNRecord() {
  ZNRecord record = new ZNRecord("TestDB");
  record.setSimpleField(IdealStateProperty.REBALANCE_MODE.toString(),
      RebalanceMode.CUSTOMIZED.toString());
  Map<String, String> map = new HashMap<String, String>();
  map.put("localhost_12918", "MASTER");
  map.put("localhost_12919", "SLAVE");
  record.setMapField("TestDB_0", map);

  List<String> list = new ArrayList<String>();
  list.add("localhost_12918");
  list.add("localhost_12919");
  record.setListField("TestDB_0", list);
  return record;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
protected List<IdealState> setupIdealState(int nodes, String[] resources, int partitions,
    int replicas, RebalanceMode rebalanceMode, String stateModelName, String rebalanceClassName,
    String rebalanceStrategyName) {
  List<IdealState> idealStates = new ArrayList<IdealState>();
  for (int i = 0; i < resources.length; i++) {
    String resourceName = resources[i];
    ZNRecord record = new ZNRecord(resourceName);
    for (int p = 0; p < partitions; p++) {
      List<String> value = new ArrayList<String>();
      for (int r = 0; r < replicas; r++) {
        value.add(HOSTNAME_PREFIX + (p + r + 1) % nodes);
      }
      record.setListField(resourceName + "_" + p, value);
    }
    IdealState idealState = new IdealState(record);
    idealState.setStateModelDefRef(stateModelName);
    if (rebalanceClassName != null) {
      idealState.setRebalancerClassName(rebalanceClassName);
    }
    if (rebalanceStrategyName != null) {
      idealState.setRebalanceStrategy(rebalanceStrategyName);
    }
    idealState.setRebalanceMode(rebalanceMode);
    idealState.setNumPartitions(partitions);
    idealStates.add(idealState);
    idealState.setReplicas(String.valueOf(replicas));

    Builder keyBuilder = accessor.keyBuilder();

    accessor.setProperty(keyBuilder.idealStates(resourceName), idealState);
  }
  return idealStates;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
@Test (dependsOnMethods = "testUpdateConfigFields")
public void testDeleteConfigFields()
    throws IOException {
  System.out.println("Start test :" + TestHelper.getTestMethodName());
  String cluster = _clusters.iterator().next();
  ClusterConfig config = getClusterConfigFromRest(cluster);

  ZNRecord record = config.getRecord();

  String simpleKey = record.getSimpleFields().keySet().iterator().next();
  String value = record.getSimpleField(simpleKey);
  record.getSimpleFields().clear();
  record.setSimpleField(simpleKey, value);

  String listKey = record.getListFields().keySet().iterator().next();
  List<String> list = record.getListField(listKey);
  record.getListFields().clear();
  record.setListField(listKey, list);

  String mapKey = record.getMapFields().keySet().iterator().next();
  Map<String, String> map = record.getMapField(mapKey);
  record.getMapFields().clear();
  record.setMapField(mapKey, map);

  ClusterConfig prevConfig = getClusterConfigFromRest(cluster);
  updateClusterConfigFromRest(cluster, config, Command.delete);
  ClusterConfig newConfig = getClusterConfigFromRest(cluster);

  Assert.assertFalse(newConfig.getRecord().getSimpleFields().containsKey(simpleKey),
      "Failed to delete key " + simpleKey + " from cluster config");
  Assert.assertFalse(newConfig.getRecord().getListFields().containsKey(listKey),
      "Failed to delete key " + listKey + " from cluster config");
  Assert.assertFalse(newConfig.getRecord().getSimpleFields().containsKey(mapKey),
      "Failed to delete key " + mapKey + " from cluster config");

  prevConfig.getRecord().subtract(config.getRecord());
  Assert.assertEquals(newConfig, prevConfig,
      "cluster config from response: " + newConfig + " vs cluster config actually: "
          + prevConfig);
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
StringRepresentation getHostedEntitiesRepresentation(String clusterName)
    throws JsonGenerationException, JsonMappingException, IOException {
  // Get all resources
  ZkClient zkclient =
      ResourceUtil.getAttributeFromCtx(getContext(), ResourceUtil.ContextKey.RAW_ZKCLIENT);
  PropertyKey.Builder keyBuilder = new PropertyKey.Builder(clusterName);
  Map<String, String> idealStateMap =
      ResourceUtil.readZkChildrenAsBytesMap(zkclient, keyBuilder.idealStates());

  // Create the result
  ZNRecord hostedEntitiesRecord = new ZNRecord("ResourceGroups");

  // Figure out which tags are present on which resources
  Map<String, String> tagMap = Maps.newHashMap();
  for (String resourceName : idealStateMap.keySet()) {
    String idealStateStr = idealStateMap.get(resourceName);
    String tag =
        ResourceUtil.extractSimpleFieldFromZNRecord(idealStateStr,
            IdealState.IdealStateProperty.INSTANCE_GROUP_TAG.toString());
    if (tag != null) {
      tagMap.put(resourceName, tag);
    }
  }

  // Populate the result
  List<String> allResources = Lists.newArrayList(idealStateMap.keySet());
  hostedEntitiesRecord.setListField("ResourceGroups", allResources);
  if (!tagMap.isEmpty()) {
    hostedEntitiesRecord.setMapField("ResourceTags", tagMap);
  }

  StringRepresentation representation =
      new StringRepresentation(ClusterRepresentationUtil.ZNRecordToJson(hostedEntitiesRecord),
          MediaType.APPLICATION_JSON);

  return representation;
}