Java ZNRecord.setMapField方法代码示例

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
/**
 * Create a ZNRecord for a message, which stores the content of the message (stored in
 * simple fields) into the ZNRecord mapFields. In this way, the message update can be
 * merged with the previous status update record in the zookeeper. See ZNRecord.merge()
 * for more details.
 */
ZNRecord createMessageLogRecord(Message message) {
  ZNRecord result = new ZNRecord(getStatusUpdateRecordName(message));
  String mapFieldKey = "MESSAGE " + message.getMsgId();
  result.setMapField(mapFieldKey, new TreeMap<String, String>());

  // Store all the simple fields of the message in the new ZNRecord's map
  // field.
  for (String simpleFieldKey : message.getRecord().getSimpleFields().keySet()) {
    result.getMapField(mapFieldKey).put(simpleFieldKey,
        message.getRecord().getSimpleField(simpleFieldKey));
  }
  if (message.getResultMap() != null) {
    result.setMapField("MessageResult", message.getResultMap());
  }
  return result;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
@Test
public void testEquals() {
  ZNRecord record1 = new ZNRecord("id");
  record1.setSimpleField("k1", "v1");
  record1.setMapField("m1", new HashMap<String, String>());
  record1.getMapField("m1").put("k1", "v1");
  record1.setListField("l1", new ArrayList<String>());
  record1.getListField("l1").add("v1");
  ZNRecord record2 = new ZNRecord("id");
  record2.setSimpleField("k1", "v1");
  record2.setMapField("m1", new HashMap<String, String>());
  record2.getMapField("m1").put("k1", "v1");
  record2.setListField("l1", new ArrayList<String>());
  record2.getListField("l1").add("v1");

  AssertJUnit.assertEquals(record1, record2);
  record2.setSimpleField("k2", "v1");
  AssertJUnit.assertNotSame(record1, record2);
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
@Override
public ZNRecord toZNRecord() {
  ZNRecord znRecord = new ZNRecord(getId());
  znRecord.setMapField(KAFKA_HIGH_LEVEL_CONSUMER_GROUP_MAP, _groupIdMap);
  znRecord.setMapField(KAFKA_HIGH_LEVEL_CONSUMER_PARTITION_MAP, _partitionMap);
  return znRecord;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
private ZNRecord getTestInstanceZNRecord() {
  ZNRecord record = new ZNRecord("Server_lva1-app0120.corp.linkedin.com_8001");
  Map<String, String> groupIdMap = new HashMap<String, String>();
  Map<String, String> partitionMap = new HashMap<String, String>();

  for (int i = 0; i < 10; ++i) {
    groupIdMap.put("testRes" + i + "_REALTIME", "groupId" + i);
    partitionMap.put("testRes" + i + "_REALTIME", "part" + i);
  }
  record.setMapField("KAFKA_HLC_GROUP_MAP", groupIdMap);
  record.setMapField("KAFKA_HLC_PARTITION_MAP", partitionMap);
  return record;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
private static InstanceZKMetadata getInstanceZKMetadata() {
  ZNRecord record = new ZNRecord("Server_lva1-app0120.corp.linkedin.com_8001");
  Map<String, String> groupIdMap = new HashMap<String, String>();
  Map<String, String> partitionMap = new HashMap<String, String>();

  groupIdMap.put("mirror", "groupId_testTable_" + String.valueOf(System.currentTimeMillis()));
  partitionMap.put("testTable_R", "0");
  record.setMapField("KAFKA_HLC_GROUP_MAP", groupIdMap);
  record.setMapField("KAFKA_HLC_PARTITION_MAP", partitionMap);
  return new InstanceZKMetadata(record);
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
/**
 * Calculate the ideal state for list of instances clusters using consistent
 * hashing.
 * @param instanceNames
 *          List of instance names.
 * @param partitions
 *          the partition number of the database
 * @param replicas
 *          the replication degree
 * @param resourceName
 *          the name of the database
 * @param hashRingSize
 *          the size of the hash ring used by consistent hashing
 * @return The ZNRecord that contains the ideal state
 */
public static ZNRecord calculateIdealState(List<String> instanceNames, int partitions,
    int replicas, String resourceName, HashFunction hashFunc, int hashRingSize) {
  ZNRecord result = new ZNRecord(resourceName);

  int[] hashRing = generateEvenHashRing(instanceNames, hashRingSize);
  result.setSimpleField(IdealStateProperty.NUM_PARTITIONS.toString(), String.valueOf(partitions));
  Random rand = new Random(0xc0ffee);
  for (int i = 0; i < partitions; i++) {
    String partitionName = resourceName + ".partition-" + i;
    int hashPos = rand.nextInt() % hashRingSize;
    // (int)(hashFunc.getHashValue(partitionName) % hashRingSize);
    hashPos = hashPos < 0 ? (hashPos + hashRingSize) : hashPos;
    // System.out.print(hashPos+ " ");
    // if(i % 120 == 0) System.out.println();
    Map<String, String> partitionAssignment = new TreeMap<String, String>();
    // the first in the list is the node that contains the master
    int masterPos = hashRing[hashPos];
    partitionAssignment.put(instanceNames.get(masterPos), "MASTER");

    // partitionAssignment.put("hash", "" + hashPos + " " + masterPos);

    // Put slaves in next has ring positions. We need to make sure that no
    // more than 2 slaves
    // are mapped to one node.
    for (int j = 1; j <= replicas; j++) {
      String next = instanceNames.get(hashRing[(hashPos + j) % hashRingSize]);
      while (partitionAssignment.containsKey(next)) {
        hashPos++;
        next = instanceNames.get(hashRing[(hashPos + j) % hashRingSize]);
      }
      partitionAssignment.put(next, "SLAVE");
    }
    result.setMapField(partitionName, partitionAssignment);
  }
  return result;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
private static InstanceZKMetadata getInstanceZKMetadata() {
  ZNRecord record = new ZNRecord("Server_localhost_1234");
  Map<String, String> groupIdMap = new HashMap<>();
  Map<String, String> partitionMap = new HashMap<>();

  groupIdMap.put("mirror", "groupId_testTable_" + String.valueOf(System.currentTimeMillis()));
  partitionMap.put("testTable_R", "0");
  record.setMapField("KAFKA_HLC_GROUP_MAP", groupIdMap);
  record.setMapField("KAFKA_HLC_PARTITION_MAP", partitionMap);
  return new InstanceZKMetadata(record);
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
public static ZNRecord calculateIdealState(List<String> instanceNames, int partitions,
    int replicas, String resourceName, long randomSeed, String masterValue, String slaveValue) {
  if (instanceNames.size() <= replicas) {
    throw new IllegalArgumentException("Replicas must be less than number of storage nodes");
  }

  Collections.sort(instanceNames);

  ZNRecord result = new ZNRecord(resourceName);

  List<Integer> partitionList = new ArrayList<Integer>(partitions);
  for (int i = 0; i < partitions; i++) {
    partitionList.add(new Integer(i));
  }
  Random rand = new Random(randomSeed);
  // Shuffle the partition list
  Collections.shuffle(partitionList, rand);

  for (int i = 0; i < partitionList.size(); i++) {
    int partitionId = partitionList.get(i);
    Map<String, String> partitionAssignment = new TreeMap<String, String>();
    int masterNode = i % instanceNames.size();
    // the first in the list is the node that contains the master
    partitionAssignment.put(instanceNames.get(masterNode), masterValue);

    // for the jth replica, we put it on (masterNode + j) % nodes-th
    // node
    for (int j = 1; j <= replicas; j++) {
      int index = (masterNode + j * partitionList.size()) % instanceNames.size();
      while (partitionAssignment.keySet().contains(instanceNames.get(index))) {
        index = (index + 1) % instanceNames.size();
      }
      partitionAssignment.put(instanceNames.get(index), slaveValue);
    }
    String partitionName = resourceName + "_" + partitionId;
    result.setMapField(partitionName, partitionAssignment);
  }
  result.setSimpleField(IdealStateProperty.NUM_PARTITIONS.toString(), String.valueOf(partitions));
  return result;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
/**
 * Create a statusupdate that is related to a cluster manager message.
 * @param message
 *          the related cluster manager message
 * @param level
 *          the error level
 * @param classInfo
 *          class info about the class that reports the status update
 * @param additionalInfo
 *          info the additional debug information
 */
public ZNRecord createMessageStatusUpdateRecord(Message message, Level level, Class classInfo,
    String additionalInfo) {
  ZNRecord result = createEmptyStatusUpdateRecord(getStatusUpdateRecordName(message));
  Map<String, String> contentMap = new TreeMap<String, String>();

  contentMap.put("Message state", message.getMsgState().toString());
  contentMap.put("AdditionalInfo", additionalInfo);
  contentMap.put("Class", classInfo.toString());
  contentMap.put("MSG_ID", message.getMsgId());

  result.setMapField(generateMapFieldId(level, getRecordIdForMessage(message)), contentMap);

  return result;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
private ZNRecord getTestInstanceZNRecord() {
  ZNRecord record = new ZNRecord("Server_localhost_1234");
  Map<String, String> groupIdMap = new HashMap<>();
  Map<String, String> partitionMap = new HashMap<>();

  for (int i = 0; i < 10; ++i) {
    groupIdMap.put("testRes" + i + "_REALTIME", "groupId" + i);
    partitionMap.put("testRes" + i + "_REALTIME", "part" + i);
  }
  record.setMapField("KAFKA_HLC_GROUP_MAP", groupIdMap);
  record.setMapField("KAFKA_HLC_PARTITION_MAP", partitionMap);
  return record;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
public ZNRecord computeRoutingTable(List<String> instanceNames, int partitions, int replicas,
    String dbName, long randomSeed) {
  assert (instanceNames.size() > replicas);
  Collections.sort(instanceNames);

  ZNRecord result = new ZNRecord(dbName);

  Map<String, Object> externalView = new TreeMap<String, Object>();

  List<Integer> partitionList = new ArrayList<Integer>(partitions);
  for (int i = 0; i < partitions; i++) {
    partitionList.add(new Integer(i));
  }
  Random rand = new Random(randomSeed);
  // Shuffle the partition list
  Collections.shuffle(partitionList, rand);

  for (int i = 0; i < partitionList.size(); i++) {
    int partitionId = partitionList.get(i);
    Map<String, String> partitionAssignment = new TreeMap<String, String>();
    int masterNode = i % instanceNames.size();
    // the first in the list is the node that contains the master
    partitionAssignment.put(instanceNames.get(masterNode), "MASTER");

    // for the jth replica, we put it on (masterNode + j) % nodes-th
    // node
    for (int j = 1; j <= replicas; j++) {
      partitionAssignment
          .put(instanceNames.get((masterNode + j) % instanceNames.size()), "SLAVE");
    }
    String partitionName = dbName + ".partition-" + partitionId;
    result.setMapField(partitionName, partitionAssignment);
  }
  result.setSimpleField(IdealStateProperty.NUM_PARTITIONS.toString(), "" + partitions);
  return result;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
@Override
public ZNRecord toZNRecord() {
  ZNRecord znRecord = new ZNRecord(_segmentName);
  znRecord.setSimpleField(CommonConstants.Segment.SEGMENT_NAME, _segmentName);
  znRecord.setSimpleField(CommonConstants.Segment.TABLE_NAME, _tableName);
  znRecord.setEnumField(CommonConstants.Segment.SEGMENT_TYPE, _segmentType);
  if (_timeUnit == null) {
    znRecord.setSimpleField(CommonConstants.Segment.TIME_UNIT, NULL);
  } else {
    znRecord.setEnumField(CommonConstants.Segment.TIME_UNIT, _timeUnit);
  }
  znRecord.setLongField(CommonConstants.Segment.START_TIME, _startTime);
  znRecord.setLongField(CommonConstants.Segment.END_TIME, _endTime);

  znRecord.setSimpleField(CommonConstants.Segment.INDEX_VERSION, _indexVersion);
  znRecord.setLongField(CommonConstants.Segment.TOTAL_DOCS, _totalRawDocs);
  znRecord.setLongField(CommonConstants.Segment.CRC, _crc);
  znRecord.setLongField(CommonConstants.Segment.CREATION_TIME, _creationTime);

  if (_partitionMetadata != null) {
    try {
      String partitionMetadataJson = _partitionMetadata.toJsonString();
      znRecord.setSimpleField(CommonConstants.Segment.PARTITION_METADATA, partitionMetadataJson);
    } catch (IOException e) {
      LOGGER.error(
          "Exception caught while writing partition metadata into ZNRecord for segment '{}', will be dropped",
          _segmentName, e);
    }
  }
  if (_segmentUploadStartTime > 0) {
    znRecord.setLongField(CommonConstants.Segment.SEGMENT_UPLOAD_START_TIME, _segmentUploadStartTime);
  }
  if (_customMap != null) {
    znRecord.setMapField(CommonConstants.Segment.CUSTOM_MAP, _customMap);
  }
  return znRecord;
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
@Test
public void testBasicCompression() {
  ZNRecord record = new ZNRecord("testId");
  int numPartitions = 1024;
  int replicas = 3;
  int numNodes = 100;
  Random random = new Random();
  for (int p = 0; p < numPartitions; p++) {
    Map<String, String> map = new HashMap<String, String>();
    for (int r = 0; r < replicas; r++) {
      map.put("host_" + random.nextInt(numNodes), "ONLINE");
    }
    record.setMapField("TestResource_" + p, map);
  }
  ZNRecordSerializer serializer = new ZNRecordSerializer();
  byte[] serializedBytes;
  serializedBytes = serializer.serialize(record);
  int uncompressedSize = serializedBytes.length;
  System.out.println("raw serialized data length = " + serializedBytes.length);
  record.setSimpleField("enableCompression", "true");
  serializedBytes = serializer.serialize(record);
  int compressedSize = serializedBytes.length;
  System.out.println("compressed serialized data length = " + serializedBytes.length);
  System.out.printf("compression ratio: %.2f \n", (uncompressedSize * 1.0 / compressedSize));
  ZNRecord result = (ZNRecord) serializer.deserialize(serializedBytes);
  Assert.assertEquals(result, record);
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
@Test
public void testCompression() {
  int runId = 1;
  while (runId < 20) {
    int numPartitions = runId * 1000;
    int replicas = 3;
    int numNodes = 100;
    Random random = new Random();
    ZNRecord record = new ZNRecord("testId");
    System.out.println("Partitions:" + numPartitions);
    for (int p = 0; p < numPartitions; p++) {
      Map<String, String> map = new HashMap<String, String>();
      for (int r = 0; r < replicas; r++) {
        map.put("host_" + random.nextInt(numNodes), "ONLINE");
      }
      record.setMapField("TestResource_" + p, map);
    }
    ZNRecordSerializer serializer = new ZNRecordSerializer();
    byte[] serializedBytes;
    record.setSimpleField("enableCompression", "true");
    serializedBytes = serializer.serialize(record);
    int compressedSize = serializedBytes.length;
    System.out.println("compressed serialized data length = " + compressedSize);
    ZNRecord result = (ZNRecord) serializer.deserialize(serializedBytes);
    Assert.assertEquals(result, record);
    runId = runId + 1;
  }
}
 

import org.apache.helix.ZNRecord; //导入方法依赖的package包/类
/**
 * Test the normal case of serialize/deserialize where ZNRecord is well-formed
 */
@Test
public void basicTest() {
  ZNRecord record = new ZNRecord("testId");
  record.setMapField("k1", ImmutableMap.of("a", "b", "c", "d"));
  record.setMapField("k2", ImmutableMap.of("e", "f", "g", "h"));
  record.setListField("k3", ImmutableList.of("a", "b", "c", "d"));
  record.setListField("k4", ImmutableList.of("d", "e", "f", "g"));
  record.setSimpleField("k5", "a");
  record.setSimpleField("k5", "b");
  ZNRecordStreamingSerializer serializer = new ZNRecordStreamingSerializer();
  ZNRecord result = (ZNRecord) serializer.deserialize(serializer.serialize(record));
  Assert.assertEquals(result, record);
}