Java HelixAdmin.addStateModelDef方法代码示例

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
@Test
public void testDisableResource() {
  String className = TestHelper.getTestClassName();
  String methodName = TestHelper.getTestMethodName();
  String clusterName = className + "_" + methodName;
  System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
  HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
  admin.addCluster(clusterName, true);
  Assert.assertTrue(ZKUtil.isClusterSetup(clusterName, _gZkClient), "Cluster should be setup");
  String resourceName = "TestDB";
  admin.addStateModelDef(clusterName, "MasterSlave", new StateModelDefinition(
      StateModelConfigGenerator.generateConfigForMasterSlave()));
  admin.addResource(clusterName, resourceName, 4, "MasterSlave");
  admin.enableResource(clusterName, resourceName, false);
  BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
  HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
  PropertyKey.Builder keyBuilder = accessor.keyBuilder();
  IdealState idealState = accessor.getProperty(keyBuilder.idealStates(resourceName));
  Assert.assertFalse(idealState.isEnabled());
  admin.enableResource(clusterName, resourceName, true);
  idealState = accessor.getProperty(keyBuilder.idealStates(resourceName));
  Assert.assertTrue(idealState.isEnabled());
  System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
public static void createHelixClusterIfNeeded(String helixClusterName, String zkPath) {
  final HelixAdmin admin = new ZKHelixAdmin(zkPath);

  if (admin.getClusters().contains(helixClusterName)) {
    LOGGER.info(
        "cluster already exist, skipping it.. ********************************************* ");
    return;
  }

  LOGGER.info("Creating a new cluster, as the helix cluster : " + helixClusterName
      + " was not found ********************************************* ");
  admin.addCluster(helixClusterName, false);

  LOGGER.info("Enable mirror maker machines auto join.");
  final HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER)
      .forCluster(helixClusterName).build();

  final Map<String, String> props = new HashMap<String, String>();
  props.put(ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, String.valueOf(true));
  props.put(MessageType.STATE_TRANSITION + "." + HelixTaskExecutor.MAX_THREADS,
      String.valueOf(100));

  admin.setConfig(scope, props);

  LOGGER.info("Adding state model definition named : OnlineOffline generated using : "
      + OnlineOfflineStateModel.class.toString()
      + " ********************************************** ");

  // add state model definition
  admin.addStateModelDef(helixClusterName, "OnlineOffline", OnlineOfflineStateModel.build());
  LOGGER.info("New Cluster setup completed... ********************************************** ");
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
void startAdmin() throws Exception {
  HelixAdmin admin = new ZKHelixAdmin(ZK_ADDR);

  // create cluster
  System.out.println("Creating cluster: " + clusterName);
  admin.addCluster(clusterName, true);

  // add MasterSlave state mode definition
  admin.addStateModelDef(clusterName, "MasterSlave", new StateModelDefinition(
      generateConfigForMasterSlave()));

  // ideal-state znrecord
  ZNRecord record = new ZNRecord(resourceName);
  record.setSimpleField("IDEAL_STATE_MODE", "AUTO");
  record.setSimpleField("NUM_PARTITIONS", "1");
  record.setSimpleField("REPLICAS", "2");
  record.setSimpleField("STATE_MODEL_DEF_REF", "MasterSlave");
  record.setListField(resourceName, Arrays.asList("node1", "node2"));

  admin.setResourceIdealState(clusterName, resourceName, new IdealState(record));

  ConstraintItemBuilder builder = new ConstraintItemBuilder();

  // limit one transition message at a time across the entire cluster
  builder.addConstraintAttribute("MESSAGE_TYPE", "STATE_TRANSITION")
  // .addConstraintAttribute("INSTANCE", ".*") // un-comment this line if using instance-level
  // constraint
      .addConstraintAttribute("CONSTRAINT_VALUE", "1");
  admin.setConstraint(clusterName, ClusterConstraints.ConstraintType.MESSAGE_CONSTRAINT,
      "constraint1", builder.build());
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
@Test
public void testDropResource() {
  String className = TestHelper.getTestClassName();
  String methodName = TestHelper.getTestMethodName();
  String clusterName = className + "_" + methodName;

  System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));

  HelixAdmin tool = new ZKHelixAdmin(_gZkClient);
  tool.addCluster(clusterName, true);
  Assert.assertTrue(ZKUtil.isClusterSetup(clusterName, _gZkClient), "Cluster should be setup");

  tool.addStateModelDef(clusterName, "MasterSlave", new StateModelDefinition(
      StateModelConfigGenerator.generateConfigForMasterSlave()));
  tool.addResource(clusterName, "test-db", 4, "MasterSlave");
  Map<String, String> resourceConfig = new HashMap<String, String>();
  resourceConfig.put("key1", "value1");
  tool.setConfig(new HelixConfigScopeBuilder(ConfigScopeProperty.RESOURCE)
      .forCluster(clusterName).forResource("test-db").build(), resourceConfig);

  PropertyKey.Builder keyBuilder = new PropertyKey.Builder(clusterName);
  Assert.assertTrue(_gZkClient.exists(keyBuilder.idealStates("test-db").getPath()),
      "test-db ideal-state should exist");
  Assert.assertTrue(_gZkClient.exists(keyBuilder.resourceConfig("test-db").getPath()),
      "test-db resource config should exist");

  tool.dropResource(clusterName, "test-db");
  Assert.assertFalse(_gZkClient.exists(keyBuilder.idealStates("test-db").getPath()),
      "test-db ideal-state should be dropped");
  Assert.assertFalse(_gZkClient.exists(keyBuilder.resourceConfig("test-db").getPath()),
      "test-db resource config should be dropped");

  System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
/**
 * Initialize a map of dataCenter to HelixAdmin based on the given zk Connect Strings.
 * @param helixAdminFactory the {@link HelixAdminFactory} to use to instantiate {@link HelixAdmin}
 */
private void initializeAdminsAndAddCluster(HelixAdminFactory helixAdminFactory) {
  for (Map.Entry<String, ClusterMapUtils.DcZkInfo> entry : dataCenterToZkAddress.entrySet()) {
    HelixAdmin admin = helixAdminFactory.getHelixAdmin(entry.getValue().getZkConnectStr());
    adminForDc.put(entry.getKey(), admin);
    // Add a cluster entry in every DC
    if (!admin.getClusters().contains(clusterName)) {
      admin.addCluster(clusterName);
      admin.addStateModelDef(clusterName, LeaderStandbySMD.name, LeaderStandbySMD.build());
    }
  }
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
public static void createHelixClusterIfNeeded(String helixClusterName, String zkPath) {
  final HelixAdmin admin = new ZKHelixAdmin(zkPath);

  if (admin.getClusters().contains(helixClusterName)) {
    LOGGER.info("cluster already exist, skipping it.. ********************************************* ");
    return;
  }

  LOGGER.info("Creating a new cluster, as the helix cluster : " + helixClusterName
      + " was not found ********************************************* ");
  admin.addCluster(helixClusterName, false);

  LOGGER.info("Enable auto join.");
  final HelixConfigScope scope =
      new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER).forCluster(helixClusterName).build();

  final Map<String, String> props = new HashMap<String, String>();
  props.put(ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, String.valueOf(true));
  //we need only one segment to be loaded at a time
  props.put(MessageType.STATE_TRANSITION + "." + HelixTaskExecutor.MAX_THREADS, String.valueOf(1));

  admin.setConfig(scope, props);

  LOGGER.info("Adding state model definition named : "
      + PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL
      + " generated using : " + PinotHelixSegmentOnlineOfflineStateModelGenerator.class.toString()
      + " ********************************************** ");

  admin.addStateModelDef(helixClusterName,
      PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL,
      PinotHelixSegmentOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition());

  LOGGER.info("Adding state model definition named : "
      + PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.PINOT_BROKER_RESOURCE_ONLINE_OFFLINE_STATE_MODEL
      + " generated using : " + PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.class.toString()
      + " ********************************************** ");

  admin.addStateModelDef(helixClusterName,
      PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.PINOT_BROKER_RESOURCE_ONLINE_OFFLINE_STATE_MODEL,
      PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition());

  LOGGER.info("Adding empty ideal state for Broker!");
  HelixHelper.updateResourceConfigsFor(new HashMap<String, String>(), CommonConstants.Helix.BROKER_RESOURCE_INSTANCE,
      helixClusterName, admin);
  IdealState idealState =
      PinotTableIdealStateBuilder.buildEmptyIdealStateForBrokerResource(admin, helixClusterName);
  admin.setResourceIdealState(helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, idealState);
  LOGGER.info("New Cluster setup completed... ********************************************** ");
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
void testInvalidReplica2() throws Exception {
  HelixAdmin admin = new ZKHelixAdmin(ZK_ADDR);

  // create cluster
  String className = TestHelper.getTestClassName();
  String methodName = TestHelper.getTestMethodName();
  String clusterName = className + "_" + methodName;
  String db = "TestDB";

  System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));

  // System.out.println("Creating cluster: " + clusterName);
  admin.addCluster(clusterName, true);

  // add MasterSlave state mode definition
  admin.addStateModelDef(clusterName, "MasterSlave", new StateModelDefinition(
      StateModelConfigGenerator.generateConfigForMasterSlave()));

  // Add nodes to the cluster
  int n = 3;
  System.out.println("Adding " + n + " participants to the cluster");
  for (int i = 0; i < n; i++) {
    int port = 12918 + i;
    InstanceConfig instanceConfig = new InstanceConfig("localhost_" + port);
    instanceConfig.setHostName("localhost");
    instanceConfig.setPort("" + port);
    instanceConfig.setInstanceEnabled(true);
    admin.addInstance(clusterName, instanceConfig);
    // System.out.println("\t Added participant: " + instanceConfig.getInstanceName());
  }

  // construct ideal-state manually
  IdealState idealState = new IdealState(db);
  idealState.setRebalanceMode(RebalanceMode.SEMI_AUTO);
  idealState.setNumPartitions(2);
  idealState.setReplicas("" + 2); // should be 3
  idealState.setStateModelDefRef("MasterSlave");
  idealState.getRecord().setListField("TestDB_0",
      Arrays.asList("localhost_12918", "localhost_12919", "localhost_12920"));
  idealState.getRecord().setListField("TestDB_1",
      Arrays.asList("localhost_12919", "localhost_12918", "localhost_12920"));

  admin.setResourceIdealState(clusterName, "TestDB", idealState);

  // start participants
  MockParticipantManager[] participants = new MockParticipantManager[n];
  for (int i = 0; i < n; i++) {
    String instanceName = "localhost_" + (12918 + i);

    participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
    participants[i].syncStart();
  }

  ClusterControllerManager controller =
      new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
  controller.syncStart();

  boolean result =
      ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
          clusterName));
  Assert.assertTrue(result);

  // make sure localhost_12919 is master on TestDB_1
  HelixDataAccessor accessor = controller.getHelixDataAccessor();
  Builder keyBuilder = accessor.keyBuilder();
  ExternalView extView = accessor.getProperty(keyBuilder.externalView(db));
  Map<String, String> stateMap = extView.getStateMap(db + "_1");
  Assert
      .assertEquals(
          stateMap.get("localhost_12919"),
          "MASTER",
          "localhost_12919 should be MASTER even though replicas is set to 2, since we generate message based on target-state priority");

  System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
@Test
public void testGetResourcesWithTag() {
  String TEST_TAG = "TestTAG";

  final String clusterName = getShortClassName();
  String rootPath = "/" + clusterName;
  if (_gZkClient.exists(rootPath)) {
    _gZkClient.deleteRecursive(rootPath);
  }

  HelixAdmin tool = new ZKHelixAdmin(_gZkClient);
  tool.addCluster(clusterName, true);
  Assert.assertTrue(ZKUtil.isClusterSetup(clusterName, _gZkClient));

  tool.addStateModelDef(clusterName, "OnlineOffline",
      new StateModelDefinition(StateModelConfigGenerator.generateConfigForOnlineOffline()));

  for (int i = 0; i < 4; i++) {
    String instanceName = "host" + i + "_9999";
    InstanceConfig config = new InstanceConfig(instanceName);
    config.setHostName("host" + i);
    config.setPort("9999");
    // set tag to two instances
    if (i < 2) {
      config.addTag(TEST_TAG);
    }
    tool.addInstance(clusterName, config);
    tool.enableInstance(clusterName, instanceName, true);
    String path = PropertyPathBuilder.instance(clusterName, instanceName);
    AssertJUnit.assertTrue(_gZkClient.exists(path));
  }

  for (int i = 0; i < 4; i++) {
    String resourceName = "database_" + i;
    IdealState is = new IdealState(resourceName);
    is.setStateModelDefRef("OnlineOffline");
    is.setNumPartitions(2);
    is.setRebalanceMode(IdealState.RebalanceMode.FULL_AUTO);
    is.setReplicas("1");
    is.enable(true);
    if (i < 2) {
      is.setInstanceGroupTag(TEST_TAG);
    }
    tool.addResource(clusterName, resourceName, is);
  }

  List<String> allResources = tool.getResourcesInCluster(clusterName);
  List<String> resourcesWithTag = tool.getResourcesInClusterWithTag(clusterName, TEST_TAG);
  AssertJUnit.assertEquals(allResources.size(), 4);
  AssertJUnit.assertEquals(resourcesWithTag.size(), 2);
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
@Test
public void test() throws Exception {
  String className = TestHelper.getTestClassName();
  String methodName = TestHelper.getTestMethodName();
  String clusterName = className + "_" + methodName;

  System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
  HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
  admin.addCluster(clusterName);
  admin.addStateModelDef(clusterName, BuiltInStateModelDefinitions.MasterSlave.getStateModelDefinition().getId(),
                         BuiltInStateModelDefinitions.MasterSlave.getStateModelDefinition());
  ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName);
  controller.syncStart();

  // controller shall create all built-in state model definitions
  final BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
  final PropertyKey.Builder keyBuilder = new PropertyKey.Builder(clusterName);
  boolean ret = TestHelper.verify(new TestHelper.Verifier() {

    @Override
    public boolean verify() throws Exception {
      for (BuiltInStateModelDefinitions def : BuiltInStateModelDefinitions.values()) {
        String path = keyBuilder.stateModelDef(def.getStateModelDefinition().getId()).getPath();
        boolean exist = baseAccessor.exists(path, 0);
        if (!exist) {
          return false;
        }

        // make sure MasterSlave is not over-written
        if (def == BuiltInStateModelDefinitions.MasterSlave) {
          Stat stat = new Stat();
          baseAccessor.get(path, stat, 0);
          if (stat.getVersion() != 0) {
            return false;
          }
        }
      }
      return true;
    }
  }, 10 * 1000);
  Assert.assertTrue(ret);
  controller.syncStop();
  System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
/**
 * LockManagerDemo clusterName, numInstances, lockGroupName, numLocks
 * @param args
 * @throws Exception
 */
public static void main(String[] args) throws Exception {
  final String zkAddress = "localhost:2199";
  final String clusterName = "lock-manager-demo";
  final String lockGroupName = "lock-group";
  final int numInstances = 3;
  final int numPartitions = 12;
  final boolean startController = false;
  HelixManager controllerManager = null;
  Thread[] processArray;
  processArray = new Thread[numInstances];
  try {
    startLocalZookeeper(2199);
    HelixAdmin admin = new ZKHelixAdmin(zkAddress);
    admin.addCluster(clusterName, true);
    StateModelConfigGenerator generator = new StateModelConfigGenerator();
    admin.addStateModelDef(clusterName, "OnlineOffline",
        new StateModelDefinition(generator.generateConfigForOnlineOffline()));
    admin.addResource(clusterName, lockGroupName, numPartitions, "OnlineOffline",
        RebalanceMode.FULL_AUTO.toString());
    admin.rebalance(clusterName, lockGroupName, 1);
    for (int i = 0; i < numInstances; i++) {
      final String instanceName = "localhost_" + (12000 + i);
      processArray[i] = new Thread(new Runnable() {

        @Override
        public void run() {
          LockProcess lockProcess = null;

          try {
            lockProcess = new LockProcess(clusterName, zkAddress, instanceName, startController);
            lockProcess.start();
            Thread.currentThread().join();
          } catch (InterruptedException e) {
            System.out.println(instanceName + "Interrupted");
            if (lockProcess != null) {
              lockProcess.stop();
            }
          } catch (Exception e) {
            e.printStackTrace();
          }
        }

      });
      processArray[i].start();
    }
    Thread.sleep(3000);
    controllerManager =
        HelixControllerMain.startHelixController(zkAddress, clusterName, "controller",
            HelixControllerMain.STANDALONE);
    Thread.sleep(5000);
    printStatus(admin, clusterName, lockGroupName);
    System.out.println("Stopping localhost_12000");
    processArray[0].interrupt();
    Thread.sleep(3000);
    printStatus(admin, clusterName, lockGroupName);
    Thread.currentThread().join();
  } catch (Exception e) {
    e.printStackTrace();
  } finally {
    if (controllerManager != null) {
      controllerManager.disconnect();
    }
    for (Thread process : processArray) {
      if (process != null) {
        process.interrupt();
      }
    }
  }
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
public static void createHelixClusterIfNeeded(String helixClusterName, String zkPath, boolean isUpdateStateModel) {
  final HelixAdmin admin = new ZKHelixAdmin(zkPath);
  final String segmentStateModelName = PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL;

  if (admin.getClusters().contains(helixClusterName)) {
    LOGGER.info("cluster already exists ********************************************* ");
    if (isUpdateStateModel) {
      final StateModelDefinition curStateModelDef = admin.getStateModelDef(helixClusterName, segmentStateModelName);
      List<String> states = curStateModelDef.getStatesPriorityList();
      if (states.contains(PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE)) {
        LOGGER.info("State model {} already updated to contain CONSUMING state", segmentStateModelName);
        return;
      } else {
        LOGGER.info("Updating {} to add states for low level kafka consumers", segmentStateModelName);
        StateModelDefinition newStateModelDef = PinotHelixSegmentOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition();
        ZkClient zkClient = new ZkClient(zkPath);
        zkClient.waitUntilConnected(20, TimeUnit.SECONDS);
        zkClient.setZkSerializer(new ZNRecordSerializer());
        HelixDataAccessor accessor = new ZKHelixDataAccessor(helixClusterName, new ZkBaseDataAccessor<ZNRecord>(zkClient));
        PropertyKey.Builder keyBuilder = accessor.keyBuilder();
        accessor.setProperty(keyBuilder.stateModelDef(segmentStateModelName), newStateModelDef);
        LOGGER.info("Completed updating statemodel {}", segmentStateModelName);
        zkClient.close();
      }
    }
    return;
  }

  LOGGER.info("Creating a new cluster, as the helix cluster : " + helixClusterName
      + " was not found ********************************************* ");
  admin.addCluster(helixClusterName, false);

  LOGGER.info("Enable auto join.");
  final HelixConfigScope scope =
      new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER).forCluster(helixClusterName).build();

  final Map<String, String> props = new HashMap<String, String>();
  props.put(ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, String.valueOf(true));
  //we need only one segment to be loaded at a time
  props.put(MessageType.STATE_TRANSITION + "." + HelixTaskExecutor.MAX_THREADS, String.valueOf(1));

  admin.setConfig(scope, props);

  LOGGER.info("Adding state model {} (with CONSUMED state) generated using {} **********************************************",
      segmentStateModelName , PinotHelixSegmentOnlineOfflineStateModelGenerator.class.toString());

  // If this is a fresh cluster we are creating, then the cluster will see the CONSUMING state in the
  // state model. But then the servers will never be asked to go to that STATE (whether they have the code
  // to handle it or not) unil we complete the feature using low-level kafka consumers and turn the feature on.
  admin.addStateModelDef(helixClusterName, segmentStateModelName,
      PinotHelixSegmentOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition());

  LOGGER.info("Adding state model definition named : "
      + PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.PINOT_BROKER_RESOURCE_ONLINE_OFFLINE_STATE_MODEL
      + " generated using : " + PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.class.toString()
      + " ********************************************** ");

  admin.addStateModelDef(helixClusterName,
      PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.PINOT_BROKER_RESOURCE_ONLINE_OFFLINE_STATE_MODEL,
      PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition());

  LOGGER.info("Adding empty ideal state for Broker!");
  HelixHelper.updateResourceConfigsFor(new HashMap<String, String>(), CommonConstants.Helix.BROKER_RESOURCE_INSTANCE,
      helixClusterName, admin);
  IdealState idealState =
      PinotTableIdealStateBuilder.buildEmptyIdealStateForBrokerResource(admin, helixClusterName);
  admin.setResourceIdealState(helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, idealState);
  initPropertyStorePath(helixClusterName, zkPath);
  LOGGER.info("New Cluster setup completed... ********************************************** ");
}