Java RunId类代码示例

import org.apache.twill.api.RunId; //导入依赖的package包/类
@Test
public void testStopCluster() throws Exception {
  assumeNonMaprProfile();

  Cluster myCluster = createCluster();
  myCluster.setState(ClusterState.RUNNING);

  YarnController controller = Mockito.mock(YarnController.class);
  YarnService yarnService = new YarnService(new TestListener(), controller, Mockito.mock(NodeProvider.class));

  TwillController twillController = Mockito.mock(TwillController.class);
  RunId runId = RunIds.generate();

  when(controller.startCluster(any(YarnConfiguration.class), eq(myCluster.getClusterConfig().getSubPropertyList())))
    .thenReturn(twillController);
  when(twillController.getRunId()).thenReturn(runId);

  myCluster.setRunId(new com.dremio.provision.RunId(runId.getId()));
  yarnService.stopCluster(myCluster);

  assertEquals(ClusterState.STOPPED, myCluster.getState());
}
 

import org.apache.twill.api.RunId; //导入依赖的package包/类
public ApplicationMasterService(RunId runId, ZKClient zkClient,
                                TwillRuntimeSpecification twillRuntimeSpec, YarnAMClient amClient,
                                Configuration config, Location applicationLocation) throws Exception {
  super(zkClient, runId, config, applicationLocation);

  this.runId = runId;
  this.twillRuntimeSpec = twillRuntimeSpec;
  this.zkClient = zkClient;
  this.applicationLocation = applicationLocation;
  this.amClient = amClient;
  this.credentials = createCredentials();
  this.jvmOpts = loadJvmOptions();
  this.twillSpec = twillRuntimeSpec.getTwillSpecification();
  this.placementPolicyManager = new PlacementPolicyManager(twillSpec.getPlacementPolicies());
  this.environments = getEnvironments();

  this.amLiveNode = new ApplicationMasterLiveNodeData(Integer.parseInt(System.getenv(EnvKeys.YARN_APP_ID)),
                                                      Long.parseLong(System.getenv(EnvKeys.YARN_APP_ID_CLUSTER_TIME)),
                                                      amClient.getContainerId().toString(), getLocalizeFiles(),
                                                      twillRuntimeSpec.getKafkaZKConnect());

  this.expectedContainers = new ExpectedContainers(twillSpec);
  this.eventHandler = createEventHandler(twillSpec);
  this.runningContainers = createRunningContainers(amClient.getContainerId(), amClient.getHost());
}
 

import org.apache.twill.api.RunId; //导入依赖的package包/类
/**
 * Renews the {@link SecureStore} for all the running applications.
 *
 * @param liveApps set of running applications that need to have secure store renewal
 * @param renewer the {@link SecureStoreRenewer} for renewal
 * @param mergeCredentials {@code true} to merge with existing credentials
 * @return a {@link Multimap} containing the application runs that were failed to have secure store renewed
 */
private Multimap<String, RunId> renewSecureStore(Table<String, RunId, YarnTwillController> liveApps,
                                                 SecureStoreRenewer renewer, boolean mergeCredentials) {
  Multimap<String, RunId> failureRenews = HashMultimap.create();

  // Renew the secure store for each running application
  for (Table.Cell<String, RunId, YarnTwillController> liveApp : liveApps.cellSet()) {
    String application = liveApp.getRowKey();
    RunId runId = liveApp.getColumnKey();
    YarnTwillController controller = liveApp.getValue();

    try {
      renewer.renew(application, runId, new YarnSecureStoreWriter(application, runId, controller, mergeCredentials));
    } catch (Exception e) {
      LOG.warn("Failed to renew secure store for {}:{}", application, runId, e);
      failureRenews.put(application, runId);
    }
  }

  return failureRenews;
}
 

import org.apache.twill.api.RunId; //导入依赖的package包/类
/**
 * Creates an instance with an existing {@link ApplicationMasterLiveNodeData}.
 */
YarnTwillController(String appName, RunId runId, ZKClient zkClient,
                    final ApplicationMasterLiveNodeData amLiveNodeData, final YarnAppClient yarnAppClient) {
  super(appName, runId, zkClient, amLiveNodeData.getKafkaZKConnect() != null, Collections.<LogHandler>emptyList());
  this.appName = appName;
  this.amLiveNodeData = amLiveNodeData;
  this.startUp = new Callable<ProcessController<YarnApplicationReport>>() {
    @Override
    public ProcessController<YarnApplicationReport> call() throws Exception {
      return yarnAppClient.createProcessController(
        YarnUtils.createApplicationId(amLiveNodeData.getAppIdClusterTime(), amLiveNodeData.getAppId()));
    }
  };
  this.startTimeout = Constants.APPLICATION_MAX_START_SECONDS;
  this.startTimeoutUnit = TimeUnit.SECONDS;
}
 

import org.apache.twill.api.RunId; //导入依赖的package包/类
public TwillRuntimeSpecification(TwillSpecification twillSpecification, String fsUser, URI twillAppDir,
                                 String zkConnectStr, RunId twillRunId, String twillAppName,
                                 @Nullable String rmSchedulerAddr, Map<String, Map<String, String>> logLevels,
                                 Map<String, Integer> maxRetries, Map<String, String> config,
                                 Map<String, Map<String, String>> runnableConfigs) {
  this.twillSpecification = twillSpecification;
  this.fsUser = fsUser;
  this.twillAppDir = twillAppDir;
  this.zkConnectStr = zkConnectStr;
  this.twillRunId = twillRunId;
  this.twillAppName = twillAppName;
  this.rmSchedulerAddr = rmSchedulerAddr;
  this.logLevels = logLevels;
  this.maxRetries = maxRetries;
  this.config = config;
  this.runnableConfigs = runnableConfigs;
}
 

import org.apache.twill.api.RunId; //导入依赖的package包/类
public AbstractTwillController(String appName, RunId runId, ZKClient zkClient, boolean logCollectionEnabled,
                               Iterable<LogHandler> logHandlers) {
  super(runId, zkClient);
  this.appName = appName;
  this.runId = runId;
  this.logHandlers = new ConcurrentLinkedQueue<>();

  // When addressing TWILL-147, need to check if the given ZKClient is
  // actually used by the Kafka used for log collection
  if (logCollectionEnabled) {
    this.kafkaClient = new ZKKafkaClientService(ZKClients.namespace(zkClient, "/" + runId.getId() + "/kafka"));
    Iterables.addAll(this.logHandlers, logHandlers);
  } else {
    this.kafkaClient = null;
    if (!Iterables.isEmpty(logHandlers)) {
      LOG.warn("Log collection is disabled for application {} with runId {}. " +
                 "Adding log handler won't get any logs.", appName, runId);
    }
  }
}
 

import org.apache.twill.api.RunId; //导入依赖的package包/类
public BasicTwillContext(RunId runId, RunId appRunId, InetAddress host, String[] args, String[] appArgs,
                         TwillRunnableSpecification spec, int instanceId,
                         DiscoveryService discoveryService, DiscoveryServiceClient discoveryServiceClient,
                         ZKClient zkClient,
                         int instanceCount, int allowedMemoryMB, int virtualCores) {
  this.runId = runId;
  this.appRunId = appRunId;
  this.host = host;
  this.args = args;
  this.appArgs = appArgs;
  this.spec = spec;
  this.instanceId = instanceId;
  this.discoveryService = discoveryService;
  this.discoveryServiceClient = discoveryServiceClient;
  this.zkClient = zkClient;
  this.elections = new ElectionRegistry(zkClient);
  this.instanceCount = instanceCount;
  this.allowedMemoryMB = allowedMemoryMB;
  this.virtualCores = virtualCores;
}
 

import org.apache.twill.api.RunId; //导入依赖的package包/类
private Service createService(ZKClient zkClient, RunId runId) {
  return new AbstractTwillService(zkClient, runId) {

    private final CountDownLatch stopLatch = new CountDownLatch(1);

    @Override
    protected void doStart() throws Exception {
      LOG.info("Start");
    }

    @Override
    protected void doRun() throws Exception {
      stopLatch.await();
    }

    @Override
    protected void doStop() throws Exception {
      LOG.info("Stop");
    }

    @Override
    protected void triggerShutdown() {
      stopLatch.countDown();
    }
  };
}
 

import org.apache.twill.api.RunId; //导入依赖的package包/类
/**
 * Starts the application master.
 */
public static void main(String[] args) throws Exception {
  String zkConnect = System.getenv(EnvKeys.TWILL_ZK_CONNECT);
  File twillSpec = new File(Constants.Files.TWILL_SPEC);
  RunId runId = RunIds.fromString(System.getenv(EnvKeys.TWILL_RUN_ID));

  ZKClientService zkClientService =
    ZKClientServices.delegate(
      ZKClients.reWatchOnExpire(
        ZKClients.retryOnFailure(
          ZKClientService.Builder.of(zkConnect).build(),
          RetryStrategies.fixDelay(1, TimeUnit.SECONDS))));

  Configuration conf = new YarnConfiguration(new HdfsConfiguration(new Configuration()));
  setRMSchedulerAddress(conf);
  Service service = new ApplicationMasterService(runId, zkClientService, twillSpec,
                                                 new VersionDetectYarnAMClientFactory(conf), createAppLocation(conf));
  new ApplicationMasterMain(String.format("%s/%s/kafka", zkConnect, runId.getId())).doMain(zkClientService, service);
}
 

import org.apache.twill.api.RunId; //导入依赖的package包/类
void start(String runnableName, ContainerInfo containerInfo, TwillContainerLauncher launcher) {
  containerLock.lock();
  try {
    int instanceId = getStartInstanceId(runnableName);
    RunId runId = getRunId(runnableName, instanceId);
    TwillContainerController controller = launcher.start(runId, instanceId,
                                                         TwillContainerMain.class, "$HADOOP_CONF_DIR");
    containers.put(runnableName, containerInfo.getId(), controller);
    TwillRunResources resources = new DynamicTwillRunResources(instanceId,
                                                               containerInfo.getId(),
                                                               containerInfo.getVirtualCores(),
                                                               containerInfo.getMemoryMB(),
                                                               containerInfo.getHost().getHostName(),
                                                               controller);
    resourceReport.addRunResources(runnableName, resources);
    containerStats.put(runnableName, containerInfo);

    if (startSequence.isEmpty() || !runnableName.equals(startSequence.peekLast())) {
      startSequence.addLast(runnableName);
    }
    containerChange.signalAll();

  } finally {
    containerLock.unlock();
  }
}
 

import org.apache.twill.api.RunId; //导入依赖的package包/类
@Override
public TwillPreparer prepare(TwillApplication application) {
  Preconditions.checkState(isRunning(), "Service not start. Please call start() first.");
  final TwillSpecification twillSpec = application.configure();
  final String appName = twillSpec.getName();

  return new YarnTwillPreparer(yarnConfig, twillSpec, yarnAppClient, zkClientService, locationFactory, jvmOptions,
                               new YarnTwillControllerFactory() {
    @Override
    public YarnTwillController create(RunId runId, Iterable<LogHandler> logHandlers,
                                      Callable<ProcessController<YarnApplicationReport>> startUp) {
      ZKClient zkClient = ZKClients.namespace(zkClientService, "/" + appName);
      YarnTwillController controller = listenController(new YarnTwillController(appName, runId, zkClient,
                                                                                logHandlers, startUp));
      synchronized (YarnTwillRunnerService.this) {
        Preconditions.checkArgument(!controllers.contains(appName, runId),
                                    "Application %s with runId %s is already running.", appName, runId);
        controllers.put(appName, runId, controller);
      }
      return controller;
    }
  });
}
 

import org.apache.twill.api.RunId; //导入依赖的package包/类
public BasicTwillContext(RunId runId, RunId appRunId, InetAddress host, String[] args, String[] appArgs,
                         TwillRunnableSpecification spec, int instanceId,
                         DiscoveryService discoveryService, DiscoveryServiceClient discoveryServiceClient,
                         ElectionRegistry electionRegistry,
                         int instanceCount, int allowedMemoryMB, int virtualCores) {
  this.runId = runId;
  this.appRunId = appRunId;
  this.host = host;
  this.args = args;
  this.appArgs = appArgs;
  this.spec = spec;
  this.instanceId = instanceId;
  this.discoveryService = discoveryService;
  this.discoveryServiceClient = discoveryServiceClient;
  this.elections = electionRegistry;
  this.instanceCount = instanceCount;
  this.allowedMemoryMB = allowedMemoryMB;
  this.virtualCores = virtualCores;
}
 

import org.apache.twill.api.RunId; //导入依赖的package包/类
@Override
public boolean equals(Object other) {
  if (this == other) {
    return true;
  }
  if (other == null || !(other instanceof RunId)) {
    return false;
  }
  return id.equals(((RunId) other).getId());
}
 

import org.apache.twill.api.RunId; //导入依赖的package包/类
private void doMain() throws Exception {
  RunId runId = twillRuntimeSpec.getTwillAppRunId();

  ZKClientService zkClientService = createZKClient();
  Configuration conf = new YarnConfiguration(new HdfsConfiguration(new Configuration()));
  setRMSchedulerAddress(conf, twillRuntimeSpec.getRmSchedulerAddr());

  final YarnAMClient amClient = new VersionDetectYarnAMClientFactory(conf).create();
  ApplicationMasterService service =
    new ApplicationMasterService(runId, zkClientService, twillRuntimeSpec, amClient, conf,
                                 createAppLocation(conf, twillRuntimeSpec.getFsUser(),
                                                   twillRuntimeSpec.getTwillAppDir()));
  TrackerService trackerService = new TrackerService(service);

  List<Service> prerequisites = Lists.newArrayList(
    new YarnAMClientService(amClient, trackerService),
    zkClientService,
    new AppMasterTwillZKPathService(zkClientService, runId)
  );

  if (twillRuntimeSpec.isLogCollectionEnabled()) {
    prerequisites.add(new ApplicationKafkaService(zkClientService, twillRuntimeSpec.getKafkaZKConnect()));
  } else {
    LOG.info("Log collection through kafka disabled");
  }

  new ApplicationMasterMain(twillRuntimeSpec)
    .doMain(
      service,
      prerequisites.toArray(new Service[prerequisites.size()])
    );
}
 

import org.apache.twill.api.RunId; //导入依赖的package包/类
/**
 * Start a container for a runnable.
 */
void start(String runnableName, ContainerInfo containerInfo, TwillContainerLauncher launcher) {
  containerLock.lock();
  try {
    int instanceId = getStartInstanceId(runnableName);
    RunId runId = getRunId(runnableName, instanceId);
    TwillContainerController controller = launcher.start(runId, instanceId,
                                                         TwillContainerMain.class, "$HADOOP_CONF_DIR",
                                                         saveLogLevels());
    containers.put(runnableName, containerInfo.getId(), controller);
    TwillRunResources resources = new DynamicTwillRunResources(instanceId,
                                                               containerInfo.getId(),
                                                               containerInfo.getVirtualCores(),
                                                               containerInfo.getMemoryMB(),
                                                               launcher.getMaxHeapMemoryMB(),
                                                               containerInfo.getHost().getHostName(),
                                                               controller);
    resourceReport.addRunResources(runnableName, resources);
    containerStats.put(runnableName, containerInfo);

    if (startSequence.isEmpty() || !runnableName.equals(startSequence.peekLast())) {
      startSequence.addLast(runnableName);
    }
    containerChange.signalAll();       
    // call event handler containerLaunched. 
    eventHandler.containerLaunched(runnableName, instanceId, containerInfo.getId());
  } finally {
    containerLock.unlock();
  }
}