Java RuntimeSpecification类代码示例

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
RunningContainers(TwillRuntimeSpecification twillRuntimeSpec, String appId, TwillRunResources appMasterResources,
                  ZKClient zookeeperClient, Location applicationLocation,
                  Map<String, RuntimeSpecification> runnables,
                  EventHandler eventHandler) {
  containers = HashBasedTable.create();
  runnableInstances = Maps.newHashMap();
  completedContainerCount = Maps.newHashMap();
  startSequence = Lists.newLinkedList();
  containerLock = new ReentrantLock();
  containerChange = containerLock.newCondition();
  resourceReport = new DefaultResourceReport(appId, appMasterResources);
  zkClient = zookeeperClient;
  containerStats = HashMultimap.create();
  this.applicationLocation = applicationLocation;
  this.runnableNames = runnables.keySet();
  this.logLevels = new TreeMap<>();
  this.maxRetries = Maps.newHashMap(twillRuntimeSpec.getMaxRetries());
  this.numRetries = Maps.newHashMap();
  this.eventHandler = eventHandler;
}
 

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
@Override
public JsonElement serialize(TwillSpecification src, Type typeOfSrc, JsonSerializationContext context) {
  JsonObject json = new JsonObject();
  json.addProperty("name", src.getName());
  json.add("runnables", context.serialize(src.getRunnables(),
                                          new TypeToken<Map<String, RuntimeSpecification>>() { }.getType()));
  json.add("orders", context.serialize(src.getOrders(),
                                       new TypeToken<List<TwillSpecification.Order>>() { }.getType()));
  json.add("placementPolicies", context.serialize(
    src.getPlacementPolicies(), new TypeToken<List<TwillSpecification.PlacementPolicy>>() { }.getType()));
  EventHandlerSpecification eventHandler = src.getEventHandler();
  if (eventHandler != null) {
    json.add("handler", context.serialize(eventHandler, EventHandlerSpecification.class));
  }

  return json;
}
 

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
@Override
public TwillSpecification deserialize(JsonElement json, Type typeOfT,
                                      JsonDeserializationContext context) throws JsonParseException {
  JsonObject jsonObj = json.getAsJsonObject();

  String name = jsonObj.get("name").getAsString();
  Map<String, RuntimeSpecification> runnables = context.deserialize(
    jsonObj.get("runnables"), new TypeToken<Map<String, RuntimeSpecification>>() { }.getType());
  List<TwillSpecification.Order> orders = context.deserialize(
    jsonObj.get("orders"), new TypeToken<List<TwillSpecification.Order>>() { }.getType());
  List<TwillSpecification.PlacementPolicy> placementPolicies = context.deserialize(
    jsonObj.get("placementPolicies"), new TypeToken<List<TwillSpecification.PlacementPolicy>>() { }.getType());

  JsonElement handler = jsonObj.get("handler");
  EventHandlerSpecification eventHandler = null;
  if (handler != null && !handler.isJsonNull()) {
    eventHandler = context.deserialize(handler, EventHandlerSpecification.class);
  }

  return new DefaultTwillSpecification(name, runnables, orders, placementPolicies, eventHandler);
}
 

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
private TwillRuntimeSpecificationAdapter() {
  gson = new GsonBuilder()
            .serializeNulls()
            .registerTypeAdapter(TwillRuntimeSpecification.class, new TwillRuntimeSpecificationCodec())
            .registerTypeAdapter(TwillSpecification.class, new TwillSpecificationCodec())
            .registerTypeAdapter(TwillSpecification.Order.class, new TwillSpecificationOrderCoder())
            .registerTypeAdapter(TwillSpecification.PlacementPolicy.class,
                                 new TwillSpecificationPlacementPolicyCoder())
            .registerTypeAdapter(EventHandlerSpecification.class, new EventHandlerSpecificationCoder())
            .registerTypeAdapter(RuntimeSpecification.class, new RuntimeSpecificationCodec())
            .registerTypeAdapter(TwillRunnableSpecification.class, new TwillRunnableSpecificationCodec())
            .registerTypeAdapter(ResourceSpecification.class, new ResourceSpecificationCodec())
            .registerTypeAdapter(LocalFile.class, new LocalFileCodec())
            .registerTypeAdapterFactory(new TwillSpecificationTypeAdapterFactory())
            .create();
}
 

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
private void createContainerJar(ApplicationBundler bundler, Map<String, LocalFile> localFiles) throws IOException {
  try {
    Set<Class<?>> classes = Sets.newIdentityHashSet();
    classes.add(TwillContainerMain.class);
    classes.addAll(dependencies);

    ClassLoader classLoader = getClassLoader();
    for (RuntimeSpecification spec : twillSpec.getRunnables().values()) {
      classes.add(classLoader.loadClass(spec.getRunnableSpecification().getClassName()));
    }

    LOG.debug("Create and copy {}", Constants.Files.CONTAINER_JAR);
    Location location = createTempLocation(Constants.Files.CONTAINER_JAR);
    bundler.createBundle(location, classes, resources);
    LOG.debug("Done {}", Constants.Files.CONTAINER_JAR);

    localFiles.put(Constants.Files.CONTAINER_JAR, createLocalFile(Constants.Files.CONTAINER_JAR, location));

  } catch (ClassNotFoundException e) {
    throw Throwables.propagate(e);
  }
}
 

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
public DefaultTwillSpecification(String name, Map<String, RuntimeSpecification> runnables,
                                 List<Order> orders, List<PlacementPolicy> placementPolicies,
                                 EventHandlerSpecification eventHandler) {
  this.name = name;
  this.runnables = Collections.unmodifiableMap(new HashMap<String, RuntimeSpecification>(runnables));
  this.orders = Collections.unmodifiableList(new ArrayList<Order>(orders));
  this.placementPolicies = placementPolicies;
  this.eventHandler = eventHandler;
}
 

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
ExpectedContainers(TwillSpecification twillSpec) {
  Map<String, ExpectedCount> expectedCounts = Maps.newHashMap();
  long now = System.currentTimeMillis();
  for (RuntimeSpecification runtimeSpec : twillSpec.getRunnables().values()) {
    expectedCounts.put(runtimeSpec.getName(),
                       new ExpectedCount(runtimeSpec.getResourceSpecification().getInstances(), now));
  }
  this.expectedCounts = expectedCounts;
}
 

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
RunnableContainerRequest(TwillSpecification.Order.Type orderType,
                         Map<AllocationSpecification, Collection<RuntimeSpecification>> requests,
                         boolean isReadyToBeProvisioned) {
  this.orderType = orderType;
  this.requests = requests.entrySet().iterator();
  this.isReadyToBeProvisioned = isReadyToBeProvisioned;
}
 

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
/**
 * Manage Blacklist for a given request.
 */
private void manageBlacklist(Map.Entry<AllocationSpecification, ? extends Collection<RuntimeSpecification>> request) {
  amClient.clearBlacklist();

  //Check the allocation strategy
  AllocationSpecification allocationSpec = request.getKey();
  if (!allocationSpec.getType().equals(AllocationSpecification.Type.ALLOCATE_ONE_INSTANCE_AT_A_TIME)) {
    return;
  }

  //Check the placement policy
  String runnableName = allocationSpec.getRunnableName();
  TwillSpecification.PlacementPolicy placementPolicy = placementPolicyManager.getPlacementPolicy(runnableName);
  if (placementPolicy == null || placementPolicy.getType() != TwillSpecification.PlacementPolicy.Type.DISTRIBUTED) {
    return;
  }

  //Update blacklist with hosts which are running DISTRIBUTED runnables
  for (String runnable : placementPolicy.getNames()) {
    for (ContainerInfo containerInfo : runningContainers.getContainerInfo(runnable)) {
      // Yarn Resource Manager may include port in the node name depending on the setting
      // YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME. It is safe to add both
      // the names (with and without port) to the blacklist.
      LOG.debug("Adding {} to host blacklist", containerInfo.getHost().getHostName());
      amClient.addToBlacklist(containerInfo.getHost().getHostName());
      amClient.addToBlacklist(containerInfo.getHost().getHostName() + ":" + containerInfo.getPort());
    }
  }
}
 

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
/**
 * Helper method to create {@link org.apache.twill.internal.appmaster.RunnableContainerRequest}.
 */
private void addAllocationSpecification(AllocationSpecification allocationSpecification,
                                        Map<AllocationSpecification, Collection<RuntimeSpecification>> map,
                                        RuntimeSpecification runtimeSpec) {
  if (!map.containsKey(allocationSpecification)) {
    map.put(allocationSpecification, Lists.<RuntimeSpecification>newLinkedList());
  }
  map.get(allocationSpecification).add(runtimeSpec);
}
 

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
/**
 * Adds container requests with the given resource capability for each runtime.
 */
private void addContainerRequests(Resource capability,
                                  Collection<RuntimeSpecification> runtimeSpecs,
                                  Queue<ProvisionRequest> provisioning,
                                  AllocationSpecification.Type allocationType) {
  for (RuntimeSpecification runtimeSpec : runtimeSpecs) {
    String name = runtimeSpec.getName();
    int newContainers = expectedContainers.getExpected(name) - runningContainers.count(name);
    if (newContainers > 0) {
      if (allocationType.equals(AllocationSpecification.Type.ALLOCATE_ONE_INSTANCE_AT_A_TIME)) {
        //Spawning 1 instance at a time
        newContainers = 1;
      }

      // TODO: Allow user to set priority?
      LOG.info("Request {} containers with capability {} for runnable {}", newContainers, capability, name);
      YarnAMClient.ContainerRequestBuilder builder = amClient.addContainerRequest(capability, newContainers);
      builder.setPriority(0);

      TwillSpecification.PlacementPolicy placementPolicy = placementPolicyManager.getPlacementPolicy(name);
      if (placementPolicy != null) {
        builder.addHosts(placementPolicy.getHosts())
               .addRacks(placementPolicy.getRacks());
      }

      String requestId = builder.apply();
      provisioning.add(new ProvisionRequest(runtimeSpec, requestId, newContainers, allocationType));
    }
  }
}
 

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
ProvisionRequest(RuntimeSpecification runtimeSpec, String requestId, int requestCount,
                 AllocationSpecification.Type type) {
  this.runtimeSpec = runtimeSpec;
  this.requestId = requestId;
  this.requestCount = requestCount;
  this.type = type;
}
 

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
/**
 * Based on the given {@link TwillSpecification}, upload LocalFiles to Yarn Cluster.
 * @param twillSpec The {@link TwillSpecification} for populating resource.
 */
private Multimap<String, LocalFile> populateRunnableLocalFiles(TwillSpecification twillSpec) throws IOException {
  Multimap<String, LocalFile> localFiles = HashMultimap.create();

  LOG.debug("Populating Runnable LocalFiles");
  for (Map.Entry<String, RuntimeSpecification> entry: twillSpec.getRunnables().entrySet()) {
    String runnableName = entry.getKey();
    for (LocalFile localFile : entry.getValue().getLocalFiles()) {
      Location location;

      URI uri = localFile.getURI();
      if (appLocation.toURI().getScheme().equals(uri.getScheme())) {
        // If the source file location is having the same scheme as the target location, no need to copy
        location = appLocation.getLocationFactory().create(uri);
      } else {
        URL url = uri.toURL();
        LOG.debug("Create and copy {} : {}", runnableName, url);
        // Preserves original suffix for expansion.
        location = copyFromURL(url, createTempLocation(Paths.addExtension(url.getFile(), localFile.getName())));
        LOG.debug("Done {} : {}", runnableName, url);
      }

      localFiles.put(runnableName,
                     new DefaultLocalFile(localFile.getName(), location.toURI(), location.lastModified(),
                                          location.length(), localFile.isArchive(), localFile.getPattern()));
    }
  }
  LOG.debug("Done Runnable LocalFiles");
  return localFiles;
}
 

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
private TwillRuntimeSpecification saveSpecification(TwillSpecification spec, Path targetFile) throws IOException {
  final Multimap<String, LocalFile> runnableLocalFiles = populateRunnableLocalFiles(spec);

  // Rewrite LocalFiles inside twillSpec
  Map<String, RuntimeSpecification> runtimeSpec = Maps.transformEntries(
    spec.getRunnables(), new Maps.EntryTransformer<String, RuntimeSpecification, RuntimeSpecification>() {
      @Override
      public RuntimeSpecification transformEntry(String key, RuntimeSpecification value) {
        return new DefaultRuntimeSpecification(value.getName(), value.getRunnableSpecification(),
                                               value.getResourceSpecification(), runnableLocalFiles.get(key));
      }
    });

  // Serialize into a local temp file.
  LOG.debug("Creating {}", targetFile);
  try (Writer writer = Files.newBufferedWriter(targetFile, StandardCharsets.UTF_8)) {
    EventHandlerSpecification eventHandler = spec.getEventHandler();
    if (eventHandler == null) {
      eventHandler = new LogOnlyEventHandler().configure();
    }
    TwillSpecification newTwillSpec = new DefaultTwillSpecification(spec.getName(), runtimeSpec, spec.getOrders(),
                                                                    spec.getPlacementPolicies(), eventHandler);
    Map<String, String> configMap = Maps.newHashMap();
    for (Map.Entry<String, String> entry : config) {
      if (entry.getKey().startsWith("twill.")) {
        configMap.put(entry.getKey(), entry.getValue());
      }
    }

    TwillRuntimeSpecification twillRuntimeSpec = new TwillRuntimeSpecification(
      newTwillSpec, appLocation.getLocationFactory().getHomeLocation().getName(),
      appLocation.toURI(), zkConnectString, runId, twillSpec.getName(),
      config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS),
      logLevels, maxRetries, configMap, runnableConfigs);
    TwillRuntimeSpecificationAdapter.create().toJson(twillRuntimeSpec, writer);
    LOG.debug("Done {}", targetFile);
    return twillRuntimeSpec;
  }
}
 

import org.apache.twill.api.RuntimeSpecification; //导入依赖的package包/类
@Override
public JsonElement serialize(RuntimeSpecification src, Type typeOfSrc, JsonSerializationContext context) {
  JsonObject json = new JsonObject();
  json.addProperty("name", src.getName());
  json.add("runnable", context.serialize(src.getRunnableSpecification(), TwillRunnableSpecification.class));
  json.add("resources", context.serialize(src.getResourceSpecification(), ResourceSpecification.class));
  json.add("files", context.serialize(src.getLocalFiles(), new TypeToken<Collection<LocalFile>>() { }.getType()));

  return json;
}