當前位置: 首頁>>代碼示例>>Java>>正文

Java InputDescriptor類代碼示例

本文整理匯總了Java中org.apache.tez.dag.api.InputDescriptor類的典型用法代碼示例。如果您正苦於以下問題：Java InputDescriptor類的具體用法？Java InputDescriptor怎麽用？Java InputDescriptor使用的例子？那麽, 這裏精選的類代碼示例或許可以為您提供幫助。


InputDescriptor類屬於org.apache.tez.dag.api包，在下文中一共展示了InputDescriptor類的15個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。

示例1: newGroupInputEdge

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
private GroupInputEdge newGroupInputEdge(TezOperator fromOp,
        TezOperator toOp, VertexGroup from, Vertex to) throws IOException {

    EdgeProperty edgeProperty = newEdge(fromOp, toOp);

    String groupInputClass = ConcatenatedMergedKeyValueInput.class.getName();

    // In case of SCATTER_GATHER and UnorderedKVInput it will still be
    // ConcatenatedMergedKeyValueInput
    if(edgeProperty.getDataMovementType().equals(DataMovementType.SCATTER_GATHER)
            && edgeProperty.getEdgeDestination().getClassName().equals(OrderedGroupedKVInput.class.getName())) {
        groupInputClass = OrderedGroupedMergedKVInput.class.getName();
    }

    return GroupInputEdge.create(from, to, edgeProperty,
            InputDescriptor.create(groupInputClass).setUserPayload(edgeProperty.getEdgeDestination().getUserPayload())
                .setHistoryText(edgeProperty.getEdgeDestination().getHistoryText()));
}
開發者ID:sigmoidanalytics，項目名稱:spork，代碼行數:19，代碼來源:TezDagBuilder.java

示例2: testInvalidConsumerNumber

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
@Test(timeout = 5000)
public void testInvalidConsumerNumber() throws Exception {
  EventHandler mockEventHandler = mock(EventHandler.class);
  Edge edge = new Edge(EdgeProperty.create(
      EdgeManagerPluginDescriptor.create(CustomEdgeManagerWithInvalidReturnValue.class.getName())
        .setUserPayload(new CustomEdgeManagerWithInvalidReturnValue.EdgeManagerConfig(1,1,0,1).toUserPayload()),
      DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create(""),
      InputDescriptor.create("")), mockEventHandler, new TezConfiguration());
  TezVertexID v1Id = createVertexID(1);
  TezVertexID v2Id = createVertexID(2);
  edge.setSourceVertex(mockVertex("v1", v1Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.setDestinationVertex(mockVertex("v2", v2Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.initialize();
  try {
    TezEvent ireEvent = new TezEvent(InputReadErrorEvent.create("diag", 0, 1),
        new EventMetaData(EventProducerConsumerType.INPUT, "v2", "v1",
            TezTaskAttemptID.getInstance(TezTaskID.getInstance(v2Id, 1), 1)));
    edge.sendTezEventToSourceTasks(ireEvent);
    Assert.fail();
  } catch (AMUserCodeException e) {
    e.printStackTrace();
    assertTrue(e.getCause().getMessage().contains("ConsumerTaskNum must be positive"));
  }
}
開發者ID:apache，項目名稱:tez，代碼行數:27，代碼來源:TestEdge.java

示例3: createDAG

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
private DAG createDAG(String dagName, boolean uv12CommitFail, boolean v3CommitFail) {
  DAG dag = DAG.create(dagName);
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("Proc"), 1);
  Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("Proc"), 1);
  Vertex v3 = Vertex.create("v3", ProcessorDescriptor.create("Proc"), 1);
  VertexGroup uv12 = dag.createVertexGroup("uv12", v1, v2);
  DataSinkDescriptor uv12DataSink = DataSinkDescriptor.create(
      OutputDescriptor.create("dummy output"), createOutputCommitterDesc(uv12CommitFail), null);
  uv12.addDataSink("uv12Out", uv12DataSink);
  DataSinkDescriptor v3DataSink = DataSinkDescriptor.create(
      OutputDescriptor.create("dummy output"), createOutputCommitterDesc(v3CommitFail), null);
  v3.addDataSink("v3Out", v3DataSink);

  GroupInputEdge e1 = GroupInputEdge.create(uv12, v3, EdgeProperty.create(
      DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create("dummy output class"),
      InputDescriptor.create("dummy input class")), InputDescriptor
      .create("merge.class"));
  dag.addVertex(v1)
    .addVertex(v2)
    .addVertex(v3)
    .addEdge(e1);
  return dag;
}
開發者ID:apache，項目名稱:tez，代碼行數:26，代碼來源:TestMockDAGAppMaster.java

示例4: createLaunchRequestEvent

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
private AMSchedulerEventTALaunchRequest createLaunchRequestEvent(
  TezTaskAttemptID taID, TaskAttempt ta, Resource capability,
  String[] hosts, String[] racks, Priority priority,
  ContainerContext containerContext) {
  TaskLocationHint locationHint = null;
  if (hosts != null || racks != null) {
    Set<String> hostsSet = Sets.newHashSet(hosts);
    Set<String> racksSet = Sets.newHashSet(racks);
    locationHint = TaskLocationHint.createTaskLocationHint(hostsSet, racksSet);
  }
  AMSchedulerEventTALaunchRequest lr = new AMSchedulerEventTALaunchRequest(
    taID, capability, new TaskSpec(taID, "dagName", "vertexName", -1,
      ProcessorDescriptor.create("processorClassName"),
    Collections.singletonList(new InputSpec("vertexName",
        InputDescriptor.create("inputClassName"), 1)),
    Collections.singletonList(new OutputSpec("vertexName",
        OutputDescriptor.create("outputClassName"), 1)), null, null), ta, locationHint,
    priority.getPriority(), containerContext, 0, 0, 0);
  return lr;
}
開發者ID:apache，項目名稱:tez，代碼行數:21，代碼來源:TestContainerReuse.java

示例5: testCorrectUgiUsage

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
@Test (timeout = 5000)
public void testCorrectUgiUsage() throws TezException, InterruptedException {
  Vertex vertex = mock(Vertex.class);
  doReturn(mock(TezVertexID.class)).when(vertex).getVertexId();
  AppContext appContext = mock(AppContext.class);
  doReturn(new DefaultHadoopShim()).when(appContext).getHadoopShim();
  doReturn(mock(EventHandler.class)).when(appContext).getEventHandler();
  UserGroupInformation dagUgi = UserGroupInformation.createRemoteUser("fakeuser");
  StateChangeNotifier stateChangeNotifier = mock(StateChangeNotifier.class);
  RootInputInitializerManager rootInputInitializerManager = new RootInputInitializerManager(vertex, appContext, dagUgi, stateChangeNotifier);

  InputDescriptor id = mock(InputDescriptor.class);
  InputInitializerDescriptor iid = InputInitializerDescriptor.create(InputInitializerForUgiTest.class.getName());
  RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> rootInput =
      new RootInputLeafOutput<>("InputName", id, iid);
  rootInputInitializerManager.runInputInitializers(Collections.singletonList(rootInput));

  InputInitializerForUgiTest.awaitInitialize();

  assertEquals(dagUgi, InputInitializerForUgiTest.ctorUgi);
  assertEquals(dagUgi, InputInitializerForUgiTest.initializeUgi);
}
開發者ID:apache，項目名稱:tez，代碼行數:23，代碼來源:TestRootInputInitializerManager.java

示例6: testVertexSetParallelismIncrease

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
@Test(timeout = 5000)
public void testVertexSetParallelismIncrease() throws Exception {
  VertexImpl v3 = vertices.get("vertex3");
  v3.vertexReconfigurationPlanned();
  initAllVertices(VertexState.INITED);
  Assert.assertEquals(2, v3.getTotalTasks());
  Assert.assertEquals(2, v3.getTasks().size());

  VertexImpl v1 = vertices.get("vertex1");
  startVertex(vertices.get("vertex2"));
  startVertex(v1);

  EdgeManagerPluginDescriptor mockEdgeManagerDescriptor =
      EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
  EdgeProperty edgeProp = EdgeProperty.create(mockEdgeManagerDescriptor, 
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), 
      InputDescriptor.create("In"));
  Map<String, EdgeProperty> edgeManagerDescriptors =
      Collections.singletonMap(
     v1.getName(), edgeProp);
  v3.reconfigureVertex(10, null, edgeManagerDescriptors);
  v3.doneReconfiguringVertex();
  assertTrue(v3.sourceVertices.get(v1).getEdgeManager() instanceof
      EdgeManagerForTest);
  checkTasks(v3, 10);
}
開發者ID:apache，項目名稱:tez，代碼行數:27，代碼來源:TestVertexImpl.java

示例7: testScalingNoProcessor2

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
@Test(timeout = 5000)
public void testScalingNoProcessor2() throws TezException {
  // Real world values
  MemoryDistributor dist = new MemoryDistributor(2, 0, conf);
  
  dist.setJvmMemory(209715200l);

  // First request
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  InputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor();
  dist.requestMemory(104857600l, e1Callback, e1InputContext1, e1InDesc1);
  
  // Second request
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  InputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor();
  dist.requestMemory(157286400l, e2Callback, e2InputContext2, e2InDesc2);
  
  dist.makeInitialAllocations();

  assertEquals(58720256l, e1Callback.assigned);
  assertEquals(88080384l, e2Callback.assigned);
}
開發者ID:apache，項目名稱:tez，代碼行數:25，代碼來源:TestMemoryDistributor.java

示例8: toProto

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
public RecoveryProtos.VertexInitializedProto toProto() {
  VertexInitializedProto.Builder builder = VertexInitializedProto.newBuilder();
  if (additionalInputs != null
    && !additionalInputs.isEmpty()) {
    for (RootInputLeafOutputDescriptor<InputDescriptor> input :
      additionalInputs.values()) {
      RootInputLeafOutputProto.Builder inputBuilder
          = RootInputLeafOutputProto.newBuilder();
      inputBuilder.setName(input.getEntityName());
      if (input.getInitializerClassName() != null) {
        inputBuilder.setInitializerClassName(input.getInitializerClassName());
      }
      inputBuilder.setEntityDescriptor(
          DagTypeConverters.convertToDAGPlan(input.getDescriptor()));
      builder.addInputs(inputBuilder.build());
    }
  }
  return builder.setVertexId(vertexID.toString())
      .setVertexName(vertexName)
      .setInitRequestedTime(initRequestedTime)
      .setInitTime(initedTime)
      .setNumTasks(numTasks)
      .build();
}
開發者ID:apache，項目名稱:incubator-tez，代碼行數:25，代碼來源:VertexInitializedEvent.java

示例9: createInitializer

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
@Override
protected TezRootInputInitializer createInitializer(
    RootInputLeafOutputDescriptor<InputDescriptor> input) {

  return new TezRootInputInitializer() {
    @Override
    public List<Event> initialize(TezRootInputInitializerContext inputVertexContext) throws
        Exception {
      return null;
    }

    @Override
    public void handleInputInitializerEvent(List<RootInputInitializerEvent> events) throws
        Exception {
    }
  };
}
開發者ID:apache，項目名稱:incubator-tez，代碼行數:18，代碼來源:TestVertexImpl.java

示例10: testInvalidSourceTaskIndex

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
@Test(timeout = 5000)
public void testInvalidSourceTaskIndex() throws Exception {
  EventHandler mockEventHandler = mock(EventHandler.class);
  Edge edge = new Edge(EdgeProperty.create(
      EdgeManagerPluginDescriptor.create(CustomEdgeManagerWithInvalidReturnValue.class.getName())
        .setUserPayload(new CustomEdgeManagerWithInvalidReturnValue.EdgeManagerConfig(1,1,1,-1).toUserPayload()),
      DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create(""),
      InputDescriptor.create("")), mockEventHandler, new TezConfiguration());
  TezVertexID v1Id = createVertexID(1);
  TezVertexID v2Id = createVertexID(2);
  edge.setSourceVertex(mockVertex("v1", v1Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.setDestinationVertex(mockVertex("v2", v2Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.initialize();
  try {
    TezEvent ireEvent = new TezEvent(InputReadErrorEvent.create("diag", 0, 1),
        new EventMetaData(EventProducerConsumerType.INPUT, "v2", "v1",
            TezTaskAttemptID.getInstance(TezTaskID.getInstance(v2Id, 1), 1)));
    edge.sendTezEventToSourceTasks(ireEvent);
    Assert.fail();
  } catch (AMUserCodeException e) {
    e.printStackTrace();
    assertTrue(e.getCause().getMessage().contains("SourceTaskIndex should not be negative"));
  }
}
開發者ID:apache，項目名稱:tez，代碼行數:27，代碼來源:TestEdge.java

示例11: TezInputContextImpl

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
@Private
public TezInputContextImpl(Configuration conf, String[] workDirs, int appAttemptNumber,
    TezUmbilical tezUmbilical, String dagName, String taskVertexName,
    String sourceVertexName, TezTaskAttemptID taskAttemptID,
    TezCounters counters, int inputIndex, @Nullable byte[] userPayload,
    RuntimeTask runtimeTask, Map<String, ByteBuffer> serviceConsumerMetadata,
    Map<String, String> auxServiceEnv, MemoryDistributor memDist,
    InputDescriptor inputDescriptor,  Input input, InputReadyTracker inputReadyTracker) {
  super(conf, workDirs, appAttemptNumber, dagName, taskVertexName, taskAttemptID,
      wrapCounters(counters, taskVertexName, sourceVertexName, conf),
      runtimeTask, tezUmbilical, serviceConsumerMetadata,
      auxServiceEnv, memDist, inputDescriptor);
  checkNotNull(inputIndex, "inputIndex is null");
  checkNotNull(sourceVertexName, "sourceVertexName is null");
  checkNotNull(input, "input is null");
  checkNotNull(inputReadyTracker, "inputReadyTracker is null");
  this.userPayload = DagTypeConverters.convertToTezUserPayload(userPayload);
  this.inputIndex = inputIndex;
  this.sourceVertexName = sourceVertexName;
  this.sourceInfo = new EventMetaData(
      EventProducerConsumerType.INPUT, taskVertexName, sourceVertexName,
      taskAttemptID);
  this.input = input;
  this.inputReadyTracker = inputReadyTracker;
}
開發者ID:apache，項目名稱:incubator-tez，代碼行數:26，代碼來源:TezInputContextImpl.java

示例12: setupInputInitializerManager

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
private void setupInputInitializerManager() throws TezException {
  rootInputInitializerManager = createRootInputInitializerManager(
      getDAG().getName(), getName(), getVertexId(),
      eventHandler, getTotalTasks(),
      appContext.getTaskScheduler().getNumClusterNodes(),
      getTaskResource(),
      appContext.getTaskScheduler().getTotalResources(taskSchedulerIdentifier));
  List<RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor>>
      inputList = Lists.newArrayListWithCapacity(inputsWithInitializers.size());
  for (String inputName : inputsWithInitializers) {
    inputList.add(rootInputDescriptors.get(inputName));
  }
  LOG.info("Starting " + inputsWithInitializers.size() + " inputInitializers for vertex " +
      logIdentifier);
  initWaitsForRootInitializers = true;
  rootInputInitializerManager.runInputInitializers(inputList);
  // Send pending rootInputInitializerEvents
  rootInputInitializerManager.handleInitializerEvents(pendingInitializerEvents);
  pendingInitializerEvents.clear();
}
開發者ID:apache，項目名稱:tez，代碼行數:21，代碼來源:VertexImpl.java

示例13: testInvalidPhysicalOutputCount

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
@Test(timeout = 5000)
public void testInvalidPhysicalOutputCount() throws Exception {
  EventHandler mockEventHandler = mock(EventHandler.class);
  Edge edge = new Edge(EdgeProperty.create(
      EdgeManagerPluginDescriptor.create(CustomEdgeManagerWithInvalidReturnValue.class.getName())
        .setUserPayload(new CustomEdgeManagerWithInvalidReturnValue.EdgeManagerConfig(1,-1,1,1).toUserPayload()),
      DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL,
      OutputDescriptor.create(""),
      InputDescriptor.create("")), mockEventHandler, new TezConfiguration());
  TezVertexID v1Id = createVertexID(1);
  TezVertexID v2Id = createVertexID(2);
  edge.setSourceVertex(mockVertex("v1", v1Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.setDestinationVertex(mockVertex("v2", v2Id, new LinkedHashMap<TezTaskID, Task>()));
  edge.initialize();
  try {
    edge.getSourceSpec(0);
    Assert.fail();
  } catch (AMUserCodeException e) {
    e.printStackTrace();
    assertTrue(e.getCause().getMessage().contains("PhysicalOutputCount should not be negative"));
  }
}
開發者ID:apache，項目名稱:tez，代碼行數:24，代碼來源:TestEdge.java

示例14: testScalingNoProcessor2

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
@Test(timeout = 5000)
public void testScalingNoProcessor2() {
  // Real world values
  MemoryDistributor dist = new MemoryDistributor(2, 0, conf);
  
  dist.setJvmMemory(209715200l);

  // First request
  MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
  TezInputContext e1InputContext1 = createTestInputContext();
  InputDescriptor e1InDesc1 = createTestInputDescriptor();
  dist.requestMemory(104857600l, e1Callback, e1InputContext1, e1InDesc1);
  
  // Second request
  MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
  TezInputContext e2InputContext2 = createTestInputContext();
  InputDescriptor e2InDesc2 = createTestInputDescriptor();
  dist.requestMemory(157286400l, e2Callback, e2InputContext2, e2InDesc2);
  
  dist.makeInitialAllocations();

  assertEquals(58720256l, e1Callback.assigned);
  assertEquals(88080384l, e2Callback.assigned);
}
開發者ID:apache，項目名稱:incubator-tez，代碼行數:25，代碼來源:TestMemoryDistributor.java

示例15: createInitializer

import org.apache.tez.dag.api.InputDescriptor; //導入依賴的package包/類
@Override
protected InputInitializer createInitializer(
    RootInputLeafOutput<InputDescriptor, InputInitializerDescriptor> input,
    InputInitializerContext context) {

  return new InputInitializer(context) {
    @Override
    public List<Event> initialize() throws
        Exception {
      return null;
    }

    @Override
    public void handleInputInitializerEvent(List<InputInitializerEvent> events) throws
        Exception {
    }
  };
}
開發者ID:apache，項目名稱:tez，代碼行數:19，代碼來源:TestVertexImpl.java


注：本文中的org.apache.tez.dag.api.InputDescriptor類示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。