Java DataSourceType.PERSISTED屬性代碼示例

@Test(timeout = 5000)
public void testVerifyScatterGather() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

@Test(timeout = 5000)
public void testVerifyCustomEdge() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(new EdgeManagerDescriptor("emClass"),
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

@Test(timeout = 5000)
public void testVerifyOneToOneNoInferParallelism() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      -1, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
  Assert.assertEquals(-1, v2.getParallelism());
}
 

@Test(timeout = 5000)
public void testVerifyBroadcast() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.BROADCAST, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

@Test(expected = IllegalStateException.class, timeout = 5000)
public void testInputAndInputVertexNameCollision() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  
  v2.addInput("v1", new InputDescriptor(), null);
  
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

@Test(expected = IllegalStateException.class, timeout = 5000)
public void testOutputAndOutputVertexNameCollision() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  
  v1.addOutput("v2", new OutputDescriptor());
  
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

public TezEdgeDescriptor() {
    combinePlan = new PhysicalPlan();

    // The default is shuffle edge.
    inputClassName = OrderedGroupedKVInput.class.getName();
    outputClassName = OrderedPartitionedKVOutput.class.getName();
    partitionerClass = null;
    schedulingType = SchedulingType.SEQUENTIAL;
    dataSourceType = DataSourceType.PERSISTED;
    dataMovementType = DataMovementType.SCATTER_GATHER;
}
 

public static DataSourceType convertFromDAGPlan(PlanEdgeDataSourceType sourceType){
  switch(sourceType){
    case PERSISTED : return DataSourceType.PERSISTED;
    case PERSISTED_RELIABLE : return DataSourceType.PERSISTED_RELIABLE;
    case EPHEMERAL :  return DataSourceType.EPHEMERAL;
    default : throw new RuntimeException("unknown 'dataSourceType': " + sourceType);
  }
}
 

@Test(timeout = 5000)
public void testEdgeManagerSerde() {
  DAG dag = new DAG("testDag");
  ProcessorDescriptor pd1 = new ProcessorDescriptor("processor1")
      .setUserPayload("processor1Bytes".getBytes());
  ProcessorDescriptor pd2 = new ProcessorDescriptor("processor2")
      .setUserPayload("processor2Bytes".getBytes());
  Vertex v1 = new Vertex("v1", pd1, 10, Resource.newInstance(1024, 1));
  Vertex v2 = new Vertex("v2", pd2, 1, Resource.newInstance(1024, 1));
  v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .setTaskLocalFiles(new HashMap<String, LocalResource>());
  v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .setTaskLocalFiles(new HashMap<String, LocalResource>());

  InputDescriptor inputDescriptor = new InputDescriptor("input").setUserPayload("inputBytes"
      .getBytes());
  OutputDescriptor outputDescriptor = new OutputDescriptor("output").setUserPayload("outputBytes"
      .getBytes());
  Edge edge = new Edge(v1, v2, new EdgeProperty(
      new EdgeManagerDescriptor("emClass").setUserPayload("emPayload".getBytes()),
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));

  dag.addVertex(v1).addVertex(v2).addEdge(edge);

  DAGPlan dagProto = dag.createDag(new TezConfiguration());

  EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dagProto
      .getEdgeList().get(0));

  EdgeManagerDescriptor emDesc = edgeProperty.getEdgeManagerDescriptor();
  Assert.assertNotNull(emDesc);
  Assert.assertEquals("emClass", emDesc.getClassName());
  Assert.assertTrue(Arrays.equals("emPayload".getBytes(), emDesc.getUserPayload()));
}
 

@Test(timeout = 5000)
// v1 (known) -> v2 (-1) -> v3 (-1)
// The test checks resiliency to ordering of the vertices/edges
public void testVerifyOneToOneInferParallelismReverseOrder() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  Edge e2 = new Edge(v2, v3,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v3);
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e2);
  dag.addEdge(e1);
  dag.verify();
  Assert.assertEquals(dummyTaskCount, v2.getParallelism());
  Assert.assertEquals(dummyTaskCount, v3.getParallelism());
}
 

@Test(timeout = 5000)
// v1 (-1) -> v2 (known) -> v3 (-1)
public void testVerifyOneToOneIncorrectParallelism1() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      -1, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v3,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  Edge e2 = new Edge(v2, v3,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addEdge(e1);
  dag.addEdge(e2);
  try {
    dag.verify();
    Assert.assertTrue(false);
  } catch (TezUncheckedException e) {
    Assert.assertTrue(e.getMessage().contains(
        "1-1 Edge. Destination vertex parallelism must match source vertex"));
  }
}
 

@Test(timeout = 5000)
public void BinaryOutput() {
  IllegalStateException ex = null;
  try {
    Vertex v1 = new Vertex("v1",
        new ProcessorDescriptor("MapProcessor"),
        dummyTaskCount, dummyTaskResource);
    Vertex v2 = new Vertex("v2",
        new ProcessorDescriptor("MapProcessor"),
        dummyTaskCount, dummyTaskResource);
    Vertex v3 = new Vertex("v3",
        new ProcessorDescriptor("MapProcessor"),
        dummyTaskCount, dummyTaskResource);
    Edge e1 = new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER, 
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
            new OutputDescriptor("dummy output class"),
            new InputDescriptor("dummy input class")));
    Edge e2 = new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER, 
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
            new OutputDescriptor("dummy output class"),
            new InputDescriptor("dummy input class")));
    DAG dag = new DAG("testDag");
    dag.addVertex(v1);
    dag.addVertex(v2);
    dag.addVertex(v3);
    dag.addEdge(e1);
    dag.addEdge(e2);
    dag.verify();
  }
  catch (IllegalStateException e){
    ex = e;
  }
  Assert.assertNull(ex);
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test (timeout = 5000)
public void testCompositeEventHandling() {
  EventHandler eventHandler = mock(EventHandler.class);
  EdgeProperty edgeProp = new EdgeProperty(DataMovementType.SCATTER_GATHER,
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, mock(OutputDescriptor.class),
      mock(InputDescriptor.class));
  Edge edge = new Edge(edgeProp, eventHandler);
  
  TezVertexID srcVertexID = createVertexID(1);
  TezVertexID destVertexID = createVertexID(2);
  LinkedHashMap<TezTaskID, Task> srcTasks = mockTasks(srcVertexID, 1);
  LinkedHashMap<TezTaskID, Task> destTasks = mockTasks(destVertexID, 5);
  
  TezTaskID srcTaskID = srcTasks.keySet().iterator().next();
  
  Vertex srcVertex = mockVertex("src", srcVertexID, srcTasks);
  Vertex destVertex = mockVertex("dest", destVertexID, destTasks);
  
  edge.setSourceVertex(srcVertex);
  edge.setDestinationVertex(destVertex);
  edge.initialize();
  
  TezTaskAttemptID srcTAID = createTAIDForTest(srcTaskID, 2); // Task0, Attempt 0
  
  EventMetaData srcMeta = new EventMetaData(EventProducerConsumerType.OUTPUT, "consumerVertex", "producerVertex", srcTAID);
  
  // Verification via a CompositeEvent
  CompositeDataMovementEvent cdmEvent = new CompositeDataMovementEvent(0, destTasks.size(), "bytes".getBytes());
  cdmEvent.setVersion(2); // AttemptNum
  TezEvent tezEvent = new TezEvent(cdmEvent, srcMeta);
  // Event setup to look like it would after the Vertex is done with it.

  edge.sendTezEventToDestinationTasks(tezEvent);
  
  ArgumentCaptor<Event> args = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(destTasks.size())).handle(args.capture());
  
  verifyEvents(args.getAllValues(), srcTAID, destTasks);
  
  
  // Same Verification via regular DataMovementEvents
  reset(eventHandler);
  for (int i = 0 ; i < destTasks.size() ; i++) {
    DataMovementEvent dmEvent = new DataMovementEvent(i, "bytes".getBytes());
    dmEvent.setVersion(2);
    tezEvent = new TezEvent(dmEvent, srcMeta);
    edge.sendTezEventToDestinationTasks(tezEvent);
  }
  args = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(destTasks.size())).handle(args.capture());
  
  verifyEvents(args.getAllValues(), srcTAID, destTasks);

}
 

@Test(timeout = 5000)
public void testUserPayloadSerde() {
  DAG dag = new DAG("testDag");
  ProcessorDescriptor pd1 = new ProcessorDescriptor("processor1").
      setUserPayload("processor1Bytes".getBytes());
  ProcessorDescriptor pd2 = new ProcessorDescriptor("processor2").
      setUserPayload("processor2Bytes".getBytes());
  Vertex v1 = new Vertex("v1", pd1, 10, Resource.newInstance(1024, 1));
  Vertex v2 = new Vertex("v2", pd2, 1, Resource.newInstance(1024, 1));
  v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .setTaskLocalFiles(new HashMap<String, LocalResource>());
  v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .setTaskLocalFiles(new HashMap<String, LocalResource>());

  InputDescriptor inputDescriptor = new InputDescriptor("input").
      setUserPayload("inputBytes".getBytes());
  OutputDescriptor outputDescriptor = new OutputDescriptor("output").
      setUserPayload("outputBytes".getBytes());
  Edge edge = new Edge(v1, v2, new EdgeProperty(
      DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
      SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));

  dag.addVertex(v1).addVertex(v2).addEdge(edge);

  DAGPlan dagProto = dag.createDag(new TezConfiguration());

  assertEquals(2, dagProto.getVertexCount());
  assertEquals(1, dagProto.getEdgeCount());

  VertexPlan v1Proto = dagProto.getVertex(0);
  VertexPlan v2Proto = dagProto.getVertex(1);
  EdgePlan edgeProto = dagProto.getEdge(0);

  assertEquals("processor1Bytes", new String(v1Proto.getProcessorDescriptor()
      .getUserPayload().toByteArray()));
  assertEquals("processor1", v1Proto.getProcessorDescriptor().getClassName());

  assertEquals("processor2Bytes", new String(v2Proto.getProcessorDescriptor()
      .getUserPayload().toByteArray()));
  assertEquals("processor2", v2Proto.getProcessorDescriptor().getClassName());

  assertEquals("inputBytes", new String(edgeProto.getEdgeDestination()
      .getUserPayload().toByteArray()));
  assertEquals("input", edgeProto.getEdgeDestination().getClassName());

  assertEquals("outputBytes", new String(edgeProto.getEdgeSource()
      .getUserPayload().toByteArray()));
  assertEquals("output", edgeProto.getEdgeSource().getClassName());

  EdgeProperty edgeProperty = DagTypeConverters
      .createEdgePropertyMapFromDAGPlan(dagProto.getEdgeList().get(0));

  byte[] ib = edgeProperty.getEdgeDestination().getUserPayload();
  assertEquals("inputBytes", new String(ib));
  assertEquals("input", edgeProperty.getEdgeDestination().getClassName());

  byte[] ob = edgeProperty.getEdgeSource().getUserPayload();
  assertEquals("outputBytes", new String(ob));
  assertEquals("output", edgeProperty.getEdgeSource().getClassName());
}
 

@Test(timeout = 5000)
public void testVertexGroupOneToOne() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v4 = new Vertex("v4",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v5 = new Vertex("v5",
      new ProcessorDescriptor("Processor"),
      -1, dummyTaskResource);
  
  DAG dag = new DAG("testDag");
  String groupName1 = "uv12";
  VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
  OutputDescriptor outDesc = new OutputDescriptor();
  uv12.addOutput("uvOut", outDesc, null);
  
  String groupName2 = "uv23";
  VertexGroup uv23 = dag.createVertexGroup(groupName2, v2, v3);
  
  GroupInputEdge e1 = new GroupInputEdge(uv12, v4,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")),
          new InputDescriptor("dummy input class"));
  GroupInputEdge e2 = new GroupInputEdge(uv23, v5,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")),
          new InputDescriptor("dummy input class"));
  
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addVertex(v4);
  dag.addVertex(v5);
  dag.addEdge(e1);
  dag.addEdge(e2);
  dag.verify();
  Assert.assertEquals(dummyTaskCount, v5.getParallelism());
}