当前位置: 首页>>代码示例>>Java>>正文

Java DataMovementType类代码示例

本文整理汇总了Java中org.apache.tez.dag.api.EdgeProperty.DataMovementType类的典型用法代码示例。如果您正苦于以下问题：Java DataMovementType类的具体用法？Java DataMovementType怎么用？Java DataMovementType使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。


DataMovementType类属于org.apache.tez.dag.api.EdgeProperty包，在下文中一共展示了DataMovementType类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: newGroupInputEdge

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
private GroupInputEdge newGroupInputEdge(TezOperator fromOp,
        TezOperator toOp, VertexGroup from, Vertex to) throws IOException {

    EdgeProperty edgeProperty = newEdge(fromOp, toOp);

    String groupInputClass = ConcatenatedMergedKeyValueInput.class.getName();

    // In case of SCATTER_GATHER and UnorderedKVInput it will still be
    // ConcatenatedMergedKeyValueInput
    if(edgeProperty.getDataMovementType().equals(DataMovementType.SCATTER_GATHER)
            && edgeProperty.getEdgeDestination().getClassName().equals(OrderedGroupedKVInput.class.getName())) {
        groupInputClass = OrderedGroupedMergedKVInput.class.getName();
    }

    return GroupInputEdge.create(from, to, edgeProperty,
            InputDescriptor.create(groupInputClass).setUserPayload(edgeProperty.getEdgeDestination().getUserPayload())
                .setHistoryText(edgeProperty.getEdgeDestination().getHistoryText()));
}
开发者ID:sigmoidanalytics，项目名称:spork，代码行数:19，代码来源:TezDagBuilder.java

示例2: createDAG

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(payload), 
          TestInput.getInputDesc(payload))));
  return dag;
}
开发者ID:apache，项目名称:incubator-tez，代码行数:20，代码来源:SimpleTestDAG.java

示例3: testDAGInvalidParallelism4

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
@Test//(timeout = 5000)
public void testDAGInvalidParallelism4() {
  DAG dag = DAG.create("testDAG");
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName));
  dag.addVertex(v1);
  try {
    dag.verify();
    Assert.fail();
  } catch (Exception e) {
    Assert.assertEquals(
        "v1 has -1 tasks but does not have input initializers, 1-1 uninited sources or custom vertex manager to set it at runtime",
        e.getMessage());
  }

  Vertex v0 = Vertex.create("v2", ProcessorDescriptor.create(dummyProcessorClassName));
  v0.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(dummyVMPluginClassName));
  dag.addVertex(v0);
  dag.addEdge(Edge.create(v0, v1, EdgeProperty.create(DataMovementType.ONE_TO_ONE,
      DataSourceType.PERSISTED,SchedulingType.SEQUENTIAL,
      OutputDescriptor.create(dummyOutputClassName),
      InputDescriptor.create(dummyInputClassName))));
  dag.verify();
}
开发者ID:apache，项目名称:tez，代码行数:24，代码来源:TestDAGVerify.java

示例4: createDAG

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  byte[] payload = null;
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = new DAG(name);
  Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2, 
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, 
          SchedulingType.SEQUENTIAL, 
          TestOutput.getOutputDesc(payload), 
          TestInput.getInputDesc(payload))));
  dag.addVertex(v3).addEdge(new Edge(v2, v3, 
          new EdgeProperty(DataMovementType.SCATTER_GATHER, 
              DataSourceType.PERSISTED, 
              SchedulingType.SEQUENTIAL, 
              TestOutput.getOutputDesc(payload), 
              TestInput.getInputDesc(payload))));
  return dag;
}
开发者ID:apache，项目名称:incubator-tez，代码行数:27，代码来源:SimpleTestDAG3Vertices.java

示例5: initialize

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
public void initialize() {
  byte[] bb = null;
  if (edgeProperty.getDataMovementType() == DataMovementType.CUSTOM) {
    if (edgeProperty.getEdgeManagerDescriptor() != null && 
        edgeProperty.getEdgeManagerDescriptor().getUserPayload() != null) {
      bb = edgeProperty.getEdgeManagerDescriptor().getUserPayload();
    }
  }
  edgeManagerContext = new EdgeManagerContextImpl(bb);
  if (edgeManager != null) {
    edgeManager.initialize(edgeManagerContext);
  }
  destinationMetaInfo = new EventMetaData(EventProducerConsumerType.INPUT, 
      destinationVertex.getName(), 
      sourceVertex.getName(), 
      null);
}
开发者ID:apache，项目名称:incubator-tez，代码行数:18，代码来源:Edge.java

示例6: testInputAndInputVertexNameCollision

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
@Test(expected = IllegalStateException.class, timeout = 5000)
public void testInputAndInputVertexNameCollision() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  
  v2.addDataSource("v1", DataSourceDescriptor.create(null, null, null));
  
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create("dummy output class"),
          InputDescriptor.create("dummy input class")));
  
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
开发者ID:apache，项目名称:tez，代码行数:24，代码来源:TestDAGVerify.java

示例7: testOutputAndOutputVertexNameCollision

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
@Test(expected = IllegalStateException.class, timeout = 5000)
public void testOutputAndOutputVertexNameCollision() {
  Vertex v1 = Vertex.create("v1",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = Vertex.create("v2",
      ProcessorDescriptor.create("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  
  v1.addDataSink("v2", DataSinkDescriptor.create(null, null, null));
  
  Edge e1 = Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          OutputDescriptor.create("dummy output class"),
          InputDescriptor.create("dummy input class")));
  
  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
开发者ID:apache，项目名称:tez，代码行数:24，代码来源:TestDAGVerify.java

示例8: testVerifyScatterGather

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
@Test(timeout = 5000)
public void testVerifyScatterGather() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
开发者ID:apache，项目名称:incubator-tez，代码行数:20，代码来源:TestDAGVerify.java

示例9: testVerifyOneToOne

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
@Test(timeout = 5000)
public void testVerifyOneToOne() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
开发者ID:apache，项目名称:incubator-tez，代码行数:20，代码来源:TestDAGVerify.java

示例10: testVerifyOneToOneNoInferParallelism

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
@Test(timeout = 5000)
public void testVerifyOneToOneNoInferParallelism() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      -1, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
  Assert.assertEquals(-1, v2.getParallelism());
}
开发者ID:apache，项目名称:incubator-tez，代码行数:21，代码来源:TestDAGVerify.java

示例11: testVerifyBroadcast

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
@Test(timeout = 5000)
public void testVerifyBroadcast() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.BROADCAST, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
开发者ID:apache，项目名称:incubator-tez，代码行数:20，代码来源:TestDAGVerify.java

示例12: testVerify4

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
@Test(expected = IllegalStateException.class, timeout = 5000)  
public void testVerify4() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.EPHEMERAL, SchedulingType.CONCURRENT, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
开发者ID:apache，项目名称:incubator-tez，代码行数:20，代码来源:TestDAGVerify.java

示例13: testOutputAndOutputVertexNameCollision

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
@Test(expected = IllegalStateException.class, timeout = 5000)
public void testOutputAndOutputVertexNameCollision() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  
  v1.addOutput("v2", new OutputDescriptor());
  
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
开发者ID:apache，项目名称:incubator-tez，代码行数:24，代码来源:TestDAGVerify.java

示例14: createDAG

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
public static DAG createDAG(String name, 
    Configuration conf) throws Exception {
  UserPayload payload = UserPayload.create(null);
  int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
  if (conf != null) {
    taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
    payload = TezUtils.createUserPayloadFromConf(conf);
  }
  DAG dag = DAG.create(name);
  Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
  dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2,
      EdgeProperty.create(DataMovementType.SCATTER_GATHER,
          DataSourceType.PERSISTED,
          SchedulingType.SEQUENTIAL,
          TestOutput.getOutputDesc(payload),
          TestInput.getInputDesc(payload))));
  return dag;
}
开发者ID:apache，项目名称:tez，代码行数:20，代码来源:SimpleTestDAG.java

示例15: addDAGVerticesAndEdges

import org.apache.tez.dag.api.EdgeProperty.DataMovementType; //导入依赖的package包/类
protected static void addDAGVerticesAndEdges() {
    ThreeLevelsFailingDAG.addDAGVerticesAndEdges();
    l4v1 = Vertex.create("l4v1", TestProcessor.getProcDesc(payload), 10, defaultResource);
    dag.addVertex(l4v1);
    addEdge(l3v1, l4v1, DataMovementType.SCATTER_GATHER);
    addEdge(l3v2, l4v1, DataMovementType.SCATTER_GATHER);
    l5v1 = Vertex.create("l5v1", TestProcessor.getProcDesc(payload), 2, defaultResource);
    dag.addVertex(l5v1);
    addEdge(l4v1, l5v1, DataMovementType.SCATTER_GATHER);
    l5v2 = Vertex.create("l5v2", TestProcessor.getProcDesc(payload), 4, defaultResource);
    dag.addVertex(l5v2);
    addEdge(l4v1, l5v2, DataMovementType.SCATTER_GATHER);
    l5v3 = Vertex.create("l5v3", TestProcessor.getProcDesc(payload), 1, defaultResource);
    dag.addVertex(l5v3);
    addEdge(l4v1, l5v3, DataMovementType.SCATTER_GATHER);
    l6v1 = Vertex.create("l6v1", TestProcessor.getProcDesc(payload), 4, defaultResource);
    dag.addVertex(l6v1);
    addEdge(l5v1, l6v1, DataMovementType.SCATTER_GATHER);
    addEdge(l5v2, l6v1, DataMovementType.SCATTER_GATHER);
    addEdge(l5v3, l6v1, DataMovementType.SCATTER_GATHER);
}
开发者ID:apache，项目名称:tez，代码行数:22，代码来源:SixLevelsFailingDAG.java


注：本文中的org.apache.tez.dag.api.EdgeProperty.DataMovementType类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。