Java DataMovementType.ONE_TO_ONE属性代码示例

@Test(timeout = 5000)
public void testVerifyOneToOne() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

@Test(timeout = 5000)
public void testVerifyOneToOneNoInferParallelism() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      -1, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
  Assert.assertEquals(-1, v2.getParallelism());
}
 

static public void configureValueOnlyTupleOutput(TezEdgeDescriptor edge, DataMovementType dataMovementType) {
    edge.dataMovementType = dataMovementType;
    if (dataMovementType == DataMovementType.BROADCAST || dataMovementType == DataMovementType.ONE_TO_ONE) {
        edge.outputClassName = UnorderedKVOutput.class.getName();
        edge.inputClassName = UnorderedKVInput.class.getName();
    } else if (dataMovementType == DataMovementType.SCATTER_GATHER) {
        edge.outputClassName = UnorderedPartitionedKVOutput.class.getName();
        edge.inputClassName = UnorderedKVInput.class.getName();
        edge.partitionerClass = RoundRobinPartitioner.class;
    }
    edge.setIntermediateOutputKeyClass(POValueOutputTez.EmptyWritable.class.getName());
    edge.setIntermediateOutputValueClass(TUPLE_CLASS);
}
 

public static DataMovementType convertFromDAGPlan(PlanEdgeDataMovementType type){
  switch(type){
    case ONE_TO_ONE : return DataMovementType.ONE_TO_ONE;
    case BROADCAST : return DataMovementType.BROADCAST;
    case SCATTER_GATHER : return DataMovementType.SCATTER_GATHER;
    default : throw new IllegalArgumentException("unknown 'dataMovementType': " + type);
  }
}
 

@Test(timeout = 5000)
// v1 (known) -> v2 (-1) -> v3 (-1)
public void testVerifyOneToOneInferParallelism() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  Edge e2 = new Edge(v2, v3,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addEdge(e1);
  dag.addEdge(e2);
  dag.verify();
  Assert.assertEquals(dummyTaskCount, v2.getParallelism());
  Assert.assertEquals(dummyTaskCount, v3.getParallelism());
}
 

@Test(timeout = 5000)
// v1 (known) -> v2 (-1) -> v3 (-1)
// The test checks resiliency to ordering of the vertices/edges
public void testVerifyOneToOneInferParallelismReverseOrder() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  Edge e2 = new Edge(v2, v3,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v3);
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e2);
  dag.addEdge(e1);
  dag.verify();
  Assert.assertEquals(dummyTaskCount, v2.getParallelism());
  Assert.assertEquals(dummyTaskCount, v3.getParallelism());
}
 

@Test(timeout = 5000)
// v1 (-1) -> v2 (known) -> v3 (-1)
public void testVerifyOneToOneIncorrectParallelism1() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      -1, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("MapProcessor"),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v3,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  Edge e2 = new Edge(v2, v3,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addEdge(e1);
  dag.addEdge(e2);
  try {
    dag.verify();
    Assert.assertTrue(false);
  } catch (TezUncheckedException e) {
    Assert.assertTrue(e.getMessage().contains(
        "1-1 Edge. Destination vertex parallelism must match source vertex"));
  }
}
 

public static DataMovementType convertFromDAGPlan(PlanEdgeDataMovementType type){
  switch(type){
    case ONE_TO_ONE : return DataMovementType.ONE_TO_ONE;
    case BROADCAST : return DataMovementType.BROADCAST;
    case SCATTER_GATHER : return DataMovementType.SCATTER_GATHER;
    case CUSTOM : return DataMovementType.CUSTOM;
    default : throw new IllegalArgumentException("unknown 'dataMovementType': " + type);
  }
}
 

@Override
public int estimateParallelism(TezOperPlan plan, TezOperator tezOper, Configuration conf) throws IOException {

    if (tezOper.isVertexGroup()) {
        return -1;
    }

    boolean intermediateReducer = TezCompilerUtil.isIntermediateReducer(tezOper);

    // TODO: If map opts and reduce opts are same estimate higher parallelism
    // for tasks based on the count of number of map tasks else be conservative as now
    maxTaskCount = conf.getInt(PigReducerEstimator.MAX_REDUCER_COUNT_PARAM,
            PigReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM);

    // If parallelism is set explicitly, respect it
    if (!intermediateReducer && tezOper.getRequestedParallelism()!=-1) {
        return tezOper.getRequestedParallelism();
    }

    // If we have already estimated parallelism, use that one
    if (tezOper.getEstimatedParallelism()!=-1) {
        return tezOper.getEstimatedParallelism();
    }

    List<TezOperator> preds = plan.getPredecessors(tezOper);
    if (preds==null) {
        throw new IOException("Cannot estimate parallelism for source vertex");
    }

    double estimatedParallelism = 0;

    for (Entry<OperatorKey, TezEdgeDescriptor> entry : tezOper.inEdges.entrySet()) {
        TezOperator pred = getPredecessorWithKey(plan, tezOper, entry.getKey().toString());

        // Don't include broadcast edge, broadcast edge is used for
        // replicated join (covered in TezParallelismFactorVisitor.visitFRJoin)
        // and sample/scalar (does not impact parallelism)
        if (entry.getValue().dataMovementType==DataMovementType.SCATTER_GATHER ||
                entry.getValue().dataMovementType==DataMovementType.ONE_TO_ONE) {
            double predParallelism = pred.getEffectiveParallelism();
            if (predParallelism==-1) {
                throw new IOException("Cannot estimate parallelism for " + tezOper.getOperatorKey().toString()
                        + ", effective parallelism for predecessor " + tezOper.getOperatorKey().toString()
                        + " is -1");
            }

            //For cases like Union we can just limit to sum of pred vertices parallelism
            boolean applyFactor = !tezOper.isUnion();
            if (pred.plan!=null && applyFactor) { // pred.plan can be null if it is a VertexGroup
                TezParallelismFactorVisitor parallelismFactorVisitor = new TezParallelismFactorVisitor(pred.plan, tezOper.getOperatorKey().toString());
                parallelismFactorVisitor.visit();
                predParallelism = predParallelism * parallelismFactorVisitor.getFactor();
            }
            estimatedParallelism += predParallelism;
        }
    }

    int roundedEstimatedParallelism = (int)Math.ceil(estimatedParallelism);

    if (intermediateReducer && tezOper.isOverrideIntermediateParallelism()) {
        // Estimated reducers should not be more than the configured limit
        roundedEstimatedParallelism = Math.min(roundedEstimatedParallelism, maxTaskCount);
        int userSpecifiedParallelism = pc.defaultParallel;
        if (tezOper.getRequestedParallelism() != -1) {
            userSpecifiedParallelism = tezOper.getRequestedParallelism();
        }
        int intermediateParallelism = Math.max(userSpecifiedParallelism, roundedEstimatedParallelism);
        if (userSpecifiedParallelism != -1 &&
                (intermediateParallelism > 200 && intermediateParallelism > (2 * userSpecifiedParallelism))) {
            // Estimated reducers shall not be more than 2x of requested parallelism
            // if greater than 200 and we are overriding user specified values
            intermediateParallelism = 2 * userSpecifiedParallelism;
        }
        roundedEstimatedParallelism = intermediateParallelism;
    } else {
        roundedEstimatedParallelism = Math.min(roundedEstimatedParallelism, maxTaskCount);
    }

    return roundedEstimatedParallelism;
}
 

@Test(timeout = 5000)
// v1 (-1) -> v3 (-1), v2 (known) -> v3 (-1)
// order of edges should not matter
public void testVerifyOneToOneIncorrectParallelism2() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      -1, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor(dummyProcessorClassName),
      -1, dummyTaskResource);
  Vertex v4 = new Vertex("v4",
      new ProcessorDescriptor(dummyProcessorClassName),
      -1, dummyTaskResource);
  Edge e1 = new Edge(v1, v4,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  Edge e2 = new Edge(v2, v4,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  Edge e3 = new Edge(v3, v4,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addVertex(v4);
  dag.addEdge(e1);
  dag.addEdge(e2);
  dag.addEdge(e3);
  try {
    dag.verify();
    Assert.assertTrue(false);
  } catch (TezUncheckedException e) {
    Assert.assertTrue(e.getMessage().contains(
        "1-1 Edge. Destination vertex parallelism must match source vertex"));
  }
}
 

@Test(timeout = 5000)
public void testVertexGroupOneToOne() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v4 = new Vertex("v4",
      new ProcessorDescriptor("Processor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v5 = new Vertex("v5",
      new ProcessorDescriptor("Processor"),
      -1, dummyTaskResource);
  
  DAG dag = new DAG("testDag");
  String groupName1 = "uv12";
  VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
  OutputDescriptor outDesc = new OutputDescriptor();
  uv12.addOutput("uvOut", outDesc, null);
  
  String groupName2 = "uv23";
  VertexGroup uv23 = dag.createVertexGroup(groupName2, v2, v3);
  
  GroupInputEdge e1 = new GroupInputEdge(uv12, v4,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")),
          new InputDescriptor("dummy input class"));
  GroupInputEdge e2 = new GroupInputEdge(uv23, v5,
      new EdgeProperty(DataMovementType.ONE_TO_ONE, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")),
          new InputDescriptor("dummy input class"));
  
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addVertex(v4);
  dag.addVertex(v5);
  dag.addEdge(e1);
  dag.addEdge(e2);
  dag.verify();
  Assert.assertEquals(dummyTaskCount, v5.getParallelism());
}