Java DataMovementType.SCATTER_GATHER属性代码示例

@Test(timeout = 5000)
public void testVerifyScatterGather() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

@Test(expected = IllegalStateException.class, timeout = 5000)  
public void testVerify3() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor(dummyProcessorClassName),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.EPHEMERAL, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor(dummyOutputClassName),
          new InputDescriptor(dummyInputClassName)));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

@Test(expected = IllegalStateException.class, timeout = 5000)
public void testInputAndInputVertexNameCollision() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  
  v2.addInput("v1", new InputDescriptor(), null);
  
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

@Test(expected = IllegalStateException.class, timeout = 5000)
public void testOutputAndOutputVertexNameCollision() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  
  v1.addOutput("v2", new OutputDescriptor());
  
  Edge e1 = new Edge(v1, v2,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addEdge(e1);
  dag.verify();
}
 

@Override
public synchronized void onSourceTaskCompleted(TaskAttemptIdentifier attempt) {
  String srcVertexName = attempt.getTaskIdentifier().getVertexIdentifier().getName();
  int srcTaskId = attempt.getTaskIdentifier().getIdentifier();
  SourceVertexInfo srcInfo = srcVertexInfo.get(srcVertexName);
  if (srcInfo.vertexIsConfigured) {
    Preconditions.checkState(srcTaskId < srcInfo.numTasks,
        "Received completion for srcTaskId " + srcTaskId + " but Vertex: " + srcVertexName +
        " has only " + srcInfo.numTasks + " tasks");
  }
  //handle duplicate events and count task completions from all source vertices
  BitSet completedSourceTasks = srcInfo.finishedTaskSet;
  // duplicate notifications tracking
  if (!completedSourceTasks.get(srcTaskId)) {
    completedSourceTasks.set(srcTaskId);
    // source task has completed
    if (srcInfo.edgeProperty.getDataMovementType() == DataMovementType.SCATTER_GATHER) {
      numBipartiteSourceTasksCompleted++;
    }
  }
  processPendingTasks(attempt);
}
 

private void handleVertexStateUpdate(VertexStateUpdate stateUpdate) {
  Preconditions.checkArgument(stateUpdate.getVertexState() == VertexState.CONFIGURED,
      "Received incorrect state notification : " + stateUpdate.getVertexState() + " for vertex: "
          + stateUpdate.getVertexName() + " in vertex: " + getContext().getVertexName());
  Preconditions.checkArgument(srcVertexInfo.containsKey(stateUpdate.getVertexName()),
      "Received incorrect vertex notification : " + stateUpdate.getVertexState() + " for vertex: "
          + stateUpdate.getVertexName() + " in vertex: " + getContext().getVertexName());
  SourceVertexInfo vInfo = srcVertexInfo.get(stateUpdate.getVertexName());
  Preconditions.checkState(vInfo.vertexIsConfigured == false);
  vInfo.vertexIsConfigured = true;
  vInfo.numTasks = getContext().getVertexNumTasks(stateUpdate.getVertexName());
  if (vInfo.edgeProperty.getDataMovementType() == DataMovementType.SCATTER_GATHER) {
    totalNumBipartiteSourceTasks += vInfo.numTasks;
  }
  LOG.info("Received configured notification : {}" + " for vertex: {} in" +
      " vertex: {}" + " numBipartiteSourceTasks: {}",
      stateUpdate.getVertexState(), stateUpdate.getVertexName(),
      getContext().getVertexName(), totalNumBipartiteSourceTasks);
  processPendingTasks(null);
}
 

static public void configureValueOnlyTupleOutput(TezEdgeDescriptor edge, DataMovementType dataMovementType) {
    edge.dataMovementType = dataMovementType;
    if (dataMovementType == DataMovementType.BROADCAST || dataMovementType == DataMovementType.ONE_TO_ONE) {
        edge.outputClassName = UnorderedKVOutput.class.getName();
        edge.inputClassName = UnorderedKVInput.class.getName();
    } else if (dataMovementType == DataMovementType.SCATTER_GATHER) {
        edge.outputClassName = UnorderedPartitionedKVOutput.class.getName();
        edge.inputClassName = UnorderedKVInput.class.getName();
        edge.partitionerClass = RoundRobinPartitioner.class;
    }
    edge.setIntermediateOutputKeyClass(POValueOutputTez.EmptyWritable.class.getName());
    edge.setIntermediateOutputValueClass(TUPLE_CLASS);
}
 

public TezEdgeDescriptor() {
    combinePlan = new PhysicalPlan();

    // The default is shuffle edge.
    inputClassName = OrderedGroupedKVInput.class.getName();
    outputClassName = OrderedPartitionedKVOutput.class.getName();
    partitionerClass = null;
    schedulingType = SchedulingType.SEQUENTIAL;
    dataSourceType = DataSourceType.PERSISTED;
    dataMovementType = DataMovementType.SCATTER_GATHER;
}
 

public static DataMovementType convertFromDAGPlan(PlanEdgeDataMovementType type){
  switch(type){
    case ONE_TO_ONE : return DataMovementType.ONE_TO_ONE;
    case BROADCAST : return DataMovementType.BROADCAST;
    case SCATTER_GATHER : return DataMovementType.SCATTER_GATHER;
    default : throw new IllegalArgumentException("unknown 'dataMovementType': " + type);
  }
}
 

public static DataMovementType convertFromDAGPlan(PlanEdgeDataMovementType type){
  switch(type){
    case ONE_TO_ONE : return DataMovementType.ONE_TO_ONE;
    case BROADCAST : return DataMovementType.BROADCAST;
    case SCATTER_GATHER : return DataMovementType.SCATTER_GATHER;
    case CUSTOM : return DataMovementType.CUSTOM;
    default : throw new IllegalArgumentException("unknown 'dataMovementType': " + type);
  }
}
 

@Test(timeout = 5000)
public void BinaryInputAllowed() {
  Vertex v1 = new Vertex("v1",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v2 = new Vertex("v2",
      new ProcessorDescriptor("MapProcessor"),
      dummyTaskCount, dummyTaskResource);
  Vertex v3 = new Vertex("v3",
      new ProcessorDescriptor("ReduceProcessor"),
      dummyTaskCount, dummyTaskResource);
  Edge e1 = new Edge(v1, v3,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  Edge e2 = new Edge(v2, v3,
      new EdgeProperty(DataMovementType.SCATTER_GATHER, 
          DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
          new OutputDescriptor("dummy output class"),
          new InputDescriptor("dummy input class")));
  DAG dag = new DAG("testDag");
  dag.addVertex(v1);
  dag.addVertex(v2);
  dag.addVertex(v3);
  dag.addEdge(e1);
  dag.addEdge(e2);
  dag.verify();
}
 

@Test(timeout = 5000)
public void BinaryOutput() {
  IllegalStateException ex = null;
  try {
    Vertex v1 = new Vertex("v1",
        new ProcessorDescriptor("MapProcessor"),
        dummyTaskCount, dummyTaskResource);
    Vertex v2 = new Vertex("v2",
        new ProcessorDescriptor("MapProcessor"),
        dummyTaskCount, dummyTaskResource);
    Vertex v3 = new Vertex("v3",
        new ProcessorDescriptor("MapProcessor"),
        dummyTaskCount, dummyTaskResource);
    Edge e1 = new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER, 
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
            new OutputDescriptor("dummy output class"),
            new InputDescriptor("dummy input class")));
    Edge e2 = new Edge(v1, v2,
        new EdgeProperty(DataMovementType.SCATTER_GATHER, 
            DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, 
            new OutputDescriptor("dummy output class"),
            new InputDescriptor("dummy input class")));
    DAG dag = new DAG("testDag");
    dag.addVertex(v1);
    dag.addVertex(v2);
    dag.addVertex(v3);
    dag.addEdge(e1);
    dag.addEdge(e2);
    dag.verify();
  }
  catch (IllegalStateException e){
    ex = e;
  }
  Assert.assertNull(ex);
}
 

@Override
public int estimateParallelism(TezOperPlan plan, TezOperator tezOper, Configuration conf) throws IOException {

    if (tezOper.isVertexGroup()) {
        return -1;
    }

    boolean intermediateReducer = TezCompilerUtil.isIntermediateReducer(tezOper);

    // TODO: If map opts and reduce opts are same estimate higher parallelism
    // for tasks based on the count of number of map tasks else be conservative as now
    maxTaskCount = conf.getInt(PigReducerEstimator.MAX_REDUCER_COUNT_PARAM,
            PigReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM);

    // If parallelism is set explicitly, respect it
    if (!intermediateReducer && tezOper.getRequestedParallelism()!=-1) {
        return tezOper.getRequestedParallelism();
    }

    // If we have already estimated parallelism, use that one
    if (tezOper.getEstimatedParallelism()!=-1) {
        return tezOper.getEstimatedParallelism();
    }

    List<TezOperator> preds = plan.getPredecessors(tezOper);
    if (preds==null) {
        throw new IOException("Cannot estimate parallelism for source vertex");
    }

    double estimatedParallelism = 0;

    for (Entry<OperatorKey, TezEdgeDescriptor> entry : tezOper.inEdges.entrySet()) {
        TezOperator pred = getPredecessorWithKey(plan, tezOper, entry.getKey().toString());

        // Don't include broadcast edge, broadcast edge is used for
        // replicated join (covered in TezParallelismFactorVisitor.visitFRJoin)
        // and sample/scalar (does not impact parallelism)
        if (entry.getValue().dataMovementType==DataMovementType.SCATTER_GATHER ||
                entry.getValue().dataMovementType==DataMovementType.ONE_TO_ONE) {
            double predParallelism = pred.getEffectiveParallelism();
            if (predParallelism==-1) {
                throw new IOException("Cannot estimate parallelism for " + tezOper.getOperatorKey().toString()
                        + ", effective parallelism for predecessor " + tezOper.getOperatorKey().toString()
                        + " is -1");
            }

            //For cases like Union we can just limit to sum of pred vertices parallelism
            boolean applyFactor = !tezOper.isUnion();
            if (pred.plan!=null && applyFactor) { // pred.plan can be null if it is a VertexGroup
                TezParallelismFactorVisitor parallelismFactorVisitor = new TezParallelismFactorVisitor(pred.plan, tezOper.getOperatorKey().toString());
                parallelismFactorVisitor.visit();
                predParallelism = predParallelism * parallelismFactorVisitor.getFactor();
            }
            estimatedParallelism += predParallelism;
        }
    }

    int roundedEstimatedParallelism = (int)Math.ceil(estimatedParallelism);

    if (intermediateReducer && tezOper.isOverrideIntermediateParallelism()) {
        // Estimated reducers should not be more than the configured limit
        roundedEstimatedParallelism = Math.min(roundedEstimatedParallelism, maxTaskCount);
        int userSpecifiedParallelism = pc.defaultParallel;
        if (tezOper.getRequestedParallelism() != -1) {
            userSpecifiedParallelism = tezOper.getRequestedParallelism();
        }
        int intermediateParallelism = Math.max(userSpecifiedParallelism, roundedEstimatedParallelism);
        if (userSpecifiedParallelism != -1 &&
                (intermediateParallelism > 200 && intermediateParallelism > (2 * userSpecifiedParallelism))) {
            // Estimated reducers shall not be more than 2x of requested parallelism
            // if greater than 200 and we are overriding user specified values
            intermediateParallelism = 2 * userSpecifiedParallelism;
        }
        roundedEstimatedParallelism = intermediateParallelism;
    } else {
        roundedEstimatedParallelism = Math.min(roundedEstimatedParallelism, maxTaskCount);
    }

    return roundedEstimatedParallelism;
}
 

@Override
public void initialize(VertexManagerPluginContext context) {
  Configuration conf;
  try {
    conf = TezUtils.createConfFromUserPayload(context.getUserPayload());
  } catch (IOException e) {
    throw new TezUncheckedException(e);
  }
  
  this.context = context;

  this.slowStartMinSrcCompletionFraction = conf
      .getFloat(
          ShuffleVertexManager.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION,
          ShuffleVertexManager.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION_DEFAULT);
  this.slowStartMaxSrcCompletionFraction = conf
      .getFloat(
          ShuffleVertexManager.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION,
          ShuffleVertexManager.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT);

  if (slowStartMinSrcCompletionFraction < 0
      || slowStartMaxSrcCompletionFraction < slowStartMinSrcCompletionFraction) {
    throw new IllegalArgumentException(
        "Invalid values for slowStartMinSrcCompletionFraction"
            + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0 and "
            + "max cannot be < min.");
  }

  enableAutoParallelism = conf
      .getBoolean(
          ShuffleVertexManager.TEZ_AM_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL,
          ShuffleVertexManager.TEZ_AM_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL_DEFAULT);
  desiredTaskInputDataSize = conf
      .getLong(
          ShuffleVertexManager.TEZ_AM_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE,
          ShuffleVertexManager.TEZ_AM_SHUFFLE_VERTEX_MANAGER_DESIRED_TASK_INPUT_SIZE_DEFAULT);
  minTaskParallelism = Math.max(1, conf
      .getInt(
          ShuffleVertexManager.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM,
          ShuffleVertexManager.TEZ_AM_SHUFFLE_VERTEX_MANAGER_MIN_TASK_PARALLELISM_DEFAULT));
  LOG.info("Shuffle Vertex Manager: settings" + " minFrac:"
      + slowStartMinSrcCompletionFraction + " maxFrac:"
      + slowStartMaxSrcCompletionFraction + " auto:" + enableAutoParallelism
      + " desiredTaskIput:" + desiredTaskInputDataSize + " minTasks:"
      + minTaskParallelism);
  
  Map<String, EdgeProperty> inputs = context.getInputVertexEdgeProperties();
  for(Map.Entry<String, EdgeProperty> entry : inputs.entrySet()) {
    if (entry.getValue().getDataMovementType() == DataMovementType.SCATTER_GATHER) {
      String vertex = entry.getKey();
      bipartiteSources.put(vertex, new HashSet<Integer>());
    }
  }
  if(bipartiteSources.isEmpty()) {
    throw new TezUncheckedException("Atleast 1 bipartite source should exist");
  }
  // dont track the source tasks here since those tasks may themselves be
  // dynamically changed as the DAG progresses.

}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test (timeout = 5000)
public void testCompositeEventHandling() {
  EventHandler eventHandler = mock(EventHandler.class);
  EdgeProperty edgeProp = new EdgeProperty(DataMovementType.SCATTER_GATHER,
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, mock(OutputDescriptor.class),
      mock(InputDescriptor.class));
  Edge edge = new Edge(edgeProp, eventHandler);
  
  TezVertexID srcVertexID = createVertexID(1);
  TezVertexID destVertexID = createVertexID(2);
  LinkedHashMap<TezTaskID, Task> srcTasks = mockTasks(srcVertexID, 1);
  LinkedHashMap<TezTaskID, Task> destTasks = mockTasks(destVertexID, 5);
  
  TezTaskID srcTaskID = srcTasks.keySet().iterator().next();
  
  Vertex srcVertex = mockVertex("src", srcVertexID, srcTasks);
  Vertex destVertex = mockVertex("dest", destVertexID, destTasks);
  
  edge.setSourceVertex(srcVertex);
  edge.setDestinationVertex(destVertex);
  edge.initialize();
  
  TezTaskAttemptID srcTAID = createTAIDForTest(srcTaskID, 2); // Task0, Attempt 0
  
  EventMetaData srcMeta = new EventMetaData(EventProducerConsumerType.OUTPUT, "consumerVertex", "producerVertex", srcTAID);
  
  // Verification via a CompositeEvent
  CompositeDataMovementEvent cdmEvent = new CompositeDataMovementEvent(0, destTasks.size(), "bytes".getBytes());
  cdmEvent.setVersion(2); // AttemptNum
  TezEvent tezEvent = new TezEvent(cdmEvent, srcMeta);
  // Event setup to look like it would after the Vertex is done with it.

  edge.sendTezEventToDestinationTasks(tezEvent);
  
  ArgumentCaptor<Event> args = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(destTasks.size())).handle(args.capture());
  
  verifyEvents(args.getAllValues(), srcTAID, destTasks);
  
  
  // Same Verification via regular DataMovementEvents
  reset(eventHandler);
  for (int i = 0 ; i < destTasks.size() ; i++) {
    DataMovementEvent dmEvent = new DataMovementEvent(i, "bytes".getBytes());
    dmEvent.setVersion(2);
    tezEvent = new TezEvent(dmEvent, srcMeta);
    edge.sendTezEventToDestinationTasks(tezEvent);
  }
  args = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(destTasks.size())).handle(args.capture());
  
  verifyEvents(args.getAllValues(), srcTAID, destTasks);

}