Java Locality.THREAD_LOCAL属性代码示例

@Test
public void testParDoChaining() throws Exception {
  Pipeline p = Pipeline.create();
  long numElements = 1000;
  PCollection<Long> input = p.apply(GenerateSequence.from(0).to(numElements));
  PAssert.thatSingleton(input.apply("Count", Count.<Long>globally())).isEqualTo(numElements);

  ApexPipelineOptions options = PipelineOptionsFactory.as(ApexPipelineOptions.class);
  DAG dag = TestApexRunner.translate(p, options);

  String[] expectedThreadLocal = { "/CreateActual/FilterActuals/Window.Assign" };
  Set<String> actualThreadLocal = Sets.newHashSet();
  for (DAG.StreamMeta sm : dag.getAllStreamsMeta()) {
    DAG.OutputPortMeta opm = sm.getSource();
    if (sm.getLocality() == Locality.THREAD_LOCAL) {
       String name = opm.getOperatorMeta().getName();
       String prefix = "PAssert$";
       if (name.startsWith(prefix)) {
         // remove indeterministic prefix
         name = name.substring(prefix.length() + 1);
       }
       actualThreadLocal.add(name);
    }
  }
  Assert.assertThat(actualThreadLocal, Matchers.hasItems(expectedThreadLocal));
}
 

public Set<PTOperator> threadLocalSinks()
{
  Set<PTOperator> threadLocalOperators = null;
  if (logicalStream != null && logicalStream.getLocality() == Locality.THREAD_LOCAL) {
    threadLocalOperators = new HashSet<>();
    for (PTInput sink : this.sinks) {
      threadLocalOperators.add(sink.target);
    }
  }
  return threadLocalOperators;
}
 

public Set<PTOperator> getThreadLocalOperators()
{
  Set<PTOperator> threadLocalOperators = null;
  for (int i = 0; i < outputs.size(); i++) {
    if (outputs.get(i).logicalStream != null && outputs.get(i).logicalStream.getLocality() == Locality.THREAD_LOCAL) {
      if (threadLocalOperators == null) {
        threadLocalOperators = new HashSet<>();
      }
      threadLocalOperators.addAll(outputs.get(i).threadLocalSinks());
    }
  }
  return threadLocalOperators;
}
 

/**
 * Add logical operator to the plan. Assumes that upstream operators have been added before.
 * @param om
 */
public final void addLogicalOperator(OperatorMeta om)
{
  PMapping pnodes = new PMapping(om);
  String host = pnodes.logicalOperator.getValue(OperatorContext.LOCALITY_HOST);
  localityPrefs.add(pnodes, host);

  PMapping upstreamPartitioned = null;

  for (Map.Entry<LogicalPlan.InputPortMeta, StreamMeta> e : om.getInputStreams().entrySet()) {
    if (e.getValue().getSource().getOperatorMeta().getOperator() instanceof Operator.DelayOperator) {
      continue;
    }
    PMapping m = logicalToPTOperator.get(e.getValue().getSource().getOperatorMeta());
    if (e.getKey().getValue(PortContext.PARTITION_PARALLEL).equals(true)) {
      // operator partitioned with upstream
      if (upstreamPartitioned != null) {
        // need to have common root
        if (!upstreamPartitioned.parallelPartitions.contains(m.logicalOperator) && upstreamPartitioned != m) {
          String msg = String.format("operator cannot extend multiple partitions (%s and %s)", upstreamPartitioned.logicalOperator, m.logicalOperator);
          throw new AssertionError(msg);
        }
      }
      m.parallelPartitions.add(pnodes.logicalOperator);
      pnodes.parallelPartitions = m.parallelPartitions;
      upstreamPartitioned = m;
    }

    if (Locality.CONTAINER_LOCAL == e.getValue().getLocality() || Locality.THREAD_LOCAL == e.getValue().getLocality()) {
      inlinePrefs.setLocal(m, pnodes);
    } else if (Locality.NODE_LOCAL == e.getValue().getLocality()) {
      localityPrefs.setLocal(m, pnodes);
    }
  }

  //
  // create operator instances
  //
  this.logicalToPTOperator.put(om, pnodes);
  if (upstreamPartitioned != null) {
    // parallel partition
    //LOG.debug("Operator {} should be partitioned to {} partitions", pnodes.logicalOperator.getName(), upstreamPartitioned.partitions.size());
    initPartitioning(pnodes, upstreamPartitioned.partitions.size());
  } else {
    initPartitioning(pnodes, 0);
  }
  updateStreamMappings(pnodes);
}
 

@Override
public Locality getLocality()
{
  return Locality.THREAD_LOCAL;
}