Java InputPortMeta类代码示例

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
public boolean shouldCaptureEvent(T o)
{
  for (Entry<InputPortMeta, Collection<PartitionKeys>> entry : inputPortToPartitionMap.entrySet()) {
    StreamCodec<Object> codec = codecsToMerge.get(entry.getKey());
    Collection<PartitionKeys> partitionKeysList = entry.getValue();

    for (PartitionKeys keys : partitionKeysList) {
      if ( keys.partitions != null && keys.partitions.contains(keys.mask & codec.getPartition(o))) {
        // Then at least one of the partitions is getting this event
        // So send the event to persist operator
        return true;
      }
    }
  }

  return false;
}
 

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
private void setInput(PTOperator oper, InputPortMeta ipm, PTOperator sourceOper, PartitionKeys pks)
{
  // TODO: see if this can be handled more efficiently
  for (PTInput in : oper.inputs) {
    if (in.source.source == sourceOper && in.logicalStream == streamMeta && ipm.getPortName().equals(in.portName)) {
      return;
    }
  }
  // link to upstream output(s) for this stream
  for (PTOutput upstreamOut : sourceOper.outputs) {
    if (upstreamOut.logicalStream == streamMeta) {
      PTInput input = new PTInput(ipm.getPortName(), streamMeta, oper, pks, upstreamOut, ipm.getValue(LogicalPlan.IS_CONNECTED_TO_DELAY_OPERATOR));
      oper.inputs.add(input);
    }
  }
}
 

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
/**
 * Remove the given partition with any associated parallel partitions and
 * per-partition outputStreams.
 *
 * @param oper
 * @return
 */
private void removePartition(PTOperator oper, PMapping operatorMapping)
{
  // remove any parallel partition
  for (PTOutput out : oper.outputs) {
    // copy list as it is modified by recursive remove
    for (PTInput in : Lists.newArrayList(out.sinks)) {
      for (LogicalPlan.InputPortMeta im : in.logicalStream.getSinks()) {
        PMapping m = this.logicalToPTOperator.get(im.getOperatorMeta());
        if (m.parallelPartitions == operatorMapping.parallelPartitions) {
          // associated operator parallel partitioned
          removePartition(in.target, operatorMapping);
          m.partitions.remove(in.target);
        }
      }
    }
  }
  // remove the operator
  removePTOperator(oper);
}
 

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
private Set<PTOperator> getDependentPersistOperators(Collection<PTOperator> operators)
{
  Set<PTOperator> persistOperators = new LinkedHashSet<>();
  if (operators != null) {
    for (PTOperator operator : operators) {
      for (PTInput in : operator.inputs) {
        if (in.logicalStream.getPersistOperator() != null) {
          for (InputPortMeta inputPort : in.logicalStream.getSinksToPersist()) {
            if (inputPort.getOperatorMeta().equals(operator.operatorMeta)) {
              // Redeploy the stream wide persist operator only if the current sink is being persisted
              persistOperators.addAll(getOperators(in.logicalStream.getPersistOperator()));
              break;
            }
          }
        }
        for (Map.Entry<InputPortMeta, OperatorMeta> entry : in.logicalStream.sinkSpecificPersistOperatorMap.entrySet()) {
          // Redeploy sink specific persist operators
          persistOperators.addAll(getOperators(entry.getValue()));
        }
      }
    }
  }
  return persistOperators;
}
 

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
public StreamMeta addSinks(String id, Operator.InputPort<?>... sinks)
{
  StreamMeta sm = logicalPlan.getStream(id);
  if (sm == null) {
    throw new AssertionError("Stream " + id + " is not found!");
  }
  for (Operator.InputPort<?> sink : sinks) {
    sm.addSink(sink);
    if (physicalPlan != null) {
      for (InputPortMeta ipm : sm.getSinks()) {
        if (ipm.getPort() == sink) {
          physicalPlan.connectInput(ipm);
        }
      }
    }
  }
  return sm;
}
 

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
public static InputPortMeta getInputPortMeta(LogicalPlan.OperatorMeta operatorMeta, StreamMeta streamMeta)
{
  InputPortMeta inputPortMeta = null;
  Map<InputPortMeta, StreamMeta> inputStreams = operatorMeta.getInputStreams();
  for (Map.Entry<InputPortMeta, StreamMeta> entry : inputStreams.entrySet()) {
    if (entry.getValue() == streamMeta) {
      inputPortMeta = entry.getKey();
      break;
    }
  }
  return inputPortMeta;
}
 

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
public static InputPortMeta getIdentifyingInputPortMeta(PTOperator.PTInput input)
{
  InputPortMeta inputPortMeta;
  PTOperator inputTarget = input.target;
  StreamMeta streamMeta = input.logicalStream;
  if (!inputTarget.isUnifier()) {
    inputPortMeta = getInputPortMeta(inputTarget.getOperatorMeta(), streamMeta);
  } else {
    PTOperator destTarget = getIdentifyingOperator(inputTarget);
    inputPortMeta = getInputPortMeta(destTarget.getOperatorMeta(), streamMeta);
  }
  return inputPortMeta;
}
 

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
private void updatePersistOperatorWithSinkPartitions(InputPortMeta persistInputPort, OperatorMeta persistOperatorMeta, StreamCodecWrapperForPersistance<?> persistCodec, InputPortMeta sinkPortMeta)
{
  Collection<PTOperator> ptOperators = getOperators(sinkPortMeta.getOperatorMeta());
  Collection<PartitionKeys> partitionKeysList = new ArrayList<>();
  for (PTOperator p : ptOperators) {
    PartitionKeys keys = p.partitionKeys.get(sinkPortMeta);
    partitionKeysList.add(keys);
  }

  persistCodec.inputPortToPartitionMap.put(sinkPortMeta, partitionKeysList);
}
 

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
private List<InputPort<?>> getInputPortList(LogicalPlan.OperatorMeta operatorMeta)
{
  List<InputPort<?>> inputPortList = Lists.newArrayList();

  for (InputPortMeta inputPortMeta: operatorMeta.getInputStreams().keySet()) {
    inputPortList.add(inputPortMeta.getPort());
  }

  return inputPortList;
}
 

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
public Map<InputPort<?>, PartitionKeys> getPartitionKeys()
{
  Map<InputPort<?>, PartitionKeys> pkeys = null;
  if (partitionKeys != null) {
    pkeys = Maps.newHashMapWithExpectedSize(partitionKeys.size());
    for (Map.Entry<InputPortMeta, PartitionKeys> e : partitionKeys.entrySet()) {
      pkeys.put(e.getKey().getPort(), e.getValue());
    }
  }
  return pkeys;
}
 

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
private void printTopology(OperatorMeta operator, DAG tplg, int level)
{
  String prefix = "";
  if (level > 0) {
    prefix = StringUtils.repeat(" ", 20 * (level - 1)) + "   |" + StringUtils.repeat("-", 17);
  }
  logger.debug(prefix + operator.getName());
  for (StreamMeta downStream : operator.getOutputStreams().values()) {
    if (!downStream.getSinks().isEmpty()) {
      for (LogicalPlan.InputPortMeta targetNode : downStream.getSinks()) {
        printTopology(targetNode.getOperatorMeta(), tplg, level + 1);
      }
    }
  }
}
 

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
@Test
public void testTupleClassAttr() throws Exception
{
  String resourcePath = "/schemaTestTopology.json";
  InputStream is = this.getClass().getResourceAsStream(resourcePath);
  if (is == null) {
    fail("Could not load " + resourcePath);
  }
  StringWriter writer = new StringWriter();

  IOUtils.copy(is, writer);
  JSONObject json = new JSONObject(writer.toString());

  Configuration conf = new Configuration(false);

  LogicalPlanConfiguration planConf = new LogicalPlanConfiguration(conf);
  LogicalPlan dag = planConf.createFromJson(json, "testLoadFromJson");
  dag.validate();

  OperatorMeta operator1 = dag.getOperatorMeta("operator1");
  assertEquals("operator1.classname", SchemaTestOperator.class, operator1.getOperator().getClass());

  StreamMeta input1 = dag.getStream("inputStream");
  assertNotNull(input1);
  for (LogicalPlan.InputPortMeta targetPort : input1.getSinks()) {
    Assert.assertEquals("tuple class name required", TestSchema.class, targetPort.getAttributes().get(PortContext.TUPLE_CLASS));
  }
}
 

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
private void simpleNameInputPortAssertHelper(Attribute<?> attributeObj, LogicalPlan dag, String operatorName, Object queueCapacity, boolean set)
{
  OperatorMeta operatorMeta = dag.getOperatorMeta(operatorName);

  for (InputPortMeta inputPortMeta: operatorMeta.getInputStreams().keySet()) {
    if (set) {
      Assert.assertEquals(queueCapacity, inputPortMeta.getValue(attributeObj));
    } else {
      Assert.assertNotEquals(queueCapacity, inputPortMeta.getValue(attributeObj));
    }
  }
}
 

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
public static PropertiesConfiguration convertToProperties(LogicalPlan dag)
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  Collection<OperatorMeta> allOperators = dag.getAllOperators();

  for (OperatorMeta operatorMeta : allOperators) {
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
    Operator operator = operatorMeta.getOperator();
    props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
    BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
    @SuppressWarnings("rawtypes")
    Iterator entryIterator = operatorProperties.entryIterator();
    while (entryIterator.hasNext()) {
      try {
        @SuppressWarnings("unchecked")
        Map.Entry<String, Object> entry = (Map.Entry<String, Object>)entryIterator.next();
        if (!entry.getKey().equals("class") && !entry.getKey().equals("name") && entry.getValue() != null) {
          props.setProperty(operatorKey + "." + entry.getKey(), entry.getValue());
        }
      } catch (Exception ex) {
        LOG.warn("Error trying to get a property of operator {}", operatorMeta.getName(), ex);
      }
    }
  }
  Collection<StreamMeta> allStreams = dag.getAllStreams();

  for (StreamMeta streamMeta : allStreams) {
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamMeta.getName();
    OutputPortMeta source = streamMeta.getSource();
    Collection<InputPortMeta> sinks = streamMeta.getSinks();
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, source.getOperatorMeta().getName() + "." + source.getPortName());
    String sinksValue = "";
    for (InputPortMeta sink : sinks) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sink.getOperatorMeta().getName() + "." + sink.getPortName();
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    if (streamMeta.getLocality() != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, streamMeta.getLocality().name());
    }
  }

  // TBD: Attributes

  return props;
}
 

import com.datatorrent.stram.plan.logical.LogicalPlan.InputPortMeta; //导入依赖的package包/类
public StreamCodecWrapperForPersistance(Map<InputPortMeta, StreamCodec<Object>> inputStreamCodecs, StreamCodec<Object> specifiedStreamCodec)
{
  this.codecsToMerge = inputStreamCodecs;
  this.setSpecifiedStreamCodec(specifiedStreamCodec);
  inputPortToPartitionMap = new HashMap<>();
}