当前位置: 首页>>代码示例>>Java>>正文

Java MRPartitioner类代码示例

本文整理汇总了Java中org.apache.tez.mapreduce.partition.MRPartitioner类的典型用法代码示例。如果您正苦于以下问题：Java MRPartitioner类的具体用法？Java MRPartitioner怎么用？Java MRPartitioner使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。


MRPartitioner类属于org.apache.tez.mapreduce.partition包，在下文中一共展示了MRPartitioner类的9个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: setIntermediateOutputKeyValue

import org.apache.tez.mapreduce.partition.MRPartitioner; //导入依赖的package包/类
@SuppressWarnings("rawtypes")
private void setIntermediateOutputKeyValue(byte keyType, Configuration conf, TezOperator tezOp,
        boolean isConnectedToPackage) throws JobCreationException, ExecException {
    if (tezOp != null && tezOp.isUseSecondaryKey() && isConnectedToPackage) {
        conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS,
                NullableTuple.class.getName());
    } else if (tezOp != null && tezOp.isSkewedJoin() && isConnectedToPackage) {
        conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS,
                NullablePartitionWritable.class.getName());
    } else {
        Class<? extends WritableComparable> keyClass = HDataType
                .getWritableComparableTypes(keyType).getClass();
        conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS,
                keyClass.getName());

    }
    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS,
            NullableTuple.class.getName());
    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS,
            MRPartitioner.class.getName());
    selectOutputComparator(keyType, conf, tezOp);
}
开发者ID:sigmoidanalytics，项目名称:spork，代码行数:23，代码来源:TezDagBuilder.java

示例2: setupMRComponents

import org.apache.tez.mapreduce.partition.MRPartitioner; //导入依赖的package包/类
private static void setupMRComponents(Configuration conf) {
  if (conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS) == null) {
    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS,
        MRPartitioner.class.getName());
  }

  if (conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS) == null) {
    boolean useNewApi = conf.getBoolean("mapred.mapper.new-api", false);
    if (useNewApi) {
      if (conf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null) {
        conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
      }
    } else {
      if (conf.get("mapred.combiner.class") != null) {
        conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
      }
    }
  }
}
开发者ID:apache，项目名称:tez，代码行数:20，代码来源:MRHelpers.java

示例3: doJobClientMagic

import org.apache.tez.mapreduce.partition.MRPartitioner; //导入依赖的package包/类
/**
 * Sets up parameters which used to be set by the MR JobClient. Includes
 * setting whether to use the new api or the old api. Note: Must be called
 * before generating InputSplits
 *
 * @param conf
 *          configuration for the vertex.
 */
public static void doJobClientMagic(Configuration conf) throws IOException {
  setUseNewAPI(conf);
  // TODO Maybe add functionality to check output specifications - e.g. fail
  // early if the output directory exists.
  InetAddress ip = InetAddress.getLocalHost();
  if (ip != null) {
    String submitHostAddress = ip.getHostAddress();
    String submitHostName = ip.getHostName();
    conf.set(MRJobConfig.JOB_SUBMITHOST, submitHostName);
    conf.set(MRJobConfig.JOB_SUBMITHOSTADDR, submitHostAddress);
  }
  // conf.set("hadoop.http.filter.initializers",
  // "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer");
  // Skipping setting JOB_DIR - not used by AM.

  // Maybe generate SHUFFLE secret. The AM uses the job token generated in
  // the AM anyway.

  // TODO eventually ACLs
  conf.set(TezJobConfig.TEZ_RUNTIME_PARTITIONER_CLASS, MRPartitioner.class.getName());
  
  boolean useNewApi = conf.getBoolean("mapred.mapper.new-api", false);
  if (useNewApi) {
    if (conf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null) {
      conf.set(TezJobConfig.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
    }
  } else {
    if (conf.get("mapred.combiner.class") != null) {
      conf.set(TezJobConfig.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
    }
  }
  
  setWorkingDirectory(conf);
}
开发者ID:apache，项目名称:incubator-tez，代码行数:43，代码来源:MRHelpers.java

示例4: setUpJobConf

import org.apache.tez.mapreduce.partition.MRPartitioner; //导入依赖的package包/类
public void setUpJobConf(JobConf job) {
  job.set(TezRuntimeFrameworkConfigs.LOCAL_DIRS, workDir.toString());
  job.set(MRConfig.LOCAL_DIR, workDir.toString());
  job.setClass(
      Constants.TEZ_RUNTIME_TASK_OUTPUT_MANAGER,
      TezLocalTaskOutputFiles.class, 
      TezTaskOutput.class);
  job.set(TezJobConfig.TEZ_RUNTIME_PARTITIONER_CLASS, MRPartitioner.class.getName());
  job.setNumReduceTasks(1);
}
开发者ID:apache，项目名称:incubator-tez，代码行数:11，代码来源:TestMapProcessor.java

示例5: setUpJobConf

import org.apache.tez.mapreduce.partition.MRPartitioner; //导入依赖的package包/类
public void setUpJobConf(JobConf job) {
  job.set(TezRuntimeFrameworkConfigs.LOCAL_DIRS, workDir.toString());
  job.set(MRConfig.LOCAL_DIR, workDir.toString());
  job.setClass(
      Constants.TEZ_RUNTIME_TASK_OUTPUT_MANAGER,
      TezTaskOutputFiles.class,
      TezTaskOutput.class);
  job.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS, MRPartitioner.class.getName());
  job.setNumReduceTasks(1);
}
开发者ID:apache，项目名称:tez，代码行数:11，代码来源:TestMapProcessor.java

示例6: getStageConfs

import org.apache.tez.mapreduce.partition.MRPartitioner; //导入依赖的package包/类
/**
 * Given a single base MRR config, returns a list of complete stage
 * configurations.
 * 
 * @param conf
 * @return list of complete stage configurations given Conifiguration
 */
@Private
public static Configuration[] getStageConfs(Configuration conf) {
  int numIntermediateStages = MultiStageMRConfigUtil
      .getNumIntermediateStages(conf);
  boolean hasFinalReduceStage = (conf.getInt(MRJobConfig.NUM_REDUCES, 0) > 0);
  // Assuming no 0 map jobs, and the first stage is always a map.
  int numStages = numIntermediateStages + (hasFinalReduceStage ? 2 : 1);

  // Read split info from HDFS
  conf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false);
  
  // Setup Tez partitioner class
  conf.set(TezJobConfig.TEZ_RUNTIME_PARTITIONER_CLASS,
      MRPartitioner.class.getName());
  
  // Setup Tez Combiner class if required.
  // This would already have been set since the call is via JobClient
  boolean useNewApi = conf.getBoolean("mapred.mapper.new-api", false);
  if (useNewApi) {
    if (conf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null) {
      conf.set(TezJobConfig.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
    }
  } else {
    if (conf.get("mapred.combiner.class") != null) {
      conf.set(TezJobConfig.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
    }
  }

  Configuration confs[] = new Configuration[numStages];
  Configuration nonItermediateConf = MultiStageMRConfigUtil.extractStageConf(
      conf, "");
  if (numStages == 1) {
    confs[0] = nonItermediateConf;
    confs[0].setBoolean(MRConfig.IS_MAP_PROCESSOR, true);
  } else {
    confs[0] = nonItermediateConf;
    confs[numStages - 1] = new Configuration(nonItermediateConf);
    confs[numStages -1].setBoolean(MRConfig.IS_MAP_PROCESSOR, false);
  }
  if (numStages > 2) {
    for (int i = 1; i < numStages - 1; i++) {
      confs[i] = MultiStageMRConfigUtil.extractStageConf(conf,
          MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i, ""));
      confs[i].setBoolean(MRConfig.IS_MAP_PROCESSOR, false);
    }
  }
  return confs;
}
开发者ID:apache，项目名称:incubator-tez，代码行数:56，代码来源:MultiStageMRConfToTezTranslator.java

示例7: createDAG

import org.apache.tez.mapreduce.partition.MRPartitioner; //导入依赖的package包/类
private DAG createDAG(FileSystem fs, JobID jobId, Configuration[] stageConfs,
    String jobSubmitDir, Credentials ts,
    Map<String, LocalResource> jobLocalResources) throws IOException {

  String jobName = stageConfs[0].get(MRJobConfig.JOB_NAME,
      YarnConfiguration.DEFAULT_APPLICATION_NAME);
  DAG dag = new DAG(jobName);

  LOG.info("Number of stages: " + stageConfs.length);

  List<TaskLocationHint> mapInputLocations =
      getMapLocationHintsFromInputSplits(
          jobId, fs, stageConfs[0], jobSubmitDir);
  List<TaskLocationHint> reduceInputLocations = null;

  Vertex[] vertices = new Vertex[stageConfs.length];
  for (int i = 0; i < stageConfs.length; i++) {
    vertices[i] = createVertexForStage(stageConfs[i], jobLocalResources,
        i == 0 ? mapInputLocations : reduceInputLocations, i,
        stageConfs.length);
  }

  for (int i = 0; i < vertices.length; i++) {
    dag.addVertex(vertices[i]);
    if (i > 0) {
      // Set edge conf based on Input conf (compression etc properties for MapReduce are
      // w.r.t Outputs - MAP_OUTPUT_COMPRESS for example)
      OrderedPartitionedKVEdgeConfigurer edgeConf =
          OrderedPartitionedKVEdgeConfigurer.newBuilder(stageConfs[i - 1].get(
                  TezJobConfig.TEZ_RUNTIME_KEY_CLASS),
              stageConfs[i - 1].get(TezJobConfig.TEZ_RUNTIME_VALUE_CLASS),
              MRPartitioner.class.getName(), stageConfs[i - 1])
              .configureInput().useLegacyInput().done()
              .setFromConfiguration(stageConfs[i - 1]).build();
      Edge edge = new Edge(vertices[i-1], vertices[i], edgeConf.createDefaultEdgeProperty());
      dag.addEdge(edge);
    }

  }
  return dag;
}
开发者ID:apache，项目名称:incubator-tez，代码行数:42，代码来源:YARNRunner.java

示例8: getStageConfs

import org.apache.tez.mapreduce.partition.MRPartitioner; //导入依赖的package包/类
/**
 * Given a single base MRR config, returns a list of complete stage
 * configurations.
 * 
 * @param conf
 * @return list of complete stage configurations given Conifiguration
 */
@Private
public static Configuration[] getStageConfs(Configuration conf) {
  int numIntermediateStages = MultiStageMRConfigUtil
      .getNumIntermediateStages(conf);
  boolean hasFinalReduceStage = (conf.getInt(MRJobConfig.NUM_REDUCES, 0) > 0);
  // Assuming no 0 map jobs, and the first stage is always a map.
  int numStages = numIntermediateStages + (hasFinalReduceStage ? 2 : 1);

  // Read split info from HDFS
  conf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false);
  
  // Setup Tez partitioner class
  conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS,
      MRPartitioner.class.getName());
  
  // Setup Tez Combiner class if required.
  // This would already have been set since the call is via JobClient
  boolean useNewApi = conf.getBoolean("mapred.mapper.new-api", false);
  if (useNewApi) {
    if (conf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null) {
      conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
    }
  } else {
    if (conf.get("mapred.combiner.class") != null) {
      conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
    }
  }

  Configuration confs[] = new Configuration[numStages];
  Configuration nonItermediateConf = MultiStageMRConfigUtil.extractStageConf(
      conf, "");
  if (numStages == 1) {
    confs[0] = nonItermediateConf;
    confs[0].setBoolean(MRConfig.IS_MAP_PROCESSOR, true);
  } else {
    confs[0] = nonItermediateConf;
    confs[numStages - 1] = new Configuration(nonItermediateConf);
    confs[numStages -1].setBoolean(MRConfig.IS_MAP_PROCESSOR, false);
  }
  if (numStages > 2) {
    for (int i = 1; i < numStages - 1; i++) {
      confs[i] = MultiStageMRConfigUtil.extractStageConf(conf,
          MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i, ""));
      confs[i].setBoolean(MRConfig.IS_MAP_PROCESSOR, false);
    }
  }
  return confs;
}
开发者ID:apache，项目名称:tez，代码行数:56，代码来源:MultiStageMRConfToTezTranslator.java

示例9: createDAG

import org.apache.tez.mapreduce.partition.MRPartitioner; //导入依赖的package包/类
private DAG createDAG(FileSystem fs, JobID jobId, Configuration[] stageConfs,
    String jobSubmitDir, Credentials ts,
    Map<String, LocalResource> jobLocalResources) throws IOException {

  String jobName = stageConfs[0].get(MRJobConfig.JOB_NAME,
      YarnConfiguration.DEFAULT_APPLICATION_NAME);
  DAG dag = DAG.create(jobName);

  LOG.info("Number of stages: " + stageConfs.length);

  List<TaskLocationHint> mapInputLocations =
      getMapLocationHintsFromInputSplits(
          jobId, fs, stageConfs[0], jobSubmitDir);
  List<TaskLocationHint> reduceInputLocations = null;

  Vertex[] vertices = new Vertex[stageConfs.length];
  for (int i = 0; i < stageConfs.length; i++) {
    vertices[i] = createVertexForStage(stageConfs[i], jobLocalResources,
        i == 0 ? mapInputLocations : reduceInputLocations, i,
        stageConfs.length);
  }

  for (int i = 0; i < vertices.length; i++) {
    dag.addVertex(vertices[i]);
    if (i > 0) {
      // Set edge conf based on Input conf (compression etc properties for MapReduce are
      // w.r.t Outputs - MAP_OUTPUT_COMPRESS for example)
      Map<String, String> partitionerConf = null;
      if (stageConfs[i-1] != null) {
        partitionerConf = Maps.newHashMap();
        for (Map.Entry<String, String> entry : stageConfs[i - 1]) {
          partitionerConf.put(entry.getKey(), entry.getValue());
        }
      }
      OrderedPartitionedKVEdgeConfig edgeConf =
          OrderedPartitionedKVEdgeConfig.newBuilder(stageConfs[i - 1].get(
                  TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS),
              stageConfs[i - 1].get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS),
              MRPartitioner.class.getName(), partitionerConf)
              .setFromConfigurationUnfiltered(stageConfs[i-1])
              .configureInput().useLegacyInput().done()
              .build();
      Edge edge = Edge.create(vertices[i - 1], vertices[i], edgeConf.createDefaultEdgeProperty());
      dag.addEdge(edge);
    }

  }
  return dag;
}
开发者ID:apache，项目名称:tez，代码行数:50，代码来源:YARNRunner.java


注：本文中的org.apache.tez.mapreduce.partition.MRPartitioner类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。