Java MRCombiner类代码示例

import org.apache.tez.mapreduce.combine.MRCombiner; //导入依赖的package包/类
private void addCombiner(PhysicalPlan combinePlan, TezOperator pkgTezOp,
        Configuration conf) throws IOException {
    POPackage combPack = (POPackage) combinePlan.getRoots().get(0);
    POLocalRearrange combRearrange = (POLocalRearrange) combinePlan
            .getLeaves().get(0);
    setIntermediateOutputKeyValue(combRearrange.getKeyType(), conf, pkgTezOp);

    LoRearrangeDiscoverer lrDiscoverer = new LoRearrangeDiscoverer(
            combinePlan, pkgTezOp, combPack);
    lrDiscoverer.visit();

    combinePlan.remove(combPack);
    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS,
            MRCombiner.class.getName());
    conf.set(MRJobConfig.COMBINE_CLASS_ATTR,
            PigCombiner.Combine.class.getName());
    conf.setBoolean(MRConfiguration.MAPPER_NEW_API, true);
    conf.set("pig.pigContext", ObjectSerializer.serialize(pc));
    conf.set("udf.import.list",
            ObjectSerializer.serialize(PigContext.getPackageImportList()));
    conf.set("pig.combinePlan", ObjectSerializer.serialize(combinePlan));
    conf.set("pig.combine.package", ObjectSerializer.serialize(combPack));
    conf.set("pig.map.keytype", ObjectSerializer
            .serialize(new byte[] { combRearrange.getKeyType() }));
}
 

import org.apache.tez.mapreduce.combine.MRCombiner; //导入依赖的package包/类
private static void setupMRComponents(Configuration conf) {
  if (conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS) == null) {
    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS,
        MRPartitioner.class.getName());
  }

  if (conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS) == null) {
    boolean useNewApi = conf.getBoolean("mapred.mapper.new-api", false);
    if (useNewApi) {
      if (conf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null) {
        conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
      }
    } else {
      if (conf.get("mapred.combiner.class") != null) {
        conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
      }
    }
  }
}
 

import org.apache.tez.mapreduce.combine.MRCombiner; //导入依赖的package包/类
/**
 * Sets up parameters which used to be set by the MR JobClient. Includes
 * setting whether to use the new api or the old api. Note: Must be called
 * before generating InputSplits
 *
 * @param conf
 *          configuration for the vertex.
 */
public static void doJobClientMagic(Configuration conf) throws IOException {
  setUseNewAPI(conf);
  // TODO Maybe add functionality to check output specifications - e.g. fail
  // early if the output directory exists.
  InetAddress ip = InetAddress.getLocalHost();
  if (ip != null) {
    String submitHostAddress = ip.getHostAddress();
    String submitHostName = ip.getHostName();
    conf.set(MRJobConfig.JOB_SUBMITHOST, submitHostName);
    conf.set(MRJobConfig.JOB_SUBMITHOSTADDR, submitHostAddress);
  }
  // conf.set("hadoop.http.filter.initializers",
  // "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer");
  // Skipping setting JOB_DIR - not used by AM.

  // Maybe generate SHUFFLE secret. The AM uses the job token generated in
  // the AM anyway.

  // TODO eventually ACLs
  conf.set(TezJobConfig.TEZ_RUNTIME_PARTITIONER_CLASS, MRPartitioner.class.getName());
  
  boolean useNewApi = conf.getBoolean("mapred.mapper.new-api", false);
  if (useNewApi) {
    if (conf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null) {
      conf.set(TezJobConfig.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
    }
  } else {
    if (conf.get("mapred.combiner.class") != null) {
      conf.set(TezJobConfig.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
    }
  }
  
  setWorkingDirectory(conf);
}
 

import org.apache.tez.mapreduce.combine.MRCombiner; //导入依赖的package包/类
/**
 * Given a single base MRR config, returns a list of complete stage
 * configurations.
 * 
 * @param conf
 * @return list of complete stage configurations given Conifiguration
 */
@Private
public static Configuration[] getStageConfs(Configuration conf) {
  int numIntermediateStages = MultiStageMRConfigUtil
      .getNumIntermediateStages(conf);
  boolean hasFinalReduceStage = (conf.getInt(MRJobConfig.NUM_REDUCES, 0) > 0);
  // Assuming no 0 map jobs, and the first stage is always a map.
  int numStages = numIntermediateStages + (hasFinalReduceStage ? 2 : 1);

  // Read split info from HDFS
  conf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false);
  
  // Setup Tez partitioner class
  conf.set(TezJobConfig.TEZ_RUNTIME_PARTITIONER_CLASS,
      MRPartitioner.class.getName());
  
  // Setup Tez Combiner class if required.
  // This would already have been set since the call is via JobClient
  boolean useNewApi = conf.getBoolean("mapred.mapper.new-api", false);
  if (useNewApi) {
    if (conf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null) {
      conf.set(TezJobConfig.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
    }
  } else {
    if (conf.get("mapred.combiner.class") != null) {
      conf.set(TezJobConfig.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
    }
  }

  Configuration confs[] = new Configuration[numStages];
  Configuration nonItermediateConf = MultiStageMRConfigUtil.extractStageConf(
      conf, "");
  if (numStages == 1) {
    confs[0] = nonItermediateConf;
    confs[0].setBoolean(MRConfig.IS_MAP_PROCESSOR, true);
  } else {
    confs[0] = nonItermediateConf;
    confs[numStages - 1] = new Configuration(nonItermediateConf);
    confs[numStages -1].setBoolean(MRConfig.IS_MAP_PROCESSOR, false);
  }
  if (numStages > 2) {
    for (int i = 1; i < numStages - 1; i++) {
      confs[i] = MultiStageMRConfigUtil.extractStageConf(conf,
          MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i, ""));
      confs[i].setBoolean(MRConfig.IS_MAP_PROCESSOR, false);
    }
  }
  return confs;
}
 

import org.apache.tez.mapreduce.combine.MRCombiner; //导入依赖的package包/类
/**
 * Given a single base MRR config, returns a list of complete stage
 * configurations.
 * 
 * @param conf
 * @return list of complete stage configurations given Conifiguration
 */
@Private
public static Configuration[] getStageConfs(Configuration conf) {
  int numIntermediateStages = MultiStageMRConfigUtil
      .getNumIntermediateStages(conf);
  boolean hasFinalReduceStage = (conf.getInt(MRJobConfig.NUM_REDUCES, 0) > 0);
  // Assuming no 0 map jobs, and the first stage is always a map.
  int numStages = numIntermediateStages + (hasFinalReduceStage ? 2 : 1);

  // Read split info from HDFS
  conf.setBoolean(MRJobConfig.MR_TEZ_SPLITS_VIA_EVENTS, false);
  
  // Setup Tez partitioner class
  conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS,
      MRPartitioner.class.getName());
  
  // Setup Tez Combiner class if required.
  // This would already have been set since the call is via JobClient
  boolean useNewApi = conf.getBoolean("mapred.mapper.new-api", false);
  if (useNewApi) {
    if (conf.get(MRJobConfig.COMBINE_CLASS_ATTR) != null) {
      conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
    }
  } else {
    if (conf.get("mapred.combiner.class") != null) {
      conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS, MRCombiner.class.getName());
    }
  }

  Configuration confs[] = new Configuration[numStages];
  Configuration nonItermediateConf = MultiStageMRConfigUtil.extractStageConf(
      conf, "");
  if (numStages == 1) {
    confs[0] = nonItermediateConf;
    confs[0].setBoolean(MRConfig.IS_MAP_PROCESSOR, true);
  } else {
    confs[0] = nonItermediateConf;
    confs[numStages - 1] = new Configuration(nonItermediateConf);
    confs[numStages -1].setBoolean(MRConfig.IS_MAP_PROCESSOR, false);
  }
  if (numStages > 2) {
    for (int i = 1; i < numStages - 1; i++) {
      confs[i] = MultiStageMRConfigUtil.extractStageConf(conf,
          MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(i, ""));
      confs[i].setBoolean(MRConfig.IS_MAP_PROCESSOR, false);
    }
  }
  return confs;
}