当前位置: 首页>>代码示例>>Java>>正文


Java TezCounter.getValue方法代码示例

本文整理汇总了Java中org.apache.tez.common.counters.TezCounter.getValue方法的典型用法代码示例。如果您正苦于以下问题:Java TezCounter.getValue方法的具体用法?Java TezCounter.getValue怎么用?Java TezCounter.getValue使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在org.apache.tez.common.counters.TezCounter的用法示例。


在下文中一共展示了TezCounter.getValue方法的5个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: getAvgCounter

import org.apache.tez.common.counters.TezCounter; //导入方法依赖的package包/类
private float getAvgCounter(Collection<TaskAttemptInfo> taskAttemptInfos, String
    counterGroupName, String counterName) {
  long total = 0;
  int taskCount = 0;
  for (TaskAttemptInfo attemptInfo : taskAttemptInfos) {
    TezCounters tezCounters = attemptInfo.getTezCounters();
    TezCounter counter = tezCounters.findCounter(counterGroupName, counterName);
    if (counter != null) {
      total += counter.getValue();
      taskCount++;
    } else {
      LOG.info("Could not find counterGroupName=" + counterGroupName + ", counter=" +
          counterName + " in " + attemptInfo);
    }
  }
  return (taskCount > 0) ? (total * 1.0f / taskCount) : 0;
}
 
开发者ID:apache,项目名称:tez,代码行数:18,代码来源:SlowNodeAnalyzer.java

示例2: basicTest

import org.apache.tez.common.counters.TezCounter; //导入方法依赖的package包/类
public void basicTest(int partitions, int numKeys, int keySize,
    long initialAvailableMem, int minBlockSize) throws IOException {
  this.numOutputs = partitions; // single output
  conf.setInt(TezRuntimeConfiguration
      .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, minBlockSize >> 20);
  PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs,
      initialAvailableMem);

  writeData(sorter, numKeys, keySize);

  //partition stats;
  ReportPartitionStats partitionStats =
      ReportPartitionStats.fromString(conf.get(
      TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS,
      TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS_DEFAULT));
  if (partitionStats.isEnabled()) {
    assertTrue(sorter.getPartitionStats() != null);
  }

  verifyCounters(sorter, outputContext);
  verifyOutputPermissions(outputContext.getUniqueIdentifier());
  Path outputFile = sorter.finalOutputFile;
  FileSystem fs = outputFile.getFileSystem(conf);
  TezCounter finalOutputBytes =
      outputContext.getCounters().findCounter(TaskCounter.OUTPUT_BYTES_PHYSICAL);
  if (finalOutputBytes.getValue() > 0) {
    IFile.Reader reader = new IFile.Reader(fs, outputFile, null, null, null, false, -1, 4096);
    verifyData(reader);
    reader.close();
  }
  //Verify dataset
  verify(outputContext, atLeastOnce()).notifyProgress();
}
 
开发者ID:apache,项目名称:tez,代码行数:34,代码来源:TestPipelinedSorter.java

示例3: convertCountersToATSMap

import org.apache.tez.common.counters.TezCounter; //导入方法依赖的package包/类
public static Map<String,Object> convertCountersToATSMap(TezCounters counters) {
  Map<String,Object> object = new LinkedHashMap<String, Object>();
  if (counters == null) {
      return object;
    }
  ArrayList<Object> counterGroupsList = new ArrayList<Object>();
  for (CounterGroup group : counters) {
      ArrayList<Object> counterList = new ArrayList<Object>();
      for (TezCounter counter : group) {
        if (counter.getValue() != 0) {
          Map<String,Object> counterMap = new LinkedHashMap<String, Object>();
          counterMap.put(ATSConstants.COUNTER_NAME, counter.getName());
          if (!counter.getDisplayName().equals(counter.getName())) {
            counterMap.put(ATSConstants.COUNTER_DISPLAY_NAME,
                  counter.getDisplayName());
          }
          counterMap.put(ATSConstants.COUNTER_VALUE, counter.getValue());
          counterList.add(counterMap);
        }
      }
      if (!counterList.isEmpty()) {
        Map<String,Object> counterGroupMap = new LinkedHashMap<String, Object>();
        counterGroupMap.put(ATSConstants.COUNTER_GROUP_NAME, group.getName());
        if (!group.getDisplayName().equals(group.getName())) {
          counterGroupMap.put(ATSConstants.COUNTER_GROUP_DISPLAY_NAME,
              group.getDisplayName());
        }
        counterGroupMap.put(ATSConstants.COUNTERS, counterList);
        counterGroupsList.add(counterGroupMap);
      }
    }
  putInto(object, ATSConstants.COUNTER_GROUPS, counterGroupsList);
  return object;
}
 
开发者ID:apache,项目名称:tez,代码行数:35,代码来源:DAGUtils.java

示例4: execute

import org.apache.tez.common.counters.TezCounter; //导入方法依赖的package包/类
private int execute(String[] args, TezConfiguration tezConf, TezClient tezSession)
    throws IOException, TezException, InterruptedException {
  LOG.info("Running IntersectValidate");
  UserGroupInformation.setConfiguration(tezConf);

  String lhsDir = args[0];
  String rhsDir = args[1];
  int numPartitions = 1;
  if (args.length == 3) {
    numPartitions = Integer.parseInt(args[2]);
  }

  if (numPartitions <= 0) {
    System.err.println("NumPartitions must be > 0");
    return 4;
  }

  Path lhsPath = new Path(lhsDir);
  Path rhsPath = new Path(rhsDir);

  DAG dag = createDag(tezConf, lhsPath, rhsPath, numPartitions);
  setupURIsForCredentials(dag, lhsPath, rhsPath);

  tezSession.waitTillReady();
  DAGClient dagClient = tezSession.submitDAG(dag);
  DAGStatus dagStatus = dagClient.waitForCompletionWithAllStatusUpdates(null);
  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  } else {
    dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    TezCounter counter = dagStatus.getDAGCounters().findCounter(COUNTER_GROUP_NAME,
        MISSING_KEY_COUNTER_NAME);
    if (counter == null) {
      LOG.info("Unable to determing equality");
      return -2;
    } else {
      if (counter.getValue() != 0) {
        LOG.info("Validate failed. The two sides are not equivalent");
        return -3;
      } else {
        LOG.info("Vlidation successful. The two sides are equivalent");
        return 0;
      }
    }
  }
}
 
开发者ID:apache,项目名称:incubator-tez,代码行数:48,代码来源:IntersectValidate.java

示例5: runJob

import org.apache.tez.common.counters.TezCounter; //导入方法依赖的package包/类
@Override
protected int runJob(String[] args, TezConfiguration tezConf,
    TezClient tezClient) throws Exception {

  LOG.info("Running JoinValidate");

  String lhsDir = args[0];
  String rhsDir = args[1];
  int numPartitions = 1;
  if (args.length == 3) {
    numPartitions = Integer.parseInt(args[2]);
  }

  if (numPartitions <= 0) {
    System.err.println("NumPartitions must be > 0");
    return 4;
  }

  Path lhsPath = new Path(lhsDir);
  Path rhsPath = new Path(rhsDir);

  DAG dag = createDag(tezConf, lhsPath, rhsPath, numPartitions);

  tezClient.waitTillReady();
  DAGClient dagClient = tezClient.submitDAG(dag);
  Set<StatusGetOpts> getOpts = Sets.newHashSet();
  if (isCountersLog()) {
    getOpts.add(StatusGetOpts.GET_COUNTERS);
  }
  DAGStatus dagStatus = dagClient.waitForCompletionWithStatusUpdates(getOpts);
  if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) {
    LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics());
    return -1;
  } else {
    dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS));
    TezCounter counter = dagStatus.getDAGCounters().findCounter(COUNTER_GROUP_NAME,
        MISSING_KEY_COUNTER_NAME);
    if (counter == null) {
      LOG.info("Unable to determing equality");
      return -2;
    } else {
      if (counter.getValue() != 0) {
        LOG.info("Validate failed. The two sides are not equivalent");
        return -3;
      } else {
        LOG.info("Validation successful. The two sides are equivalent");
        return 0;
      }
    }
  }

}
 
开发者ID:apache,项目名称:tez,代码行数:53,代码来源:JoinValidate.java


注:本文中的org.apache.tez.common.counters.TezCounter.getValue方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。