Java CounterGroup类代码示例

import org.apache.hadoop.mapreduce.CounterGroup; //导入依赖的package包/类
public JobMetrics(Job job, String bytesReplicatedKey) {
  Builder<String, Long> builder = ImmutableMap.builder();
  if (job != null) {
    Counters counters;
    try {
      counters = job.getCounters();
    } catch (IOException e) {
      throw new CircusTrainException("Unable to get counters from job.", e);
    }
    if (counters != null) {
      for (CounterGroup group : counters) {
        for (Counter counter : group) {
          builder.put(DotJoiner.join(group.getName(), counter.getName()), counter.getValue());
        }
      }
    }
  }
  metrics = builder.build();
  Long bytesReplicatedValue = metrics.get(bytesReplicatedKey);
  if (bytesReplicatedValue != null) {
    bytesReplicated = bytesReplicatedValue;
  } else {
    bytesReplicated = 0L;
  }
}
 

import org.apache.hadoop.mapreduce.CounterGroup; //导入依赖的package包/类
public JobTaskAttemptCounterInfo(TaskAttempt taskattempt) {

    this.id = MRApps.toString(taskattempt.getID());
    total = taskattempt.getCounters();
    taskAttemptCounterGroup = new ArrayList<TaskCounterGroupInfo>();
    if (total != null) {
      for (CounterGroup g : total) {
        if (g != null) {
          TaskCounterGroupInfo cginfo = new TaskCounterGroupInfo(g.getName(), g);
          if (cginfo != null) {
            taskAttemptCounterGroup.add(cginfo);
          }
        }
      }
    }
  }
 

import org.apache.hadoop.mapreduce.CounterGroup; //导入依赖的package包/类
public JobCounterInfo(AppContext ctx, Job job) {
  getCounters(ctx, job);
  counterGroup = new ArrayList<CounterGroupInfo>();
  this.id = MRApps.toString(job.getID());

  if (total != null) {
    for (CounterGroup g : total) {
      if (g != null) {
        CounterGroup mg = map == null ? null : map.getGroup(g.getName());
        CounterGroup rg = reduce == null ? null : reduce
          .getGroup(g.getName());

        CounterGroupInfo cginfo = new CounterGroupInfo(g.getName(), g,
          mg, rg);
        counterGroup.add(cginfo);
      }
    }
  }
}
 

import org.apache.hadoop.mapreduce.CounterGroup; //导入依赖的package包/类
@Private
public JsonNode countersToJSON(Counters counters) {
  ObjectMapper mapper = new ObjectMapper();
  ArrayNode nodes = mapper.createArrayNode();
  if (counters != null) {
    for (CounterGroup counterGroup : counters) {
      ObjectNode groupNode = nodes.addObject();
      groupNode.put("NAME", counterGroup.getName());
      groupNode.put("DISPLAY_NAME", counterGroup.getDisplayName());
      ArrayNode countersNode = groupNode.putArray("COUNTERS");
      for (Counter counter : counterGroup) {
        ObjectNode counterNode = countersNode.addObject();
        counterNode.put("NAME", counter.getName());
        counterNode.put("DISPLAY_NAME", counter.getDisplayName());
        counterNode.put("VALUE", counter.getValue());
      }
    }
  }
  return nodes;
}
 

import org.apache.hadoop.mapreduce.CounterGroup; //导入依赖的package包/类
static JhCounters toAvro(Counters counters, String name) {
  JhCounters result = new JhCounters();
  result.name = new Utf8(name);
  result.groups = new ArrayList<JhCounterGroup>(0);
  if (counters == null) return result;
  for (CounterGroup group : counters) {
    JhCounterGroup g = new JhCounterGroup();
    g.name = new Utf8(group.getName());
    g.displayName = new Utf8(group.getDisplayName());
    g.counts = new ArrayList<JhCounter>(group.size());
    for (Counter counter : group) {
      JhCounter c = new JhCounter();
      c.name = new Utf8(counter.getName());
      c.displayName = new Utf8(counter.getDisplayName());
      c.value = counter.getValue();
      g.counts.add(c);
    }
    result.groups.add(g);
  }
  return result;
}
 

import org.apache.hadoop.mapreduce.CounterGroup; //导入依赖的package包/类
public TaskCounterGroupInfo(String name, CounterGroup group) {
  this.counterGroupName = name;
  this.counter = new ArrayList<TaskCounterInfo>();

  for (Counter c : group) {
    TaskCounterInfo cinfo = new TaskCounterInfo(c.getName(), c.getValue());
    this.counter.add(cinfo);
  }
}
 

import org.apache.hadoop.mapreduce.CounterGroup; //导入依赖的package包/类
static Counters fromAvro(JhCounters counters) {
  Counters result = new Counters();
  if(counters != null) {
    for (JhCounterGroup g : counters.getGroups()) {
      CounterGroup group =
          result.addGroup(StringInterner.weakIntern(g.getName().toString()),
              StringInterner.weakIntern(g.getDisplayName().toString()));
      for (JhCounter c : g.getCounts()) {
        group.addCounter(StringInterner.weakIntern(c.getName().toString()),
            StringInterner.weakIntern(c.getDisplayName().toString()),
                c.getValue());
      }
    }
  }
  return result;
}
 

import org.apache.hadoop.mapreduce.CounterGroup; //导入依赖的package包/类
static JhCounters toAvro(Counters counters, String name) {
  JhCounters result = new JhCounters();
  result.setName(new Utf8(name));
  result.setGroups(new ArrayList<JhCounterGroup>(0));
  if (counters == null) return result;
  for (CounterGroup group : counters) {
    JhCounterGroup g = new JhCounterGroup();
    g.setName(new Utf8(group.getName()));
    g.setDisplayName(new Utf8(group.getDisplayName()));
    g.setCounts(new ArrayList<JhCounter>(group.size()));
    for (Counter counter : group) {
      JhCounter c = new JhCounter();
      c.setName(new Utf8(counter.getName()));
      c.setDisplayName(new Utf8(counter.getDisplayName()));
      c.setValue(counter.getValue());
      g.getCounts().add(c);
    }
    result.getGroups().add(g);
  }
  return result;
}
 

import org.apache.hadoop.mapreduce.CounterGroup; //导入依赖的package包/类
/**
 * Create a {@link gobblin.metrics.GobblinMetrics} instance for this job run from the Hadoop counters.
 */
private void countersToMetrics(Optional<Counters> counters, GobblinMetrics metrics) {
  if (counters.isPresent()) {
    // Write job-level counters
    CounterGroup jobCounterGroup = counters.get().getGroup(MetricGroup.JOB.name());
    for (Counter jobCounter : jobCounterGroup) {
      metrics.getCounter(jobCounter.getName()).inc(jobCounter.getValue());
    }

    // Write task-level counters
    CounterGroup taskCounterGroup = counters.get().getGroup(MetricGroup.TASK.name());
    for (Counter taskCounter : taskCounterGroup) {
      metrics.getCounter(taskCounter.getName()).inc(taskCounter.getValue());
    }
  }
}
 

import org.apache.hadoop.mapreduce.CounterGroup; //导入依赖的package包/类
private void dumpSummary(Job job, JoinPhaseConfig joinPhaseConfig) throws IOException {
  System.out.println("Join Input Matrix.");
  CounterGroup group = job.getCounters().getGroup("DynamicCounter");
  for (String source : joinPhaseConfig.getJoinSpec().getSourceNames()) {
    System.out.print(String.format("%25s\t", source));
  }
  if (group != null) {
    Iterator<Counter> iterator = group.iterator();
    while (iterator.hasNext()) {
      Counter counter = iterator.next();
      String displayName = counter.getDisplayName();
      String[] split = displayName.replace("[", "").replace("[", "").split(",");
      for (String str : split) {
        if (str.trim().equals("1")) {
          System.out.print(String.format("%25s\t", "1"));
        } else {
          System.out.print(String.format("%25s\t", "-"));
        }
      }
    }
  }
}
 

import org.apache.hadoop.mapreduce.CounterGroup; //导入依赖的package包/类
/** {@inheritDoc} */
@Override public Iterator<CounterGroup> iterator() {
    final Iterator<String> iter = getGroupNames().iterator();

    return new Iterator<CounterGroup>() {
        @Override public boolean hasNext() {
            return iter.hasNext();
        }

        @Override public CounterGroup next() {
            if (!hasNext())
                throw new NoSuchElementException();

            return new HadoopMapReduceCounterGroup(HadoopMapReduceCounters.this, iter.next());
        }

        @Override public void remove() {
            throw new UnsupportedOperationException("not implemented");
        }
    };
}
 

import org.apache.hadoop.mapreduce.CounterGroup; //导入依赖的package包/类
@Private
public JsonNode countersToJSON(Counters counters) {
  ArrayNode nodes = FACTORY.arrayNode();
  if (counters != null) {
    for (CounterGroup counterGroup : counters) {
      ObjectNode groupNode = nodes.addObject();
      groupNode.put("NAME", counterGroup.getName());
      groupNode.put("DISPLAY_NAME", counterGroup.getDisplayName());
      ArrayNode countersNode = groupNode.putArray("COUNTERS");
      for (Counter counter : counterGroup) {
        ObjectNode counterNode = countersNode.addObject();
        counterNode.put("NAME", counter.getName());
        counterNode.put("DISPLAY_NAME", counter.getDisplayName());
        counterNode.put("VALUE", counter.getValue());
      }
    }
  }
  return nodes;
}
 

import org.apache.hadoop.mapreduce.CounterGroup; //导入依赖的package包/类
JhCounters(Counters counters, String name) {
  this.name = name;
  this.groups = new ArrayList<JhCounterGroup>();
  if (counters == null) return;
  for (CounterGroup group : counters) {
    JhCounterGroup g = new JhCounterGroup();
    g.name = group.getName();
    g.displayName = group.getDisplayName();
    g.counts = new ArrayList<JhCounter>(group.size());
    for (Counter counter : group) {
      JhCounter c = new JhCounter();
      c.name = counter.getName();
      c.displayName = counter.getDisplayName();
      c.value = counter.getValue();
      g.counts.add(c);
    }
    this.groups.add(g);
  }
}
 

import org.apache.hadoop.mapreduce.CounterGroup; //导入依赖的package包/类
/**
 * Create a {@link org.apache.gobblin.metrics.GobblinMetrics} instance for this job run from the Hadoop counters.
 */
@VisibleForTesting
void countersToMetrics(GobblinMetrics metrics) throws IOException {
  Optional<Counters> counters = Optional.fromNullable(this.job.getCounters());

  if (counters.isPresent()) {
    // Write job-level counters
    CounterGroup jobCounterGroup = counters.get().getGroup(MetricGroup.JOB.name());
    for (Counter jobCounter : jobCounterGroup) {
      metrics.getCounter(jobCounter.getName()).inc(jobCounter.getValue());
    }

    // Write task-level counters
    CounterGroup taskCounterGroup = counters.get().getGroup(MetricGroup.TASK.name());
    for (Counter taskCounter : taskCounterGroup) {
      metrics.getCounter(taskCounter.getName()).inc(taskCounter.getValue());
    }
  }
}