Java Stats类代码示例

import com.datatorrent.api.Stats; //导入依赖的package包/类
@Override
public Response processStats(BatchedOperatorStats batchedOperatorStats)
{
  List<Stats.OperatorStats> lastWindowedStats = batchedOperatorStats.getLastWindowedStats();
  if (lastWindowedStats != null) {
    for (Stats.OperatorStats os : lastWindowedStats) {
      if (os.counters != null) {
        if (os.counters instanceof BasicCounters) {
          @SuppressWarnings("unchecked")
          BasicCounters<MutableLong> cs = (BasicCounters<MutableLong>)os.counters;
          logger.debug("operatorId:{} buckets:[in-memory:{} deleted:{} evicted:{}] events:[in-memory:{} "
              + "committed-last-window:{} duplicates:{}] low:{} high:{}",
              batchedOperatorStats.getOperatorId(),
              cs.getCounter(BucketManager.CounterKeys.BUCKETS_IN_MEMORY),
              cs.getCounter(BucketManager.CounterKeys.DELETED_BUCKETS),
              cs.getCounter(BucketManager.CounterKeys.EVICTED_BUCKETS),
              cs.getCounter(BucketManager.CounterKeys.EVENTS_IN_MEMORY),
              cs.getCounter(BucketManager.CounterKeys.EVENTS_COMMITTED_LAST_WINDOW),
              cs.getCounter(CounterKeys.DUPLICATE_EVENTS));
        }
      }
    }
  }
  return null;
}
 

import com.datatorrent.api.Stats; //导入依赖的package包/类
/**
 * @param startTime the startTime to set
 */
public void setStartTime(long startTime)
{
  if (this.startTime == Stats.INVALID_TIME_MILLIS) {
    this.startTime = startTime;
  } else {
    throw new IllegalStateException("Tuple recorder has already started at " + this.startTime);
  }
}
 

import com.datatorrent.api.Stats; //导入依赖的package包/类
protected void reportStats(ContainerStats.OperatorStats stats, long windowId)
{
  stats.outputPorts = new ArrayList<>();
  for (Entry<String, Sink<Object>> e : outputs.entrySet()) {
    ContainerStats.OperatorStats.PortStats portStats = new ContainerStats.OperatorStats.PortStats(e.getKey());
    portStats.tupleCount = e.getValue().getCount(true) - controlTupleCount;
    portStats.endWindowTimestamp = endWindowEmitTime;
    stats.outputPorts.add(portStats);
  }
  controlTupleCount = 0;

  long currentCpuTime = tmb.getCurrentThreadCpuTime();
  stats.cpuTimeUsed = currentCpuTime - lastSampleCpuTime;
  lastSampleCpuTime = currentCpuTime;

  if (checkpoint != null) {
    stats.checkpoint = checkpoint;
    stats.checkpointStats = checkpointStats;
    checkpointStats = null;
    checkpoint = null;
  } else {
    Pair<FutureTask<Stats.CheckpointStats>, CheckpointWindowInfo> pair = taskQueue.peek();
    if (pair != null && pair.getFirst().isDone()) {
      taskQueue.poll();
      try {
        CheckpointWindowInfo checkpointWindowInfo = pair.getSecond();
        stats.checkpointStats = pair.getFirst().get();
        stats.checkpoint = new Checkpoint(checkpointWindowInfo.windowId, checkpointWindowInfo.applicationWindowCount,
            checkpointWindowInfo.checkpointWindowCount);
        if (operator instanceof Operator.CheckpointListener) {
          ((Operator.CheckpointListener)operator).checkpointed(checkpointWindowInfo.windowId);
        }
      } catch (Exception ex) {
        throw Throwables.propagate(ex);
      }
    }
  }

  context.report(stats, windowId);
}
 

import com.datatorrent.api.Stats; //导入依赖的package包/类
@Override
public Stats.CheckpointStats call() throws Exception
{
  agent.flush(id, windowId);
  stats.checkpointTime = System.currentTimeMillis() - stats.checkpointStartTime;
  return stats;
}
 

import com.datatorrent.api.Stats; //导入依赖的package包/类
private List<KinesisConsumer.KinesisShardStats> extractkinesisStats(BatchedOperatorStats stats)
{
  //preprocess the stats
  List<KinesisConsumer.KinesisShardStats> kmsList = new LinkedList<KinesisConsumer.KinesisShardStats>();
  for (Stats.OperatorStats os : stats.getLastWindowedStats()) {
    if (os != null && os.counters instanceof KinesisConsumer.KinesisShardStats) {
      kmsList.add((KinesisConsumer.KinesisShardStats)os.counters);
    }
  }
  return kmsList;
}
 

import com.datatorrent.api.Stats; //导入依赖的package包/类
private List<KafkaConsumer.KafkaMeterStats> extractKafkaStats(StatsListener.BatchedOperatorStats stats)
{
  //preprocess the stats
  List<KafkaConsumer.KafkaMeterStats> kmsList = new LinkedList<KafkaConsumer.KafkaMeterStats>();
  for (Stats.OperatorStats os : stats.getLastWindowedStats()) {
    if (os != null && os.counters instanceof KafkaConsumer.KafkaMeterStats) {
      kmsList.add((KafkaConsumer.KafkaMeterStats)os.counters);
    }
  }
  return kmsList;
}
 

import com.datatorrent.api.Stats; //导入依赖的package包/类
@Override
protected int getLoad(BatchedOperatorStats stats)
{
  double totalBacklog = 0;
  double statsPartitionCount = 0;
  for (Map.Entry<Integer, BatchedOperatorStats> partitionStatus : partitionedInstanceStatus.entrySet()) {
    BatchedOperatorStats batchedOperatorStats = partitionStatus.getValue();
    if (batchedOperatorStats != null) {
      List<Stats.OperatorStats> lastWindowedStats = batchedOperatorStats.getLastWindowedStats();
      if (lastWindowedStats != null && lastWindowedStats.size() > 0) {
        Stats.OperatorStats lastStats = lastWindowedStats.get(lastWindowedStats.size() - 1);
        Long queueLength = (Long)lastStats.metrics.get("queueLength");
        totalBacklog += queueLength;
        statsPartitionCount += 1;
        logger.debug("queueLength : {}, totalBacklog {},statsPartitionCount{}", queueLength, totalBacklog,
            statsPartitionCount);
      }
    }
  }

  double backlogPerPartition = totalBacklog / statsPartitionCount;
  logger.debug("backlogPerPartition : {}", backlogPerPartition);
  logger.debug("maxQueueSizePerPartition : {}, partitionedInstanceStatus.size():{}" + ", maxPartitions:{}",
      maxQueueSizePerPartition, partitionedInstanceStatus.size(), maxPartitions);

  if (backlogPerPartition > maxQueueSizePerPartition && partitionedInstanceStatus.size() < maxPartitions) {
    return 1;
  }
  logger.debug("minPartitions:{}", minPartitions);

  if (backlogPerPartition < 1.1 && partitionedInstanceStatus.size() > minPartitions) {
    return -1;
  }

  return 0;
}
 

import com.datatorrent.api.Stats; //导入依赖的package包/类
@Override
public StatsListener.Response processStats(StatsListener.BatchedOperatorStats stats)
{
  Stats.OperatorStats operatorStats = stats.getLastWindowedStats().get(stats.getLastWindowedStats().size() - 1);
  tuplesProcessedCompletely = (Integer)operatorStats.metrics.get("tuplesProcessedCompletely");
  if (tuplesProcessedCompletely >= TOTAL_TUPLES_PROCESS) {
    latch.countDown();
  }
  return null;
}
 

import com.datatorrent.api.Stats; //导入依赖的package包/类
@Override
public Response processStats(BatchedOperatorStats stats)
{
  Stats.OperatorStats operatorStats = stats.getLastWindowedStats().get(stats.getLastWindowedStats().size() - 1);
  count = (Integer)operatorStats.metrics.get("count");
  if (count >= 1000) {
    latch.countDown();
  }
  return null;
}
 

import com.datatorrent.api.Stats; //导入依赖的package包/类
@Override
public Response processStats(BatchedOperatorStats stats)
{
  Stats.OperatorStats operatorStats = stats.getLastWindowedStats().get(stats.getLastWindowedStats().size() - 1);
  tuplesProcessedCompletely = (Integer)operatorStats.metrics.get("tuplesProcessedCompletely");
  if (tuplesProcessedCompletely >= 1000) {
    latch.countDown();
  }
  return null;
}
 

import com.datatorrent.api.Stats; //导入依赖的package包/类
@Override
public Response processStats(BatchedOperatorStats stats)
{
  Stats.OperatorStats operatorStats = stats.getLastWindowedStats().get(stats.getLastWindowedStats().size() - 1);
  count = (Integer)operatorStats.metrics.get("count");
  if (count == 12) {
    latch.countDown();
  }
  return null;
}
 

import com.datatorrent.api.Stats; //导入依赖的package包/类
void checkpoint(long windowId)
{
  if (!context.stateless) {
    if (operator instanceof Operator.CheckpointNotificationListener) {
      ((Operator.CheckpointNotificationListener)operator).beforeCheckpoint(windowId);
    }

    StorageAgent ba = context.getValue(OperatorContext.STORAGE_AGENT);
    if (ba != null) {
      try {
        checkpointStats = new Stats.CheckpointStats();
        checkpointStats.checkpointStartTime = System.currentTimeMillis();
        ba.save(operator, id, windowId);
        if (ba instanceof AsyncStorageAgent) {
          AsyncStorageAgent asyncStorageAgent = (AsyncStorageAgent)ba;
          if (!asyncStorageAgent.isSyncCheckpoint()) {
            if (PROCESSING_MODE != ProcessingMode.EXACTLY_ONCE) {
              CheckpointWindowInfo checkpointWindowInfo = new CheckpointWindowInfo();
              checkpointWindowInfo.windowId = windowId;
              checkpointWindowInfo.applicationWindowCount = applicationWindowCount;
              checkpointWindowInfo.checkpointWindowCount = checkpointWindowCount;
              CheckpointHandler checkpointHandler = new CheckpointHandler();
              checkpointHandler.agent = asyncStorageAgent;
              checkpointHandler.operatorId = id;
              checkpointHandler.windowId = windowId;
              checkpointHandler.stats = checkpointStats;
              FutureTask<Stats.CheckpointStats> futureTask = new FutureTask<>(checkpointHandler);
              taskQueue.add(new Pair<>(futureTask, checkpointWindowInfo));
              executorService.submit(futureTask);
              checkpoint = null;
              checkpointStats = null;
              return;
            } else {
              asyncStorageAgent.flush(id, windowId);
            }
          }
        }
        checkpointStats.checkpointTime = System.currentTimeMillis() - checkpointStats.checkpointStartTime;
      } catch (IOException ie) {
        try {
          logger.warn("Rolling back checkpoint {} for Operator {} due to the exception {}",
              Codec.getStringWindowId(windowId), operator, ie);
          ba.delete(id, windowId);
        } catch (IOException ex) {
          logger.warn("Error while rolling back checkpoint", ex);
        }
        throw new RuntimeException(ie);
      }
    }
  }

  calculateNextCheckpointWindow();
  dagCheckpointOffsetCount = 0;
  checkpoint = new Checkpoint(windowId, applicationWindowCount, checkpointWindowCount);
  if (operator instanceof Operator.CheckpointListener) {
    ((Operator.CheckpointListener)operator).checkpointed(windowId);
  }
}
 

import com.datatorrent.api.Stats; //导入依赖的package包/类
@Override
public Response processStats(BatchedOperatorStats stats)
{
  response.repartitionRequired = false;
  if (!collectStats) {
    return response;
  }

  List<Stats.OperatorStats> lastWindowedStats = stats.getLastWindowedStats();
  if (lastWindowedStats != null && lastWindowedStats.size() > 0) {
    Stats.OperatorStats lastStats = lastWindowedStats.get(lastWindowedStats.size() - 1);
    if (lastStats.inputPorts.size() > 0) {
      backlogPerOperator.put(stats.getOperatorId(), lastStats.inputPorts.get(0).queueSize);
    }
  }

  if (System.currentTimeMillis() < nextMillis) {
    return response;
  }
  nextMillis = System.currentTimeMillis() + intervalMillis;
  LOG.debug("Proposed NextMillis = {}", nextMillis);

  long totalBacklog = 0;
  for (Map.Entry<Integer, Integer> backlog : backlogPerOperator.entrySet()) {
    totalBacklog += backlog.getValue();
  }
  LOG.debug("backlog {} partitionCount {}", totalBacklog, partitionCount);
  backlogPerOperator.clear();

  if (totalBacklog == partitionCount) {
    return response; //do not repartition
  }

  int newPartitionCount;
  if (totalBacklog > maxReaders) {
    LOG.debug("large backlog {}", totalBacklog);
    newPartitionCount = maxReaders;
  } else if (totalBacklog < minReaders) {
    LOG.debug("small backlog {}", totalBacklog);
    newPartitionCount = minReaders;
  } else {
    newPartitionCount = getAdjustedCount(totalBacklog);
    LOG.debug("moderate backlog {}", totalBacklog);
  }

  LOG.debug("backlog {} newPartitionCount {} partitionCount {}", totalBacklog, newPartitionCount, partitionCount);
  if (newPartitionCount == partitionCount) {
    return response; //do not repartition
  }

  partitionCount = newPartitionCount;
  response.repartitionRequired = true;
  LOG.debug("partition required", totalBacklog, partitionCount);

  return response;
}
 

import com.datatorrent.api.Stats; //导入依赖的package包/类
@Override
public List<Stats.OperatorStats> getLastWindowedStats()
{
  return operatorStats;
}