Java AvroValue.datum方法代码示例

import org.apache.avro.mapred.AvroValue; //导入方法依赖的package包/类
@Override
protected void reduce(LongWritable idx, Iterable<AvroValue<SerializableBill>> bills, Context context) throws IOException, InterruptedException {
    LongIndexedSerializableBill.Builder builder = LongIndexedSerializableBill.newBuilder();
    builder.setIdx(idx.get());

    // A product is likely to be several times in the same bill (i.e.
    // quantity > 1)
    // Using a Map to keep a single instance in such cases
    Map<Long, SerializableBill> allBills = new HashMap<Long, SerializableBill>();
    for (AvroValue<SerializableBill> value : bills) {
        SerializableBill bill = value.datum();
        if (!allBills.containsKey(bill.getId())) {
            allBills.put(bill.getId(), SerializableBill.newBuilder(bill).build());
        }
    }
    builder.setBills(new ArrayList<SerializableBill>(allBills.values()));

    context.write(new AvroKey<LongIndexedSerializableBill>(builder.build()), NullWritable.get());
}
 

import org.apache.avro.mapred.AvroValue; //导入方法依赖的package包/类
@Override
public void reduce(IntWritable key, Iterable<AvroValue<GenericRecord>> values, Context context)
        throws IOException, InterruptedException
{
  for (AvroValue<GenericRecord> value : values)
  {
    GenericRecord record = value.datum();
    context.write(new AvroKey<GenericRecord>(record), NullWritable.get());
  }
}
 

import org.apache.avro.mapred.AvroValue; //导入方法依赖的package包/类
@Override
protected void reduce(LongWritable idx, Iterable<AvroValue<PostcodeCategoryTurnoverTmp>> fragments, Context context) throws IOException,
        InterruptedException {
    CharSequence postcode = null;
    Set<ProductData> products = new HashSet<ProductData>();

    for (AvroValue<PostcodeCategoryTurnoverTmp> value : fragments) {
        PostcodeCategoryTurnoverTmp fragment = value.datum();
        if (fragment.getPostcode() != null) {
            postcode = fragment.getPostcode();
        } else {
            products.add(new ProductData(fragment));
        }
    }
    if (products.size() == 0) {
        products.add(null);
    }

    PostcodeCategoryTurnoverTmp.Builder merged = PostcodeCategoryTurnoverTmp.newBuilder();
    merged.setClientId(idx.get());
    merged.setPostcode(postcode);
    merged.setAmount(0.0);

    for (ProductData product : products) {
        merged.setProductId(product.id);
        merged.setCategoryLabel(product.category);
        merged.setAmount(product.amount);
        context.write(new AvroKey<PostcodeCategoryTurnoverTmp>(merged.build()), NullWritable.get());
    }
}
 

import org.apache.avro.mapred.AvroValue; //导入方法依赖的package包/类
@Override
protected void reduce(LongWritable idx, Iterable<AvroValue<PostcodeCategoryTurnoverTmp>> fragments, Context context) throws IOException,
        InterruptedException {
    List<CustomerData> customersData = new ArrayList<CustomerData>();
    CharSequence productCategory = null;
    double amount = 0.0;

    for (AvroValue<PostcodeCategoryTurnoverTmp> value : fragments) {
        PostcodeCategoryTurnoverTmp fragment = value.datum();
        if (fragment.getClientId() != null) {
            customersData.add(new CustomerData(fragment));
        } else {
            productCategory = fragment.getCategoryLabel();
            amount = fragment.getAmount();
        }
    }

    PostcodeCategoryTurnoverTmp.Builder merged = PostcodeCategoryTurnoverTmp.newBuilder();
    merged.setProductId(idx.get());
    merged.setCategoryLabel(productCategory);
    merged.setAmount(amount);

    for (CustomerData customerData : customersData) {
        merged.setClientId(customerData.id);
        merged.setPostcode(customerData.postcode);
        context.write(new AvroKey<PostcodeCategoryTurnoverTmp>(merged.build()), NullWritable.get());
    }
}
 

import org.apache.avro.mapred.AvroValue; //导入方法依赖的package包/类
@Override
public int compare(AvroValue<GenericRecord> o1, AvroValue<GenericRecord> o2) {
  GenericRecord record1= o1.datum();
  GenericRecord record2 = o2.datum();
  for (String deltaFieldName : this.deltaSchemaProvider.getDeltaFieldNames(record1)) {
    if (record1.get(deltaFieldName).equals(record2.get(deltaFieldName))) {
      continue;
    }
    return ((Comparable)record1.get(deltaFieldName)).compareTo(record2.get(deltaFieldName));
  }

  return 0;
}
 

import org.apache.avro.mapred.AvroValue; //导入方法依赖的package包/类
@Override
public void reduce(IntWritable key, Iterable<AvroValue<GenericRecord>> values, Context context)
    throws IOException, InterruptedException {
  for (AvroValue<GenericRecord> value : values) {
    GenericRecord record = value.datum();
    context.write(new AvroKey<GenericRecord>(record), NullWritable.get());
  }
}
 

import org.apache.avro.mapred.AvroValue; //导入方法依赖的package包/类
@Override
protected void map(AvroKey<String> key, AvroValue<LogData> value, Context context) throws java.io.IOException, java.lang.InterruptedException
{ 
  LogData data = value.datum();
  
  if (data.getEntry() != null && data.getEntry() instanceof com.linkedin.whiteelephant.parsing.Job)
  {
    com.linkedin.whiteelephant.parsing.Job job = (com.linkedin.whiteelephant.parsing.Job)data.getEntry();
    for (com.linkedin.whiteelephant.parsing.Task task : job.getTasks())
    {
      for (com.linkedin.whiteelephant.parsing.Attempt attempt : task.getAttempts())
      {
        if (attempt.getTaskStatus() == null)
        {
          System.out.println("Status null for job " + attempt.getJobId() + " attempt " + attempt.getTaskAttemptId());            
          context.getCounter("Job Analysis", "Status null").increment(1);            
          continue;
        }
        else
        {
          context.getCounter("Job Analysis", "Status " + attempt.getTaskStatus()).increment(1);
        }
        
        if (attempt.getStartTime() == null || attempt.getFinishTime() == null)
        {
          System.out.println("Missing either startTime or finishTime");
          context.getCounter("Job Analysis", "Missing start or finish").increment(1);
          continue;
        }
        
        if (!(attempt.getStartTime() > 0 && attempt.getFinishTime() > 0))
        {
          System.out.println("Either startTime or finishTime is not positive");
          context.getCounter("Job Analysis", "Not positive start or finish").increment(1);
          continue;
        }

        AttemptStatsKey statsKey = new AttemptStatsKey();
        statsKey.setCluster(clusterName); 
        statsKey.setExcess(((com.linkedin.whiteelephant.parsing.DerivedAttemptData)attempt.getDerived()).getExcess());
        statsKey.setStatus(TaskStatus.valueOf(attempt.getTaskStatus().toString()));
        statsKey.setType(TaskType.valueOf(attempt.getType().toString().toUpperCase()));
        statsKey.setUser(job.getUser());
        
        writeStats(statsKey, attempt, context);
      }
    }
  }        
}
 

import org.apache.avro.mapred.AvroValue; //导入方法依赖的package包/类
@Override
protected void reduce(AvroKey<AttemptStatsKey> key, Iterable<AvroValue<AttemptStatsValue>> values, final Context context) throws IOException, InterruptedException 
{ 
  AttemptStatsValue merged = new AttemptStatsValue();
        
  merged.setElapsedMinutes(0.0);
  merged.setStarted(0);
  merged.setFinished(0);
  
  for (AvroValue<AttemptStatsValue> valueWrapped : values)
  {
    AttemptStatsValue value = valueWrapped.datum();
    merged.setElapsedMinutes(value.getElapsedMinutes() + merged.getElapsedMinutes());
    merged.setStarted(value.getStarted() + merged.getStarted());
    merged.setFinished(value.getFinished() + merged.getFinished());
    
    if (value.getCpuMinutes() != null)
    {
      if (merged.getCpuMinutes() == null)
      {
        merged.setCpuMinutes(value.getCpuMinutes());
      }
      else
      {
        merged.setCpuMinutes(merged.getCpuMinutes() + value.getCpuMinutes());
      }
    }
    
    if (value.getSpilledRecords() != null)
    {
      if (merged.getSpilledRecords() == null)
      {
        merged.setSpilledRecords(value.getSpilledRecords());
      }
      else
      {
        merged.setSpilledRecords(merged.getSpilledRecords() + value.getSpilledRecords());
      }
    }
    
    if (value.getReduceShuffleBytes() != null)
    {
      if (merged.getReduceShuffleBytes() == null)
      {
        merged.setReduceShuffleBytes(value.getReduceShuffleBytes());
      }
      else 
      {
        merged.setReduceShuffleBytes(merged.getReduceShuffleBytes() + value.getReduceShuffleBytes());
      }
    }
  }
  
  context.write(key, new AvroWrapper<AttemptStatsValue>(merged));
}