Java LongWritable.get方法代码示例

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
@Override
public void run() {
  for (int i = 0; i < count; i++) {
    try {
      final long param = RANDOM.nextLong();
      LongWritable value = call(client, param, server, conf);
      if (value.get() != param) {
        LOG.fatal("Call failed!");
        failed = true;
        break;
      }
    } catch (Exception e) {
      LOG.fatal("Caught: " + StringUtils.stringifyException(e));
      failed = true;
    }
  }
}
 

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
@Override
public List<InputSplit> getSplits(JobContext jobCtxt) throws IOException {
  final JobConf jobConf = new JobConf(jobCtxt.getConfiguration());
  final JobClient client = new JobClient(jobConf);
  ClusterStatus stat = client.getClusterStatus(true);
  int numTrackers = stat.getTaskTrackers();
  final int fileCount = jobConf.getInt(GRIDMIX_DISTCACHE_FILE_COUNT, -1);

  // Total size of distributed cache files to be generated
  final long totalSize = jobConf.getLong(GRIDMIX_DISTCACHE_BYTE_COUNT, -1);
  // Get the path of the special file
  String distCacheFileList = jobConf.get(GRIDMIX_DISTCACHE_FILE_LIST);
  if (fileCount < 0 || totalSize < 0 || distCacheFileList == null) {
    throw new RuntimeException("Invalid metadata: #files (" + fileCount
        + "), total_size (" + totalSize + "), filelisturi ("
        + distCacheFileList + ")");
  }

  Path sequenceFile = new Path(distCacheFileList);
  FileSystem fs = sequenceFile.getFileSystem(jobConf);
  FileStatus srcst = fs.getFileStatus(sequenceFile);
  // Consider the number of TTs * mapSlotsPerTracker as number of mappers.
  int numMapSlotsPerTracker = jobConf.getInt(TTConfig.TT_MAP_SLOTS, 2);
  int numSplits = numTrackers * numMapSlotsPerTracker;

  List<InputSplit> splits = new ArrayList<InputSplit>(numSplits);
  LongWritable key = new LongWritable();
  BytesWritable value = new BytesWritable();

  // Average size of data to be generated by each map task
  final long targetSize = Math.max(totalSize / numSplits,
                            DistributedCacheEmulator.AVG_BYTES_PER_MAP);
  long splitStartPosition = 0L;
  long splitEndPosition = 0L;
  long acc = 0L;
  long bytesRemaining = srcst.getLen();
  SequenceFile.Reader reader = null;
  try {
    reader = new SequenceFile.Reader(fs, sequenceFile, jobConf);
    while (reader.next(key, value)) {

      // If adding this file would put this split past the target size,
      // cut the last split and put this file in the next split.
      if (acc + key.get() > targetSize && acc != 0) {
        long splitSize = splitEndPosition - splitStartPosition;
        splits.add(new FileSplit(
            sequenceFile, splitStartPosition, splitSize, (String[])null));
        bytesRemaining -= splitSize;
        splitStartPosition = splitEndPosition;
        acc = 0L;
      }
      acc += key.get();
      splitEndPosition = reader.getPosition();
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
  if (bytesRemaining != 0) {
    splits.add(new FileSplit(
        sequenceFile, splitStartPosition, bytesRemaining, (String[])null));
  }

  return splits;
}
 

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
/**
 * Emits random words sequence of desired size. Note that the desired output
 * size is passed as the value parameter to this map.
 */
@Override
public void map(NullWritable key, LongWritable value, Context context)
throws IOException, InterruptedException {
  //TODO Control the extra data written ..
  //TODO Should the key\tvalue\n be considered for measuring size?
  //     Can counters like BYTES_WRITTEN be used? What will be the value of
  //     such counters in LocalJobRunner?
  for (long bytes = value.get(); bytes > 0;) {
    String randomKey = rtg.getRandomWord();
    String randomValue = rtg.getRandomWord();
    context.write(new Text(randomKey), new Text(randomValue));
    bytes -= (randomValue.getBytes(charsetUTF8).length +
        randomKey.getBytes(charsetUTF8).length);
  }
}
 

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
@Override
public void map(LongWritable key, BytesWritable value, Context context)
    throws IOException, InterruptedException {

  String fileName = new String(value.getBytes(), 0,
      value.getLength(), charsetUTF8);
  Path path = new Path(fileName);

  FSDataOutputStream dos =
      FileSystem.create(fs, path, new FsPermission(GRIDMIX_DISTCACHE_FILE_PERM));

  int size = 0;
  for (long bytes = key.get(); bytes > 0; bytes -= size) {
    r.nextBytes(val.getBytes());
    size = (int)Math.min(val.getLength(), bytes);
    dos.write(val.getBytes(), 0, size);// Write to distCache file
  }
  dos.close();
}
 

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
/**
 * Map file name and offset into statistical data.
 * <p>
 * The map task is to get the 
 * <tt>key</tt>, which contains the file name, and the 
 * <tt>value</tt>, which is the offset within the file.
 * 
 * The parameters are passed to the abstract method 
 * {@link #doIO(Reporter,String,long)}, which performs the io operation, 
 * usually read or write data, and then 
 * {@link #collectStats(OutputCollector,String,long,Object)} 
 * is called to prepare stat data for a subsequent reducer.
 */
public void map(Text key, 
                LongWritable value,
                OutputCollector<Text, Text> output, 
                Reporter reporter) throws IOException {
  String name = key.toString();
  long longValue = value.get();
  
  reporter.setStatus("starting " + name + " ::host = " + hostName);

  this.stream = getIOStream(name);
  T statValue = null;
  long tStart = System.currentTimeMillis();
  try {
    statValue = doIO(reporter, name, longValue);
  } finally {
    if(stream != null) stream.close();
  }
  long tEnd = System.currentTimeMillis();
  long execTime = tEnd - tStart;
  collectStats(output, name, execTime, statValue);
  
  reporter.setStatus("finished " + name + " ::host = " + hostName);
}
 

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
@Override
protected void reduce(LongWritable key, Iterable<indexToWordWritable> values, Context context) throws IOException, InterruptedException {
    if(key.get()<0) {
        for (indexToWordWritable value : values) {
            String word = value.getWord();
            if(!wordToIndex.containsKey(word)) {
                wordToIndex.put(word, wordToIndex.size());
            }
        }
    } else {
        for(indexToWordWritable words : values) {
            context.write(new twoDimensionIndexWritable(m, words.getIndex()), new Text(words.getWord()));
        }
        log.info("m: {}",m);
        m++;
    }
}
 

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
/** Compute the (offset+1)th to (offset+length)th digits. */
protected void map(LongWritable offset, IntWritable length,
    final Context context) throws IOException, InterruptedException {
  LOG.info("offset=" + offset + ", length=" + length);

  // compute digits
  final byte[] bytes = new byte[length.get() >> 1];
  long d = offset.get();
  for (int i = 0; i < bytes.length; d += 4) {
    final long digits = hexDigits(d);
    bytes[i++] = (byte) (digits >> 8);
    bytes[i++] = (byte) digits;
  }

  // output map results
  context.write(offset, new BytesWritable(bytes));
}
 

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
public void map(LongWritable row, NullWritable ignored,
    Context context) throws IOException, InterruptedException {
  if (rand == null) {
    rowId = new Unsigned16(row.get());
    rand = Random16.skipAhead(rowId);
    checksumCounter = context.getCounter(Counters.CHECKSUM);
  }
  Random16.nextRand(rand);
  GenSort.generateRecord(buffer, rand, rowId);
  key.set(buffer, 0, TeraInputFormat.KEY_LENGTH);
  value.set(buffer, TeraInputFormat.KEY_LENGTH, 
            TeraInputFormat.VALUE_LENGTH);
  context.write(key, value);
  crc32.reset();
  crc32.update(buffer, 0, 
               TeraInputFormat.KEY_LENGTH + TeraInputFormat.VALUE_LENGTH);
  checksum.set(crc32.getValue());
  total.add(checksum);
  rowId.add(ONE);
}
 

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
@Override
protected void reduce(LongWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {

    int m = context.getConfiguration().getInt("mw", -1);

    double[] ai = new double[m];
    double[] bi = new double[m];

    int i = (int)key.get() - 1;

    for (Text value : values) {
        if (value.toString().contains("\t")) {
            String[] keyVal = value.toString().split("\\t");
            ai[Integer.parseInt(keyVal[0]) - 1] = Double.parseDouble(keyVal[1]);
        } else {
            String[] vals = value.toString().split(",");
            for (int j = 0; j < m; j++) {
                bi[j] = Double.parseDouble(vals[j]);
            }
        }

    }

    double difference = 0d;

    for (int j = 0; j < m; j++) {
        difference += Math.pow(ai[j] - bi[j], 2d);
    }
    context.write(new LongWritable(i + 1), new Text(difference + ""));

}
 

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
@Override
protected void reduce(LongWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {

    int k = context.getConfiguration().getInt("k", -1);

    String fpath = context.getConfiguration().get("mpath");
    String datLoc = context.getConfiguration().get("datLoc", "od");
    String wd = context.getConfiguration().get(datLoc);
    Path file = new Path(wd, fpath);

    FileSystem fs = file.getFileSystem(new Configuration());
    FSDataInputStream ds = fs.open(file);
    long start = (key.get() - 1) * 8 * (k + 1);
    ds.seek(start + 8);

    double[] wi = new double[k];
    for (int counter = 0; counter < k; counter++) {
        wi[counter] = ds.readDouble();
    }

    ds.close();

    for (Text value : values) {
        String[] keyVal = value.toString().split("\\t");
        long j = Long.parseLong(keyVal[0]);
        double aij = Double.parseDouble(keyVal[1]);

        StringBuilder result = new StringBuilder();
        for (int cou = 0; cou < k; cou++) {
            result.append(aij * wi[cou]);
            if (cou < k) {
                result.append(",");
            }
        }
        context.write(new LongWritable(j), new Text(result.toString()));
    }
}
 

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
/**
 * Validate setupGenerateDistCacheData by validating <li>permissions of the
 * distributed cache directory and <li>content of the generated sequence file.
 * This includes validation of dist cache file paths and their file sizes.
 */
private void doValidateSetupGenDC(
    RecordReader<LongWritable, BytesWritable> reader, FileSystem fs,
    long[] sortedFileSizes) throws IOException, InterruptedException {

  // Validate permissions of dist cache directory
  Path distCacheDir = dce.getDistributedCacheDir();
  assertEquals(
      "Wrong permissions for distributed cache dir " + distCacheDir,
      fs.getFileStatus(distCacheDir).getPermission().getOtherAction()
          .and(FsAction.EXECUTE), FsAction.EXECUTE);

  // Validate the content of the sequence file generated by
  // dce.setupGenerateDistCacheData().
  LongWritable key = new LongWritable();
  BytesWritable val = new BytesWritable();
  for (int i = 0; i < sortedFileSizes.length; i++) {
    assertTrue("Number of files written to the sequence file by "
        + "setupGenerateDistCacheData is less than the expected.",
        reader.nextKeyValue());
    key = reader.getCurrentKey();
    val = reader.getCurrentValue();
    long fileSize = key.get();
    String file = new String(val.getBytes(), 0, val.getLength());

    // Dist Cache files should be sorted based on file size.
    assertEquals("Dist cache file size is wrong.", sortedFileSizes[i],
        fileSize);

    // Validate dist cache file path.

    // parent dir of dist cache file
    Path parent = new Path(file).getParent().makeQualified(fs.getUri(),fs.getWorkingDirectory());
    // should exist in dist cache dir
    assertTrue("Public dist cache file path is wrong.",
        distCacheDir.equals(parent));
  }
}
 

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
@Override
protected void map(LongWritable key, LongWritable value, Context context)
    throws IOException, InterruptedException {
  long chainId = value.get();
  LOG.info("Starting mapper with chainId:" + chainId);

  byte[] chainIdArray = Bytes.toBytes(chainId);
  long currentRow = 0;

  long chainLength = context.getConfiguration().getLong(CHAIN_LENGTH_KEY, CHAIN_LENGTH);
  long nextRow = getNextRow(0, chainLength);

  for (long i = 0; i < chainLength; i++) {
    byte[] rk = Bytes.toBytes(currentRow);

    // Next link in the chain.
    KeyValue linkKv = new KeyValue(rk, CHAIN_FAM, chainIdArray, Bytes.toBytes(nextRow));
    // What link in the chain this is.
    KeyValue sortKv = new KeyValue(rk, SORT_FAM, chainIdArray, Bytes.toBytes(i));
    // Added data so that large stores are created.
    KeyValue dataKv = new KeyValue(rk, DATA_FAM, chainIdArray,
      Bytes.toBytes(RandomStringUtils.randomAlphabetic(50))
    );

    // Emit the key values.
    context.write(new ImmutableBytesWritable(rk), linkKv);
    context.write(new ImmutableBytesWritable(rk), sortKv);
    context.write(new ImmutableBytesWritable(rk), dataKv);
    // Move to the next row.
    currentRow = nextRow;
    nextRow = getNextRow(i+1, chainLength);
  }
}
 

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
public void map(Text key, LongWritable value,
                OutputCollector<Text, LongWritable> collector,
                Reporter reporter)
  throws IOException {
  
  String name = key.toString();
  long size = value.get();
  long seed = Long.parseLong(name);

  random.setSeed(seed);
  reporter.setStatus("creating " + name);

  // write to temp file initially to permit parallel execution
  Path tempFile = new Path(DATA_DIR, name+suffix);
  OutputStream out = fs.create(tempFile);

  long written = 0;
  try {
    while (written < size) {
      if (fastCheck) {
        Arrays.fill(buffer, (byte)random.nextInt(Byte.MAX_VALUE));
      } else {
        random.nextBytes(buffer);
      }
      long remains = size - written;
      int length = (remains<=buffer.length) ? (int)remains : buffer.length;
      out.write(buffer, 0, length);
      written += length;
      reporter.setStatus("writing "+name+"@"+written+"/"+size);
    }
  } finally {
    out.close();
  }
  // rename to final location
  fs.rename(tempFile, new Path(DATA_DIR, name));

  collector.collect(new Text("bytes"), new LongWritable(written));

  reporter.setStatus("wrote " + name);
}
 

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
@Override
public void map(NullWritable key, LongWritable value, Context context)
    throws IOException, InterruptedException {
  for (long bytes = value.get(); bytes > 0; bytes -= val.getLength()) {
    r.nextBytes(val.getBytes());
    val.setSize((int)Math.min(val.getLength(), bytes));
    context.write(key, val);
  }
}
 

import org.apache.hadoop.io.LongWritable; //导入方法依赖的package包/类
/** Map method.
 * @param offset samples starting from the (offset+1)th sample.
 * @param size the number of samples for this map
 * @param context output {ture->numInside, false->numOutside}
 */
public void map(LongWritable offset,
                LongWritable size,
                Context context) 
    throws IOException, InterruptedException {

  final HaltonSequence haltonsequence = new HaltonSequence(offset.get());
  long numInside = 0L;
  long numOutside = 0L;

  for(long i = 0; i < size.get(); ) {
    //generate points in a unit square
    final double[] point = haltonsequence.nextPoint();

    //count points inside/outside of the inscribed circle of the square
    final double x = point[0] - 0.5;
    final double y = point[1] - 0.5;
    if (x*x + y*y > 0.25) {
      numOutside++;
    } else {
      numInside++;
    }

    //report status
    i++;
    if (i % 1000 == 0) {
      context.setStatus("Generated " + i + " samples.");
    }
  }

  //output map results
  context.write(new BooleanWritable(true), new LongWritable(numInside));
  context.write(new BooleanWritable(false), new LongWritable(numOutside));
}