本文整理汇总了Java中org.apache.hadoop.io.WritableComparable类的典型用法代码示例。如果您正苦于以下问题:Java WritableComparable类的具体用法?Java WritableComparable怎么用?Java WritableComparable使用的例子?那么, 这里精选的类代码示例或许可以为您提供帮助。
WritableComparable类属于org.apache.hadoop.io包,在下文中一共展示了WritableComparable类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: configure
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
public void configure(JobConf job) {
// 'key' == sortInput for sort-input; key == sortOutput for sort-output
key = deduceInputFile(job);
if (key == sortOutput) {
partitioner = new HashPartitioner<WritableComparable, Writable>();
// Figure the 'current' partition and no. of reduces of the 'sort'
try {
URI inputURI = new URI(job.get(JobContext.MAP_INPUT_FILE));
String inputFile = inputURI.getPath();
// part file is of the form part-r-xxxxx
partition = Integer.valueOf(inputFile.substring(
inputFile.lastIndexOf("part") + 7)).intValue();
noSortReducers = job.getInt(SORT_REDUCES, -1);
} catch (Exception e) {
System.err.println("Caught: " + e);
System.exit(-1);
}
}
}
示例2: map
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
/** The waiting function. The map exits once it gets a signal. Here the
* signal is the file existence.
*/
public void map(WritableComparable key, Writable val,
OutputCollector<WritableComparable, Writable> output,
Reporter reporter)
throws IOException {
if (shouldWait(id)) {
if (fs != null) {
while (!fs.exists(getSignalFile(id))) {
try {
reporter.progress();
synchronized (this) {
this.wait(1000); // wait for 1 sec
}
} catch (InterruptedException ie) {
System.out.println("Interrupted while the map was waiting for "
+ " the signal.");
break;
}
}
} else {
throw new IOException("Could not get the DFS!!");
}
}
}
示例3: reduce
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
/** The waiting function. The reduce exits once it gets a signal. Here the
* signal is the file existence.
*/
public void reduce(WritableComparable key, Iterator<Writable> val,
OutputCollector<WritableComparable, Writable> output,
Reporter reporter)
throws IOException {
if (fs != null) {
while (!fs.exists(signal)) {
try {
reporter.progress();
synchronized (this) {
this.wait(1000); // wait for 1 sec
}
} catch (InterruptedException ie) {
System.out.println("Interrupted while the map was waiting for the"
+ " signal.");
break;
}
}
} else {
throw new IOException("Could not get the DFS!!");
}
}
示例4: reduce
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
public void reduce(WritableComparable key, Iterator values,
OutputCollector output, Reporter reporter
) throws IOException {
if (first) {
first = false;
MapOutputFile mapOutputFile = new MROutputFiles();
mapOutputFile.setConf(conf);
Path input = mapOutputFile.getInputFile(0);
FileSystem fs = FileSystem.get(conf);
assertTrue("reduce input exists " + input, fs.exists(input));
SequenceFile.Reader rdr =
new SequenceFile.Reader(fs, input, conf);
assertEquals("is reduce input compressed " + input,
compressInput,
rdr.isCompressed());
rdr.close();
}
}
示例5: map
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
/**
* Given an output filename, write a bunch of random records to it.
*/
public void map(WritableComparable key,
Writable value,
Context context) throws IOException,InterruptedException {
int itemCount = 0;
while (numBytesToWrite > 0) {
int keyLength = minKeySize +
(keySizeRange != 0 ? random.nextInt(keySizeRange) : 0);
randomKey.setSize(keyLength);
randomizeBytes(randomKey.getBytes(), 0, randomKey.getLength());
int valueLength = minValueSize +
(valueSizeRange != 0 ? random.nextInt(valueSizeRange) : 0);
randomValue.setSize(valueLength);
randomizeBytes(randomValue.getBytes(), 0, randomValue.getLength());
context.write(randomKey, randomValue);
numBytesToWrite -= keyLength + valueLength;
context.getCounter(Counters.BYTES_WRITTEN).increment(keyLength + valueLength);
context.getCounter(Counters.RECORDS_WRITTEN).increment(1);
if (++itemCount % 200 == 0) {
context.setStatus("wrote record " + itemCount + ". " +
numBytesToWrite + " bytes left.");
}
}
context.setStatus("done with " + itemCount + " records.");
}
示例6: writePartitionFile
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
private static <T extends WritableComparable<?>> Path writePartitionFile(
String testname, Configuration conf, T[] splits) throws IOException {
final FileSystem fs = FileSystem.getLocal(conf);
final Path testdir = new Path(System.getProperty("test.build.data", "/tmp")
).makeQualified(fs);
Path p = new Path(testdir, testname + "/_partition.lst");
TotalOrderPartitioner.setPartitionFile(conf, p);
conf.setInt(MRJobConfig.NUM_REDUCES, splits.length + 1);
SequenceFile.Writer w = null;
try {
w = SequenceFile.createWriter(fs, conf, p,
splits[0].getClass(), NullWritable.class,
SequenceFile.CompressionType.NONE);
for (int i = 0; i < splits.length; ++i) {
w.append(splits[i], NullWritable.get());
}
} finally {
if (null != w)
w.close();
}
return p;
}
示例7: map
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
/** Run a FileOperation */
public void map(Text key, FileOperation value,
OutputCollector<WritableComparable<?>, Text> out, Reporter reporter
) throws IOException {
try {
value.run(jobconf);
++succeedcount;
reporter.incrCounter(Counter.SUCCEED, 1);
} catch (IOException e) {
++failcount;
reporter.incrCounter(Counter.FAIL, 1);
String s = "FAIL: " + value + ", " + StringUtils.stringifyException(e);
out.collect(null, new Text(s));
LOG.info(s);
} finally {
reporter.setStatus(getCountString());
}
}
示例8: getRecord
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
/**
* It a single record from the map file for the given index
*
* @param index Index, between 0 and numRecords()-1
* @return Value from the MapFile
* @throws IOException If an error occurs during reading
*/
public V getRecord(long index) throws IOException {
//First: determine which reader to read from...
int readerIdx = -1;
for (int i = 0; i < recordIndexesEachReader.size(); i++) {
Pair<Long, Long> p = recordIndexesEachReader.get(i);
if (index >= p.getFirst() && index <= p.getSecond()) {
readerIdx = i;
break;
}
}
if (readerIdx == -1) {
throw new IllegalStateException("Index not found in any reader: " + index);
}
WritableComparable key = indexToKey.getKeyForIndex(index);
Writable value = ReflectionUtils.newInstance(recordClass, null);
V v = (V) readers[readerIdx].get(key, value);
return v;
}
示例9: readSortedMap
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
public <K extends WritableComparable<? super K>>
SortedMapWritable<K> readSortedMap(SortedMapWritable<K> mw)
throws IOException {
if (mw == null) {
mw = new SortedMapWritable<K>();
}
int length = in.readMapHeader();
for (int i = 0; i < length; i++) {
@SuppressWarnings("unchecked")
K key = (K) read();
Writable value = read();
mw.put(key, value);
}
return mw;
}
示例10: getRecordWriter
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
public RecordWriter<WritableComparable<?>, Writable> getRecordWriter(
final FileSystem fs, JobConf job, String name,
final Progressable progress) throws IOException {
final Path segmentDumpFile = new Path(
FileOutputFormat.getOutputPath(job), name);
// Get the old copy out of the way
if (fs.exists(segmentDumpFile))
fs.delete(segmentDumpFile, true);
final PrintStream printStream = new PrintStream(
fs.create(segmentDumpFile));
return new RecordWriter<WritableComparable<?>, Writable>() {
public synchronized void write(WritableComparable<?> key, Writable value)
throws IOException {
printStream.println(value);
}
public synchronized void close(Reporter reporter) throws IOException {
printStream.close();
}
};
}
示例11: compare
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
@Override
public int compare(WritableComparable o1, WritableComparable o2) {
ReadKey rk1 = (ReadKey) o1;
ReadKey rk2 = (ReadKey) o2;
if (rk1.sample.equals(rk2.sample)) {
if (rk1.sequence.equals(rk2.sequence)) {
return 0;
} else {
return rk1.sequence.compareTo(rk2.sequence);
}
} else {
return rk1.sample.compareTo(rk2.sample);
}
}
示例12: createRecordReader
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
@Override
public RecordReader<WritableComparable, HCatRecord>
createRecordReader(InputSplit split,
TaskAttemptContext taskContext)
throws IOException, InterruptedException {
LOG.debug("Creating a SqoopHCatRecordReader");
return new SqoopHCatRecordReader(split, taskContext, this);
}
示例13: createHCatRecordReader
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
public RecordReader<WritableComparable, HCatRecord>
createHCatRecordReader(InputSplit split,
TaskAttemptContext taskContext)
throws IOException, InterruptedException {
LOG.debug("Creating a base HCatRecordReader");
return super.createRecordReader(split, taskContext);
}
示例14: map
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
@Override
public void map(WritableComparable key, HCatRecord value,
Context context) throws IOException, InterruptedException {
try {
recsRead.add(value);
readRecordCount++;
} catch (Exception e) {
if (LOG.isDebugEnabled()) {
e.printStackTrace(System.err);
}
throw new IOException(e);
}
}
示例15: getContent
import org.apache.hadoop.io.WritableComparable; //导入依赖的package包/类
private String[] getContent(Configuration conf, Path path) throws Exception {
ClassLoader prevClassLoader = ClassLoaderStack.addJarFile(
new Path(new Path(new SqoopOptions().getJarOutputDir()), getTableName() + ".jar").toString(),
getTableName());
FileSystem fs = FileSystem.getLocal(conf);
FileStatus[] stats = fs.listStatus(path);
Path[] paths = new Path[stats.length];
for (int i = 0; i < stats.length; i++) {
paths[i] = stats[i].getPath();
}
// Read all the files adding the value lines to the list.
List<String> strings = new ArrayList<String>();
for (Path filePath : paths) {
if (filePath.getName().startsWith("_") || filePath.getName().startsWith(".")) {
continue;
}
// Need to use new configuration object so that it has the proper classloaders.
SequenceFile.Reader reader = new SequenceFile.Reader(fs, filePath, new Configuration());
WritableComparable key = (WritableComparable)
reader.getKeyClass().newInstance();
Writable value = (Writable) reader.getValueClass().newInstance();
while (reader.next(key, value)) {
strings.add(value.toString());
}
}
ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
return strings.toArray(new String[0]);
}