本文整理汇总了Java中org.apache.pig.tools.pigstats.JobStats.getOutputSize方法的典型用法代码示例。如果您正苦于以下问题:Java JobStats.getOutputSize方法的具体用法?Java JobStats.getOutputSize怎么用?Java JobStats.getOutputSize使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.pig.tools.pigstats.JobStats
的用法示例。
在下文中一共展示了JobStats.getOutputSize方法的7个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: testGetOuputSizeUsingNonFileBasedStorage4
import org.apache.pig.tools.pigstats.JobStats; //导入方法依赖的package包/类
@Test
public void testGetOuputSizeUsingNonFileBasedStorage4() throws Exception {
// Register a comma-separated list of readers in configuration, and
// verify that the one that supports a non-file-based uri is used.
Configuration conf = new Configuration();
conf.set(PigStatsOutputSizeReader.OUTPUT_SIZE_READER_KEY,
FileBasedOutputSizeReader.class.getName() + ","
+ DummyOutputSizeReader.class.getName());
// ClientSystemProps needs to be initialized to instantiate HBaseStorage
UDFContext.getUDFContext().setClientSystemProps(new Properties());
long outputSize = JobStats.getOutputSize(
createPOStoreForNonFileBasedSystem(new HBaseStorage("colName"), conf), conf);
assertEquals("The dummy output size reader always returns " + DummyOutputSizeReader.SIZE,
DummyOutputSizeReader.SIZE, outputSize);
}
示例2: testGetOuputSizeUsingNonFileBasedStorage5
import org.apache.pig.tools.pigstats.JobStats; //导入方法依赖的package包/类
@Test
public void testGetOuputSizeUsingNonFileBasedStorage5() throws Exception {
Configuration conf = new Configuration();
long size = 2L * 1024 * 1024 * 1024;
long outputSize = JobStats.getOutputSize(
createPOStoreForFileBasedSystem(size, new PigStorageWithStatistics(), conf), conf);
// By default, FileBasedOutputSizeReader is used to compute the size of output.
assertEquals("The returned output size is expected to be the same as the file size",
size, outputSize);
// Now add PigStorageWithStatistics to the unsupported store funcs list, and
// verify that JobStats.getOutputSize() returns -1.
conf.set(PigStatsOutputSizeReader.OUTPUT_SIZE_READER_UNSUPPORTED,
PigStorageWithStatistics.class.getName());
outputSize = JobStats.getOutputSize(
createPOStoreForFileBasedSystem(size, new PigStorageWithStatistics(), conf), conf);
assertEquals("The default output size reader returns -1 for unsupported store funcs",
-1, outputSize);
}
示例3: testGetOuputSizeUsingFileBasedStorage
import org.apache.pig.tools.pigstats.JobStats; //导入方法依赖的package包/类
@Test
public void testGetOuputSizeUsingFileBasedStorage() throws Exception {
// By default, FileBasedOutputSizeReader is used to compute the size of output.
Configuration conf = new Configuration();
long size = 2L * 1024 * 1024 * 1024;
long outputSize = JobStats.getOutputSize(
createPOStoreForFileBasedSystem(size, new PigStorageWithStatistics(), conf), conf);
assertEquals("The returned output size is expected to be the same as the file size",
size, outputSize);
}
示例4: testGetOuputSizeUsingNonFileBasedStorage1
import org.apache.pig.tools.pigstats.JobStats; //导入方法依赖的package包/类
@Test
public void testGetOuputSizeUsingNonFileBasedStorage1() throws Exception {
// By default, FileBasedOutputSizeReader is used to compute the size of output.
Configuration conf = new Configuration();
// ClientSystemProps is needed to instantiate HBaseStorage
UDFContext.getUDFContext().setClientSystemProps(new Properties());
long outputSize = JobStats.getOutputSize(
createPOStoreForNonFileBasedSystem(new HBaseStorage("colName"), conf), conf);
assertEquals("The default output size reader returns -1 for a non-file-based uri",
-1, outputSize);
}
示例5: testGetOuputSizeUsingNonFileBasedStorage2
import org.apache.pig.tools.pigstats.JobStats; //导入方法依赖的package包/类
@Test
public void testGetOuputSizeUsingNonFileBasedStorage2() throws Exception {
// Register a custom output size reader in configuration
Configuration conf = new Configuration();
conf.set(PigStatsOutputSizeReader.OUTPUT_SIZE_READER_KEY,
DummyOutputSizeReader.class.getName());
// ClientSystemProps is needed to instantiate HBaseStorage
UDFContext.getUDFContext().setClientSystemProps(new Properties());
long outputSize = JobStats.getOutputSize(
createPOStoreForNonFileBasedSystem(new HBaseStorage("colName"), conf), conf);
assertEquals("The dummy output size reader always returns " + DummyOutputSizeReader.SIZE,
DummyOutputSizeReader.SIZE, outputSize);
}
示例6: testGetOuputSizeUsingNonFileBasedStorage3
import org.apache.pig.tools.pigstats.JobStats; //导入方法依赖的package包/类
@Test(expected = RuntimeException.class)
public void testGetOuputSizeUsingNonFileBasedStorage3() throws Exception {
// Register an invalid output size reader in configuration, and verify
// that an exception is thrown at run-time.
Configuration conf = new Configuration();
conf.set(PigStatsOutputSizeReader.OUTPUT_SIZE_READER_KEY, "bad_output_size_reader");
// ClientSystemProps is needed to instantiate HBaseStorage
UDFContext.getUDFContext().setClientSystemProps(new Properties());
JobStats.getOutputSize(
createPOStoreForNonFileBasedSystem(new HBaseStorage("colName"), conf), conf);
}
示例7: addOutputStatistics
import org.apache.pig.tools.pigstats.JobStats; //导入方法依赖的package包/类
public void addOutputStatistics() {
if (stores == null) {
return;
}
for (POStore sto : stores) {
if (sto.isTmpStore()) {
continue;
}
long records = -1;
long hdfsBytesWritten = -1;
String filename = sto.getSFile().getFileName();
if (counters != null) {
if (sto.isMultiStore()) {
Map<String, Long> msGroup = counters.get(PigStatsUtil.MULTI_STORE_COUNTER_GROUP);
if (msGroup != null) {
multiStoreCounters.putAll(msGroup);
Long n = msGroup.get(PigStatsUtil.getMultiStoreCounterName(sto));
if (n != null) records = n;
}
} else if (counters.get(TASK_COUNTER_GROUP) != null
&& counters.get(TASK_COUNTER_GROUP).get(TaskCounter.OUTPUT_RECORDS.name()) != null) {
records = counters.get(TASK_COUNTER_GROUP).get(TaskCounter.OUTPUT_RECORDS.name());
}
if (records != -1) {
if (this.isMapOpts) {
mapOutputRecords += records;
} else {
reduceOutputRecords += records;
}
}
}
/* TODO: Need to check FILE_BYTES_WRITTEN for local mode */
if (!sto.isMultiStore() && counters.get(FS_COUNTER_GROUP)!= null &&
counters.get(FS_COUNTER_GROUP).get(PigStatsUtil.HDFS_BYTES_WRITTEN) != null) {
hdfsBytesWritten = counters.get(FS_COUNTER_GROUP).get(PigStatsUtil.HDFS_BYTES_WRITTEN);
} else {
try {
hdfsBytesWritten = JobStats.getOutputSize(sto, conf);
} catch (Exception e) {
LOG.warn("Error while getting the bytes written for the output " + sto.getSFile(), e);
}
}
OutputStats os = new OutputStats(filename, hdfsBytesWritten,
records, (state == JobState.SUCCESS));
os.setPOStore(sto);
os.setConf(conf);
outputs.add(os);
}
}