Java JobStats.getOutputSize方法代码示例

import org.apache.pig.tools.pigstats.JobStats; //导入方法依赖的package包/类
@Test
public void testGetOuputSizeUsingNonFileBasedStorage4() throws Exception {
    // Register a comma-separated list of readers in configuration, and
    // verify that the one that supports a non-file-based uri is used.
    Configuration conf = new Configuration();
    conf.set(PigStatsOutputSizeReader.OUTPUT_SIZE_READER_KEY,
            FileBasedOutputSizeReader.class.getName() + ","
                    + DummyOutputSizeReader.class.getName());

    // ClientSystemProps needs to be initialized to instantiate HBaseStorage
    UDFContext.getUDFContext().setClientSystemProps(new Properties());
    long outputSize = JobStats.getOutputSize(
            createPOStoreForNonFileBasedSystem(new HBaseStorage("colName"), conf), conf);

    assertEquals("The dummy output size reader always returns " + DummyOutputSizeReader.SIZE,
            DummyOutputSizeReader.SIZE, outputSize);
}
 

import org.apache.pig.tools.pigstats.JobStats; //导入方法依赖的package包/类
@Test
public void testGetOuputSizeUsingNonFileBasedStorage5() throws Exception {
    Configuration conf = new Configuration();

    long size = 2L * 1024 * 1024 * 1024;
    long outputSize = JobStats.getOutputSize(
            createPOStoreForFileBasedSystem(size, new PigStorageWithStatistics(), conf), conf);

    // By default, FileBasedOutputSizeReader is used to compute the size of output.
    assertEquals("The returned output size is expected to be the same as the file size",
            size, outputSize);

    // Now add PigStorageWithStatistics to the unsupported store funcs list, and
    // verify that JobStats.getOutputSize() returns -1.
    conf.set(PigStatsOutputSizeReader.OUTPUT_SIZE_READER_UNSUPPORTED,
            PigStorageWithStatistics.class.getName());

    outputSize = JobStats.getOutputSize(
            createPOStoreForFileBasedSystem(size, new PigStorageWithStatistics(), conf), conf);
    assertEquals("The default output size reader returns -1 for unsupported store funcs",
            -1, outputSize);
}
 

import org.apache.pig.tools.pigstats.JobStats; //导入方法依赖的package包/类
@Test
public void testGetOuputSizeUsingFileBasedStorage() throws Exception {
    // By default, FileBasedOutputSizeReader is used to compute the size of output.
    Configuration conf = new Configuration();

    long size = 2L * 1024 * 1024 * 1024;
    long outputSize = JobStats.getOutputSize(
            createPOStoreForFileBasedSystem(size, new PigStorageWithStatistics(), conf), conf);

    assertEquals("The returned output size is expected to be the same as the file size",
            size, outputSize);
}
 

import org.apache.pig.tools.pigstats.JobStats; //导入方法依赖的package包/类
@Test
public void testGetOuputSizeUsingNonFileBasedStorage1() throws Exception {
    // By default, FileBasedOutputSizeReader is used to compute the size of output.
    Configuration conf = new Configuration();

    // ClientSystemProps is needed to instantiate HBaseStorage
    UDFContext.getUDFContext().setClientSystemProps(new Properties());
    long outputSize = JobStats.getOutputSize(
            createPOStoreForNonFileBasedSystem(new HBaseStorage("colName"), conf), conf);

    assertEquals("The default output size reader returns -1 for a non-file-based uri",
            -1, outputSize);
}
 

import org.apache.pig.tools.pigstats.JobStats; //导入方法依赖的package包/类
@Test
public void testGetOuputSizeUsingNonFileBasedStorage2() throws Exception {
    // Register a custom output size reader in configuration
    Configuration conf = new Configuration();
    conf.set(PigStatsOutputSizeReader.OUTPUT_SIZE_READER_KEY,
            DummyOutputSizeReader.class.getName());

    // ClientSystemProps is needed to instantiate HBaseStorage
    UDFContext.getUDFContext().setClientSystemProps(new Properties());
    long outputSize = JobStats.getOutputSize(
            createPOStoreForNonFileBasedSystem(new HBaseStorage("colName"), conf), conf);

    assertEquals("The dummy output size reader always returns " + DummyOutputSizeReader.SIZE,
            DummyOutputSizeReader.SIZE, outputSize);
}
 

import org.apache.pig.tools.pigstats.JobStats; //导入方法依赖的package包/类
@Test(expected = RuntimeException.class)
public void testGetOuputSizeUsingNonFileBasedStorage3() throws Exception {
    // Register an invalid output size reader in configuration, and verify
    // that an exception is thrown at run-time.
    Configuration conf = new Configuration();
    conf.set(PigStatsOutputSizeReader.OUTPUT_SIZE_READER_KEY, "bad_output_size_reader");

    // ClientSystemProps is needed to instantiate HBaseStorage
    UDFContext.getUDFContext().setClientSystemProps(new Properties());
    JobStats.getOutputSize(
            createPOStoreForNonFileBasedSystem(new HBaseStorage("colName"), conf), conf);
}
 

import org.apache.pig.tools.pigstats.JobStats; //导入方法依赖的package包/类
public void addOutputStatistics() {
    if (stores == null) {
        return;
    }

    for (POStore sto : stores) {
        if (sto.isTmpStore()) {
            continue;
        }
        long records = -1;
        long hdfsBytesWritten = -1;
        String filename = sto.getSFile().getFileName();
        if (counters != null) {
            if (sto.isMultiStore()) {
                Map<String, Long> msGroup = counters.get(PigStatsUtil.MULTI_STORE_COUNTER_GROUP);
                if (msGroup != null) {
                    multiStoreCounters.putAll(msGroup);
                    Long n = msGroup.get(PigStatsUtil.getMultiStoreCounterName(sto));
                    if (n != null) records = n;
                }
            } else if (counters.get(TASK_COUNTER_GROUP) != null
                    && counters.get(TASK_COUNTER_GROUP).get(TaskCounter.OUTPUT_RECORDS.name()) != null) {
                records = counters.get(TASK_COUNTER_GROUP).get(TaskCounter.OUTPUT_RECORDS.name());
            }
            if (records != -1) {
                if (this.isMapOpts) {
                    mapOutputRecords += records;
                } else {
                    reduceOutputRecords += records;
                }
            }
        }
        /* TODO: Need to check FILE_BYTES_WRITTEN for local mode */
        if (!sto.isMultiStore() && counters.get(FS_COUNTER_GROUP)!= null &&
                counters.get(FS_COUNTER_GROUP).get(PigStatsUtil.HDFS_BYTES_WRITTEN) != null) {
            hdfsBytesWritten = counters.get(FS_COUNTER_GROUP).get(PigStatsUtil.HDFS_BYTES_WRITTEN);
        } else {
            try {
                hdfsBytesWritten = JobStats.getOutputSize(sto, conf);
            } catch (Exception e) {
                LOG.warn("Error while getting the bytes written for the output " + sto.getSFile(), e);
            }
        }

        OutputStats os = new OutputStats(filename, hdfsBytesWritten,
                records, (state == JobState.SUCCESS));
        os.setPOStore(sto);
        os.setConf(conf);
        outputs.add(os);
    }
}