本文整理汇总了Java中org.apache.pig.tools.pigstats.OutputStats类的典型用法代码示例。如果您正苦于以下问题:Java OutputStats类的具体用法?Java OutputStats怎么用?Java OutputStats使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
OutputStats类属于org.apache.pig.tools.pigstats包,在下文中一共展示了OutputStats类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: getJobs
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
/**
* Retrieves a list of Job objects from the PigStats object
* @param stats
* @return A list of ExecJob objects
*/
protected List<ExecJob> getJobs(PigStats stats) {
LinkedList<ExecJob> jobs = new LinkedList<ExecJob>();
JobGraph jGraph = stats.getJobGraph();
Iterator<JobStats> iter = jGraph.iterator();
while (iter.hasNext()) {
JobStats js = iter.next();
for (OutputStats output : js.getOutputs()) {
if (js.isSuccessful()) {
jobs.add(new HJob(HJob.JOB_STATUS.COMPLETED, pigContext, output
.getPOStore(), output.getAlias(), stats));
} else {
HJob hjob = new HJob(HJob.JOB_STATUS.FAILED, pigContext, output
.getPOStore(), output.getAlias(), stats);
hjob.setException(js.getException());
jobs.add(hjob);
}
}
}
return jobs;
}
示例2: addOneOutputStats
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
private void addOneOutputStats(POStore sto) {
long records = -1;
if (sto.isMultiStore()) {
Long n = multiStoreCounters.get(MRPigStatsUtil.getMultiStoreCounterName(sto));
if (n != null) records = n;
} else {
records = mapOutputRecords;
}
long bytes = getOutputSize(sto, conf);
String location = sto.getSFile().getFileName();
OutputStats ds = new OutputStats(location, bytes, records,
(state == JobState.SUCCESS));
ds.setPOStore(sto);
ds.setConf(conf);
outputs.add(ds);
if (state == JobState.SUCCESS) {
MRScriptState.get().emitOutputCompletedNotification(ds);
}
}
示例3: accumulateStats
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
/**
* Updates the statistics after a DAG is finished.
*/
public void accumulateStats(TezJob tezJob) throws IOException {
DAGStatus dagStatus = tezJob.getDAGStatus();
TezDAGStats tezDAGStats = tezDAGStatsMap.get(tezJob.getName());
if (dagStatus == null) {
tezDAGStats.setSuccessful(false);
tezScriptState.emitJobFailedNotification(tezDAGStats);
return;
} else {
tezDAGStats.accumulateStats(tezJob);
for(OutputStats output: tezDAGStats.getOutputs()) {
tezScriptState.emitOutputCompletedNotification(output);
}
if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) {
tezDAGStats.setSuccessful(true);
tezScriptState.emitjobFinishedNotification(tezDAGStats);
} else if (dagStatus.getState() == DAGStatus.State.FAILED) {
tezDAGStats.setSuccessful(false);
tezDAGStats.setErrorMsg(tezJob.getDiagnostics());
tezScriptState.emitJobFailedNotification(tezDAGStats);
}
tezScriptState.dagCompletedNotification(tezJob.getName(), tezDAGStats);
}
}
示例4: outputCompletedNotification
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Override
public void outputCompletedNotification(String scriptId,
OutputStats outputStats) {
synchronized (listeners) {
for (PigProgressNotificationListener listener : listeners) {
listener.outputCompletedNotification(scriptId, outputStats);
}
}
}
示例5: testDuplicateCounterName2
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test
public void testDuplicateCounterName2() throws Exception {
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = filter A by a0 > 3;");
w1.println("store A into 'output';");
w1.println("store B into 'tmp/output';");
w1.close();
try {
String[] args = { PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener());
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<OutputStats> outputs = stats.getOutputStats();
assertEquals(2, outputs.size());
for (OutputStats outstats : outputs) {
if (outstats.getLocation().endsWith("tmp/output")) {
assertEquals(2, outstats.getNumberRecords());
} else {
assertEquals(5, outstats.getNumberRecords());
}
}
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
Util.deleteFile(cluster, "tmp/output");
}
}
示例6: testEmptyFileCounter2
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test //PIG-1893
public void testEmptyFileCounter2() throws Exception {
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = filter A by a0 < 0;");
w1.println("store A into '" + OUTPUT_FILE + "';");
w1.println("store B into 'output2';");
w1.close();
try {
String[] args = { PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener());
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<OutputStats> outputs = stats.getOutputStats();
assertEquals(2, outputs.size());
for (OutputStats outstats : outputs) {
if (outstats.getLocation().endsWith("output2")) {
assertEquals(0, outstats.getNumberRecords());
} else {
assertEquals(5, outstats.getNumberRecords());
}
}
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
Util.deleteFile(cluster, "output2");
}
}
示例7: testDisablePigCounters
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters() throws Exception {
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("C = join A by a0, B by a0;");
w1.println("store C into '" + OUTPUT_FILE + "';");
w1.close();
try {
String[] args = { "-Dpig.disable.counter=true", PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener());
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<InputStats> inputs = stats.getInputStats();
assertEquals(2, inputs.size());
for (InputStats instats : inputs) {
// the multi-input counters are disabled
assertEquals(-1, instats.getNumberRecords());
}
List<OutputStats> outputs = stats.getOutputStats();
assertEquals(1, outputs.size());
OutputStats outstats = outputs.get(0);
assertEquals(9, outstats.getNumberRecords());
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
}
}
示例8: testDisablePigCounters2
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters2() throws Exception {
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = filter A by a0 > 3;");
w1.println("store A into 'output';");
w1.println("store B into 'tmp/output';");
w1.close();
try {
String[] args = { "-Dpig.disable.counter=true", PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener());
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<OutputStats> outputs = stats.getOutputStats();
assertEquals(2, outputs.size());
for (OutputStats outstats : outputs) {
// the multi-output counters are disabled
assertEquals(-1, outstats.getNumberRecords());
}
List<InputStats> inputs = stats.getInputStats();
assertEquals(1, inputs.size());
InputStats instats = inputs.get(0);
assertEquals(5, instats.getNumberRecords());
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
Util.deleteFile(cluster, "tmp/output");
}
}
示例9: outputCompletedNotification
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Override
public void outputCompletedNotification(String scriptId,
OutputStats outputStats) {
synchronized (listeners) {
for (PigProgressNotificationListener listener : listeners) {
listener.outputCompletedNotification(scriptId, outputStats);
}
}
}
示例10: getDisplayString
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Override
public String getDisplayString() {
StringBuilder sb = new StringBuilder();
String id = (jobId == null) ? "N/A" : jobId.toString();
if (state == JobState.FAILED) {
sb.append(id).append("\t")
.append(getAlias()).append("\t")
.append(getFeature()).append("\t");
if (state == JobState.FAILED) {
sb.append("Message: ").append(getErrorMessage()).append("\t");
}
} else if (state == JobState.SUCCESS) {
sb.append(id).append("\t")
.append(numberMaps).append("\t")
.append(numberReduces).append("\t");
appendStat(maxMapTime, sb);
appendStat(minMapTime, sb);
appendStat(avgMapTime, sb);
appendStat(medianMapTime, sb);
appendStat(maxReduceTime, sb);
appendStat(minReduceTime, sb);
appendStat(avgReduceTime, sb);
appendStat(medianReduceTime, sb);
sb.append(getAlias()).append("\t")
.append(getFeature()).append("\t");
}
for (OutputStats os : outputs) {
sb.append(os.getLocation()).append(",");
}
sb.append("\n");
return sb.toString();
}
示例11: getJobs
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
/**
* Retrieves a list of Job objects from the PigStats object
* @param stats
* @return A list of ExecJob objects
*/
protected List<ExecJob> getJobs(PigStats stats) {
LinkedList<ExecJob> jobs = new LinkedList<ExecJob>();
if (stats instanceof EmptyPigStats) {
HJob job = new HJob(HJob.JOB_STATUS.COMPLETED, pigContext, stats.result(null)
.getPOStore(), null);
jobs.add(job);
return jobs;
}
JobGraph jGraph = stats.getJobGraph();
Iterator<JobStats> iter = jGraph.iterator();
while (iter.hasNext()) {
JobStats js = iter.next();
for (OutputStats output : js.getOutputs()) {
if (js.isSuccessful()) {
jobs.add(new HJob(HJob.JOB_STATUS.COMPLETED, pigContext, output
.getPOStore(), output.getAlias(), stats));
} else {
HJob hjob = new HJob(HJob.JOB_STATUS.FAILED, pigContext, output
.getPOStore(), output.getAlias(), stats);
hjob.setException(js.getException());
jobs.add(hjob);
}
}
}
return jobs;
}
示例12: testDuplicateCounterName2
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test
public void testDuplicateCounterName2() throws Exception {
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = filter A by a0 > 3;");
w1.println("store A into 'output';");
w1.println("store B into 'tmp/output';");
w1.close();
try {
String[] args = { "-x", execType, PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<OutputStats> outputs = stats.getOutputStats();
assertEquals(2, outputs.size());
for (OutputStats outstats : outputs) {
if (outstats.getLocation().endsWith("tmp/output")) {
assertEquals(2, outstats.getNumberRecords());
} else {
assertEquals(5, outstats.getNumberRecords());
}
}
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
Util.deleteFile(cluster, "tmp/output");
}
}
示例13: testEmptyFileCounter2
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test //PIG-1893
public void testEmptyFileCounter2() throws Exception {
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = filter A by a0 < 0;");
w1.println("store A into '" + OUTPUT_FILE + "';");
w1.println("store B into 'output2';");
w1.close();
try {
String[] args = { "-x", execType, PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<OutputStats> outputs = stats.getOutputStats();
assertEquals(2, outputs.size());
for (OutputStats outstats : outputs) {
if (outstats.getLocation().endsWith("output2")) {
assertEquals(0, outstats.getNumberRecords());
} else {
assertEquals(5, outstats.getNumberRecords());
}
}
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
Util.deleteFile(cluster, "output2");
}
}
示例14: testDisablePigCounters
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters() throws Exception {
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("C = join A by a0, B by a0;");
w1.println("store C into '" + OUTPUT_FILE + "';");
w1.close();
try {
String[] args = {"-Dpig.disable.counter=true", "-x", execType, PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<InputStats> inputs = stats.getInputStats();
assertEquals(2, inputs.size());
if (execType.equals("tez")) {
assertEquals(5, inputs.get(0).getNumberRecords());
assertEquals(5, inputs.get(1).getNumberRecords());
} else {
for (InputStats instats : inputs) {
// the multi-input counters are disabled
assertEquals(-1, instats.getNumberRecords());
}
}
List<OutputStats> outputs = stats.getOutputStats();
assertEquals(1, outputs.size());
OutputStats outstats = outputs.get(0);
assertEquals(9, outstats.getNumberRecords());
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
}
}
示例15: testDisablePigCounters2
import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters2() throws Exception {
PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
w1.println("B = filter A by a0 > 3;");
w1.println("store A into 'output';");
w1.println("store B into 'tmp/output';");
w1.close();
try {
String[] args = { "-Dpig.disable.counter=true", "-x", execType, PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));
assertTrue(stats.isSuccessful());
assertEquals(1, stats.getNumberJobs());
List<OutputStats> outputs = stats.getOutputStats();
assertEquals(2, outputs.size());
if (execType.equals("tez")) {
assertEquals(outputs.get(0).getNumberRecords(), 5);
assertEquals(outputs.get(1).getNumberRecords(), 2);
} else {
for (OutputStats outstats : outputs) {
// the multi-output counters are disabled
assertEquals(-1, outstats.getNumberRecords());
}
}
List<InputStats> inputs = stats.getInputStats();
assertEquals(1, inputs.size());
InputStats instats = inputs.get(0);
assertEquals(5, instats.getNumberRecords());
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);
Util.deleteFile(cluster, "tmp/output");
}
}