本文整理汇总了Java中org.apache.hadoop.examples.SleepJob.setConf方法的典型用法代码示例。如果您正苦于以下问题:Java SleepJob.setConf方法的具体用法?Java SleepJob.setConf怎么用?Java SleepJob.setConf使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.hadoop.examples.SleepJob
的用法示例。
在下文中一共展示了SleepJob.setConf方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: doAnAction
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
@Override
public void doAnAction() throws Exception {
System.out.println("==============================\n" +
"Submitting job\n" +
"==================================");
SleepJob job = new SleepJob();
job.setConf(conf);
JobConf jobConf = job.setupJobConf(1, 0, 1, 1, 1, 1);
JobClient jc = new JobClient(jobConf);
RunningJob rj;
try {
rj = jc.submitJob(jobConf);
} catch (IOException e) {
// Job submission is not idempotent, so ignore a failure
System.out.println("==============================\n" +
"Job submission failed. Ignore.\n" +
"==================================");
return;
}
System.out.println("==============================\n" +
"Successfully submitted job " + rj.getJobID() + "\n" +
"==================================");
if (!jc.monitorAndPrintJob(jobConf, rj)) {
throw new IOException("Job failed! " + rj.getFailureInfo());
}
}
示例2: testJobSummaryInfoOfKilledJob
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
/**
* Verifying the job summary information for killed job.
*/
@Test
public void testJobSummaryInfoOfKilledJob() throws IOException,
InterruptedException {
SleepJob job = new SleepJob();
conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false);
job.setConf(conf);
conf = job.setupJobConf(2, 1, 4000, 4000, 100, 100);
JobConf jobConf = new JobConf(conf);
RunningJob runJob = jobClient.submitJob(jobConf);
JobID jobId = runJob.getID();
Assert.assertTrue("Job has not been started for 1 min.",
jtClient.isJobStarted(jobId));
jobClient.killJob(jobId);
LOG.info("Waiting till the job is completed...");
Assert.assertTrue("Job has not been completed for 1 min.",
jtClient.isJobStopped(jobId));
JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
Assert.assertEquals("Job has not been succeeded",
jInfo.getStatus().getRunState(), JobStatus.KILLED);
verifyJobSummaryInfo(jInfo,jobId);
}
示例3: runSleepJob
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
public JobID runSleepJob(boolean signalJob) throws Exception{
SleepJob job = new SleepJob();
job.setConf(conf);
conf = job.setupJobConf(5, 1, 100, 5, 100, 5);
JobConf jconf = new JobConf(conf);
//Controls the job till all verification is done
FinishTaskControlAction.configureControlActionForJob(conf);
//Submitting the job
RunningJob rJob = cluster.getJTClient().getClient().submitJob(jconf);
JobID jobId = rJob.getID();
JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
LOG.info("jInfo is :" + jInfo);
boolean jobStarted = cluster.getJTClient().isJobStarted(jobId);
Assert.assertTrue("Job has not started even after a minute",
jobStarted );
if(signalJob) {
cluster.signalAllTasks(jobId);
Assert.assertTrue("Job has not stopped yet",
cluster.getJTClient().isJobStopped(jobId));
}
return jobId;
}
示例4: setupJobAndRun
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
private void setupJobAndRun() throws IOException {
SleepJob job = new SleepJob();
job.setConf(conf);
conf = job.setupJobConf(3, 1, 60000, 100, 60000, 100);
JobConf jobConf = new JobConf(conf);
cleanup(outputDir, conf);
jtClient = cluster.getJTClient();
JobClient client = jtClient.getClient();
wovenClient = cluster.getJTClient().getProxy();
RunningJob runJob = client.submitJob(jobConf);
jID = runJob.getID();
jInfo = wovenClient.getJobInfo(jID);
Assert.assertNotNull("Job information is null",jInfo);
Assert.assertTrue("Job has not been started for 1 min.",
jtClient.isJobStarted(jID));
JobStatus jobStatus = jInfo.getStatus();
// Make sure that job should run and completes 40%.
while (jobStatus.getRunState() != JobStatus.RUNNING &&
jobStatus.mapProgress() < 0.4f) {
UtilsForTests.waitFor(100);
jobStatus = wovenClient.getJobInfo(jID).getStatus();
}
}
示例5: submitSleepJob
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
RunningJob submitSleepJob(final int numMappers, final int numReducers,
final long mapSleepTime,
final long reduceSleepTime, final boolean shouldComplete, String userInfo,
String queueName)
throws IOException, InterruptedException {
JobConf clientConf = new JobConf();
clientConf.set("mapred.job.tracker", "localhost:"
+ miniMRCluster.getJobTrackerPort());
UserGroupInformation ugi;
SleepJob job = new SleepJob();
job.setConf(clientConf);
clientConf = job.setupJobConf(numMappers, numReducers,
mapSleepTime, (int)mapSleepTime/100,
reduceSleepTime, (int)reduceSleepTime/100);
if (queueName != null) {
clientConf.setQueueName(queueName);
}
final JobConf jc = new JobConf(clientConf);
if (userInfo != null) {
String[] splits = userInfo.split(",");
String[] groups = new String[splits.length - 1];
System.arraycopy(splits, 1, groups, 0, splits.length - 1);
ugi = UserGroupInformation.createUserForTesting(splits[0], groups);
} else {
ugi = UserGroupInformation.getCurrentUser();
}
RunningJob rJob = ugi.doAs(new PrivilegedExceptionAction<RunningJob>() {
public RunningJob run() throws IOException {
if (shouldComplete) {
return JobClient.runJob(jc);
} else {
// Job should be submitted as 'userInfo'. So both the client as well as
// the configuration should point to the same UGI.
return new JobClient(jc).submitJob(jc);
}
}
});
return rJob;
}
示例6: testJobQueueInfoInJobSummary
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
/**
* Submit the job in different queue and verifying
* the job queue information in job summary
* after job is completed.
*/
@Test
public void testJobQueueInfoInJobSummary() throws IOException,
InterruptedException {
SleepJob job = new SleepJob();
job.setConf(conf);
conf = job.setupJobConf(2, 1, 4000, 4000, 100, 100);
conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false);
JobConf jobConf = new JobConf(conf);
JobQueueInfo [] queues = jobClient.getQueues();
for (JobQueueInfo queueInfo : queues ){
if (!queueInfo.getQueueName().equals("default")) {
queueName = queueInfo.getQueueName();
break;
}
}
Assert.assertNotNull("No multiple queues in the cluster.",queueName);
LOG.info("queueName:" + queueName);
jobConf.setQueueName(queueName);
RunningJob runJob = jobClient.submitJob(jobConf);
JobID jobId = runJob.getID();
Assert.assertTrue("Job has not been started for 1 min.",
jtClient.isJobStarted(jobId));
LOG.info("Waiting till the job is completed...");
Assert.assertTrue("Job has not been completed for 1 min.",
jtClient.isJobStopped(jobId));
JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
Assert.assertEquals("Job has not been succeeded",
jInfo.getStatus().getRunState(), JobStatus.SUCCEEDED);
verifyJobSummaryInfo(jInfo,jobId);
}
示例7: SleepJobRunnerThread
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
public SleepJobRunnerThread(Configuration conf, int nMappers,
int nReducers, int sleepTime) {
super();
jobToRun = new SleepJob();
jobToRun.setConf(conf);
this.nMappers = nMappers;
this.nReducers = nReducers;
this.sleepTime = sleepTime;
}
示例8: testFailedTaskJobStatus
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
/**
* Verifying the running job status whether it succeeds or not
* after failing some of its tasks.
*/
@Test
public void testFailedTaskJobStatus() throws IOException,
InterruptedException {
conf = remoteJTClient.getDaemonConf();
TaskInfo taskInfo = null;
SleepJob job = new SleepJob();
job.setConf(conf);
JobConf jobConf = job.setupJobConf(1, 1, 10000, 4000, 100, 100);
RunningJob runJob = jobClient.submitJob(jobConf);
JobID jobId = runJob.getID();
JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
Assert.assertTrue("Job has not been started for 1 min.",
jtClient.isJobStarted(jobId));
TaskInfo[] taskInfos = remoteJTClient.getTaskInfo(jobId);
for (TaskInfo taskinfo : taskInfos) {
if (!taskinfo.isSetupOrCleanup() && taskinfo.getTaskID().isMap()) {
taskInfo = taskinfo;
break;
}
}
Assert.assertTrue("Task has not been started for 1 min.",
jtClient.isTaskStarted(taskInfo));
// Fail the running task.
NetworkedJob networkJob = jobClient.new NetworkedJob(jInfo.getStatus());
TaskID tID = TaskID.downgrade(taskInfo.getTaskID());
TaskAttemptID taskAttID = new TaskAttemptID(tID , 0);
networkJob.killTask(taskAttID, true);
LOG.info("Waiting till the job is completed...");
while (!jInfo.getStatus().isJobComplete()) {
UtilsForTests.waitFor(100);
jInfo = remoteJTClient.getJobInfo(jobId);
}
Assert.assertEquals("JobStatus", JobStatus.SUCCEEDED,
jInfo.getStatus().getRunState());
}
示例9: testCacheFilesLocalization
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
/**
* Run the job with two distributed cache files and verify
* whether job is succeeded or not.
* @throws Exception
*/
@Test
public void testCacheFilesLocalization() throws Exception {
conf = wovenClient.getDaemonConf();
SleepJob job = new SleepJob();
job.setConf(conf);
JobConf jobConf = job.setupJobConf(4, 1, 4000, 4000, 1000, 1000);
DistributedCache.createSymlink(jobConf);
DistributedCache.addCacheFile(cacheFileURI1, jobConf);
DistributedCache.addCacheFile(cacheFileURI2, jobConf);
RunningJob runJob = jobClient.submitJob(jobConf);
JobID jobId = runJob.getID();
Assert.assertTrue("Job has not been started for 1 min.",
jtClient.isJobStarted(jobId));
TaskInfo[] taskInfos = wovenClient.getTaskInfo(jobId);
Assert.assertTrue("Cache File1 has not been localize",
checkLocalization(taskInfos,cacheFile1));
Assert.assertTrue("Cache File2 has not been localize",
checkLocalization(taskInfos,cacheFile2));
JobInfo jInfo = wovenClient.getJobInfo(jobId);
LOG.info("Waiting till the job is completed...");
while (!jInfo.getStatus().isJobComplete()) {
UtilsForTests.waitFor(100);
jInfo = wovenClient.getJobInfo(jobId);
}
Assert.assertEquals("Job has not been succeeded",
jInfo.getStatus().getRunState(), JobStatus.SUCCEEDED);
}
示例10: testDeleteCacheFileInDFSAfterLocalized
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
/**
* Run the job with distributed cache files and remove one cache
* file from the DFS when it is localized.verify whether the job
* is failed or not.
* @throws Exception
*/
@Test
public void testDeleteCacheFileInDFSAfterLocalized() throws Exception {
conf = wovenClient.getDaemonConf();
SleepJob job = new SleepJob();
job.setConf(conf);
JobConf jobConf = job.setupJobConf(4, 1, 4000, 4000, 1000, 1000);
cacheFileURI3 = createCacheFile(tmpFolderPath, cacheFile3);
DistributedCache.createSymlink(jobConf);
DistributedCache.addCacheFile(cacheFileURI3, jobConf);
RunningJob runJob = jobClient.submitJob(jobConf);
JobID jobId = runJob.getID();
Assert.assertTrue("Job has not been started for 1 min.",
jtClient.isJobStarted(jobId));
TaskInfo[] taskInfos = wovenClient.getTaskInfo(jobId);
boolean iscacheFileLocalized = checkLocalization(taskInfos,cacheFile3);
Assert.assertTrue("CacheFile has not been localized",
iscacheFileLocalized);
deleteCacheFile(new Path(tmpFolderPath, cacheFile3));
JobInfo jInfo = wovenClient.getJobInfo(jobId);
LOG.info("Waiting till the job is completed...");
while (!jInfo.getStatus().isJobComplete()) {
UtilsForTests.waitFor(100);
jInfo = wovenClient.getJobInfo(jobId);
}
Assert.assertEquals("Job has not been failed",
jInfo.getStatus().getRunState(), JobStatus.FAILED);
}
示例11: submitSleepJob
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
private RunningJob submitSleepJob(int numMappers, int numReducers,
long mapSleepTime, long reduceSleepTime,
boolean shouldComplete, String userInfo,
String queueName)
throws IOException {
JobConf clientConf = new JobConf();
clientConf.set("mapred.job.tracker", "localhost:"
+ miniMRCluster.getJobTrackerPort());
SleepJob job = new SleepJob();
job.setConf(clientConf);
clientConf = job.setupJobConf(numMappers, numReducers,
mapSleepTime, (int)mapSleepTime/100,
reduceSleepTime, (int)reduceSleepTime/100);
if (queueName != null) {
clientConf.setQueueName(queueName);
}
JobConf jc = new JobConf(clientConf);
if (userInfo != null) {
jc.set(UnixUserGroupInformation.UGI_PROPERTY_NAME, userInfo);
}
RunningJob rJob = null;
if (shouldComplete) {
rJob = JobClient.runJob(jc);
} else {
rJob = new JobClient(clientConf).submitJob(jc);
}
return rJob;
}
示例12: testFailingJobInitalization
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
/**
* Test case which checks if the jobs which fail initialization are removed
* from the {@link CapacityTaskScheduler} waiting queue.
*
* @throws Exception
*/
public void testFailingJobInitalization() throws Exception {
Properties schedulerProps = new Properties();
schedulerProps.put("mapred.capacity-scheduler.queue.default.capacity",
"100");
Properties clusterProps = new Properties();
clusterProps.put("mapred.tasktracker.map.tasks.maximum", String.valueOf(1));
clusterProps.put("mapred.tasktracker.reduce.tasks.maximum", String
.valueOf(1));
clusterProps.put("mapred.jobtracker.maxtasks.per.job", String.valueOf(1));
// cluster capacity 1 maps, 1 reduces
startCluster(1, clusterProps, schedulerProps);
JobConf conf = getJobConf();
conf.setSpeculativeExecution(false);
conf.set("mapred.committer.job.setup.cleanup.needed", "false");
conf.setNumTasksToExecutePerJvm(-1);
SleepJob sleepJob = new SleepJob();
sleepJob.setConf(conf);
JobConf job = sleepJob.setupJobConf(3, 3, 1, 1, 1, 1);
RunningJob rjob;
try {
rjob = runJob(job, false);
fail("The job should have thrown Exception");
} catch (Exception e) {
CapacityTaskScheduler scheduler = (CapacityTaskScheduler) getJobTracker()
.getTaskScheduler();
JobQueuesManager mgr = scheduler.jobQueuesManager;
assertEquals("Failed job present in Waiting queue", 0, mgr
.getWaitingJobCount("default"));
}
}
示例13: testJobSubmission
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
@Test
public void testJobSubmission() throws Exception {
Configuration conf = new Configuration(cluster.getConf());
SleepJob job = new SleepJob();
job.setConf(conf);
conf = job.setupJobConf(1, 1, 100, 100, 100, 100);
RunningJob rJob = cluster.getJTClient().submitAndVerifyJob(conf);
cluster.getJTClient().verifyJobHistory(rJob.getID());
}
示例14: testFailingJobInitalization
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
/**
* Test case which checks if the jobs which fail initialization are removed
* from the {@link CapacityTaskScheduler} waiting queue.
*
* @throws Exception
*/
@Test
public void testFailingJobInitalization() throws Exception {
Properties schedulerProps = new Properties();
schedulerProps.put("mapred.capacity-scheduler.queue.default.capacity",
"100");
Properties clusterProps = new Properties();
clusterProps.put("mapred.tasktracker.map.tasks.maximum", String.valueOf(1));
clusterProps.put("mapred.tasktracker.reduce.tasks.maximum", String
.valueOf(1));
clusterProps.put("mapred.jobtracker.maxtasks.per.job", String.valueOf(1));
// cluster capacity 1 maps, 1 reduces
startCluster(1, clusterProps, schedulerProps);
JobConf conf = getJobConf();
conf.setSpeculativeExecution(false);
conf.set("mapred.committer.job.setup.cleanup.needed", "false");
conf.setNumTasksToExecutePerJvm(-1);
SleepJob sleepJob = new SleepJob();
sleepJob.setConf(conf);
JobConf job = sleepJob.setupJobConf(3, 3, 1, 1, 1, 1);
RunningJob rjob;
try {
rjob = runJob(job, false);
fail("The job should have thrown Exception");
} catch (Exception e) {
CapacityTaskScheduler scheduler = (CapacityTaskScheduler) getJobTracker()
.getTaskScheduler();
JobQueuesManager mgr = scheduler.jobQueuesManager;
assertEquals("Failed job present in Waiting queue", 0, mgr
.getQueue("default").getNumWaitingJobs());
}
}
示例15: testJobSummaryInfoForDifferentUser
import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
@Test
public void testJobSummaryInfoForDifferentUser() throws Exception {
UserGroupInformation proxyUGI;
UserGroupInformation ugi = UserGroupInformation.getLoginUser();
ArrayList<String> users = cluster.getHadoopMultiUsersList();
Assert.assertTrue("proxy users are not found.", users.size() > 0);
if (conf.get("hadoop.security.authentication").equals("simple")) {
proxyUGI = UserGroupInformation.createRemoteUser(
users.get(0));
} else {
proxyUGI = UserGroupInformation.createProxyUser(
users.get(0), ugi);
}
SleepJob job = new SleepJob();
job.setConf(conf);
final JobConf jobConf = job.setupJobConf(2, 1, 2000, 2000, 100, 100);
final JobClient jClient =
proxyUGI.doAs(new PrivilegedExceptionAction<JobClient>() {
public JobClient run() throws IOException {
return new JobClient(jobConf);
}
});
RunningJob runJob = proxyUGI.doAs(
new PrivilegedExceptionAction<RunningJob>() {
public RunningJob run() throws IOException {
return jClient.submitJob(jobConf);
}
});
JobID jobId = runJob.getID();
Assert.assertTrue("Job has not been started for 1 min.",
jtClient.isJobStarted(jobId));
LOG.info("Waiting till the job is completed...");
Assert.assertTrue("Job has not been completed for 1 min.",
jtClient.isJobStopped(jobId));
JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
Assert.assertEquals("Job has not been succeeded",
jInfo.getStatus().getRunState(), JobStatus.SUCCEEDED);
verifyJobSummaryInfo(jInfo,jobId);
}