Java Output类代码示例

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
private List<JobInstance> listJobInstances(String project, String cube, long startTime, long endTime) {
    final List<JobInstance> result = Lists.newArrayList();
    final List<AbstractExecutable> executables = executableManager.getAllExecutables(startTime, endTime);
    final Map<String, Output> allOutputs = executableManager.getAllOutputs();
    for (AbstractExecutable executable : executables) {
        if (executable instanceof CubingJob) {
            String cubeName = CubingExecutableUtil.getCubeName(executable.getParams());
            boolean shouldExtract = false;
            if (cube == null || cube.equalsIgnoreCase(cubeName)) {
                if (project == null) {
                    shouldExtract = true;
                } else {
                    ProjectInstance projectInstance = projectManager.getProject(project);
                    if (projectInstance != null && projectInstance.containsRealization(RealizationType.CUBE, cubeName)) {
                        shouldExtract = true;
                    }
                }
            }

            if (shouldExtract) {
                result.add(parseToJobInstance((CubingJob) executable, allOutputs));
            }
        }
    }
    return result;
}
 

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
private JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i, Output stepOutput) {
    Preconditions.checkNotNull(stepOutput);
    JobInstance.JobStep result = new JobInstance.JobStep();
    result.setId(task.getId());
    result.setName(task.getName());
    result.setSequenceID(i);
    result.setStatus(parseToJobStepStatus(stepOutput.getState()));
    for (Map.Entry<String, String> entry : stepOutput.getExtra().entrySet()) {
        if (entry.getKey() != null && entry.getValue() != null) {
            result.putInfo(entry.getKey(), entry.getValue());
        }
    }
    result.setExecStartTime(AbstractExecutable.getStartTime(stepOutput));
    result.setExecEndTime(AbstractExecutable.getEndTime(stepOutput));
    if (task instanceof ShellExecutable) {
        result.setExecCmd(((ShellExecutable) task).getCmd());
    }
    if (task instanceof MapReduceExecutable) {
        result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams());
        result.setExecWaitTime(AbstractExecutable.getExtraInfoAsLong(stepOutput, MapReduceExecutable.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
    }
    if (task instanceof HadoopShellExecutable) {
        result.setExecCmd(((HadoopShellExecutable) task).getJobParams());
    }
    return result;
}
 

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
private void resumeAllRunningJobs() {
    for (final String id : executableManager.getAllJobIds()) {
        final Output output = executableManager.getOutput(id);
        AbstractExecutable executable = executableManager.getJob(id);
        if (output.getState() == ExecutableState.RUNNING && executable instanceof DefaultChainedExecutable) {
            try {
                if (!jobLock.isLocked(getLockPath(executable.getParam(SEGMENT_ID)))) {
                    executableManager.resumeRunningJobForce(executable.getId());
                    fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS);
                }
            } catch (Exception e) {
                logger.error("resume the job " + id + " fail in server: " + serverName, e);
            }
        }
    }
}
 

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
public List<JobInstance> innerSearchCheckpointJobs(final String cubeName, final String jobName,
        final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
    // prepare time range
    Calendar calendar = Calendar.getInstance();
    calendar.setTime(new Date());
    long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
    long timeEndInMillis = Long.MAX_VALUE;
    Set<ExecutableState> states = convertStatusEnumToStates(statusList);
    final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs(timeStartInMillis, timeEndInMillis);

    return Lists
            .newArrayList(FluentIterable
                    .from(innerSearchCheckpointJobs(cubeName, jobName, states, timeStartInMillis, timeEndInMillis,
                            allOutputs, false, projectName))
                    .transform(new Function<CheckpointExecutable, JobInstance>() {
                        @Override
                        public JobInstance apply(CheckpointExecutable checkpointExecutable) {
                            return JobInfoConverter.parseToJobInstanceQuietly(checkpointExecutable, allOutputs);
                        }
                    }));
}
 

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
private JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i) {
    JobInstance.JobStep result = new JobInstance.JobStep();
    result.setId(task.getId());
    result.setName(task.getName());
    result.setSequenceID(i);
    result.setStatus(parseToJobStepStatus(task.getStatus()));
    final Output output = getExecutableManager().getOutput(task.getId());
    for (Map.Entry<String, String> entry : output.getExtra().entrySet()) {
        if (entry.getKey() != null && entry.getValue() != null) {
            result.putInfo(entry.getKey(), entry.getValue());
        }
    }
    result.setExecStartTime(task.getStartTime());
    result.setExecEndTime(task.getEndTime());
    if (task instanceof ShellExecutable) {
        result.setExecCmd(((ShellExecutable) task).getCmd());
    }
    if (task instanceof MapReduceExecutable) {
        result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams());
        result.setExecWaitTime(((MapReduceExecutable) task).getMapReduceWaitTime() / 1000);
    }
    if (task instanceof HadoopShellExecutable) {
        result.setExecCmd(((HadoopShellExecutable) task).getJobParams());
    }
    return result;
}
 

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
private JobInstance parseToJobInstance(CubingJob cubeJob, Map<String, Output> outputs) {
    CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));

    Output output = outputs.get(cubeJob.getId());
    final JobInstance result = new JobInstance();
    result.setName(cubeJob.getName());
    if (cube != null) {
        result.setRelatedCube(cube.getDisplayName());
    } else {
        result.setRelatedCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
    }
    result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
    result.setLastModified(output.getLastModified());
    result.setSubmitter(cubeJob.getSubmitter());
    result.setUuid(cubeJob.getId());
    result.setType(CubeBuildTypeEnum.BUILD);
    result.setStatus(parseToJobStatus(output.getState()));
    result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
    result.setExecStartTime(AbstractExecutable.getStartTime(output));
    result.setExecEndTime(AbstractExecutable.getEndTime(output));
    result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
    result.setDuration(AbstractExecutable.getDuration(AbstractExecutable.getStartTime(output), AbstractExecutable.getEndTime(output), AbstractExecutable.getInterruptTime(output)) / 1000);
    for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
        AbstractExecutable task = cubeJob.getTasks().get(i);
        result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
    }
    return result;
}
 

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
public static JobInstance parseToJobInstanceQuietly(CubingJob job, Map<String, Output> outputs) {
    try {
        return parseToJobInstance(job, outputs);
    } catch (Exception e) {
        logger.error("Failed to parse job instance: uuid={}", job, e);
        return null;
    }
}
 

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
public static JobInstance parseToJobInstance(CubingJob job, Map<String, Output> outputs) {
    if (job == null) {
        logger.warn("job is null.");
        return null;
    }

    Output output = outputs.get(job.getId());
    if (output == null) {
        logger.warn("job output is null.");
        return null;
    }

    CubingJob cubeJob = (CubingJob) job;
    CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));

    final JobInstance result = new JobInstance();
    result.setName(job.getName());
    result.setRelatedCube(cube != null ? cube.getDisplayName() : CubingExecutableUtil.getCubeName(cubeJob.getParams()));
    result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
    result.setLastModified(output.getLastModified());
    result.setSubmitter(job.getSubmitter());
    result.setUuid(job.getId());
    result.setType(CubeBuildTypeEnum.BUILD);
    result.setStatus(parseToJobStatus(output.getState()));
    result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
    result.setExecStartTime(AbstractExecutable.getStartTime(output));
    result.setExecEndTime(AbstractExecutable.getEndTime(output));
    result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
    result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime(),
            result.getExecInterruptTime()) / 1000);
    for (int i = 0; i < job.getTasks().size(); ++i) {
        AbstractExecutable task = job.getTasks().get(i);
        result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
    }
    return result;
}
 

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
public static JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i, Output stepOutput) {
    JobInstance.JobStep result = new JobInstance.JobStep();
    result.setId(task.getId());
    result.setName(task.getName());
    result.setSequenceID(i);

    if (stepOutput == null) {
        logger.warn("Cannot found output for task: id={}", task.getId());
        return result;
    }

    result.setStatus(parseToJobStepStatus(stepOutput.getState()));
    for (Map.Entry<String, String> entry : stepOutput.getExtra().entrySet()) {
        if (entry.getKey() != null && entry.getValue() != null) {
            result.putInfo(entry.getKey(), entry.getValue());
        }
    }
    result.setExecStartTime(AbstractExecutable.getStartTime(stepOutput));
    result.setExecEndTime(AbstractExecutable.getEndTime(stepOutput));
    if (task instanceof ShellExecutable) {
        result.setExecCmd(((ShellExecutable) task).getCmd());
    }
    if (task instanceof MapReduceExecutable) {
        result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams());
        result.setExecWaitTime(
                AbstractExecutable.getExtraInfoAsLong(stepOutput, MapReduceExecutable.MAP_REDUCE_WAIT_TIME, 0L)
                        / 1000);
    }
    if (task instanceof HadoopShellExecutable) {
        result.setExecCmd(((HadoopShellExecutable) task).getJobParams());
    }
    return result;
}
 

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
private String findExtraInfo(String key, String dft, boolean backward) {
    ArrayList<AbstractExecutable> tasks = new ArrayList<AbstractExecutable>(getTasks());

    if (backward) {
        Collections.reverse(tasks);
    }

    for (AbstractExecutable child : tasks) {
        Output output = getManager().getOutput(child.getId());
        String value = output.getExtra().get(key);
        if (value != null)
            return value;
    }
    return dft;
}
 

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
@Test
public void testParseToJobInstance() {
    TestJob task = new TestJob();
    JobInstance instance = JobInfoConverter.parseToJobInstanceQuietly(task, Maps.<String, Output> newHashMap());
    // no exception thrown is expected
    Assert.assertTrue(instance == null);
}
 

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
@Override
public void onUnlock(String path, String nodeData) {
    String[] paths = path.split("/");
    String segmentId = paths[paths.length - 1];

    for (final String id : executableManager.getAllJobIds()) {
        final Output output = executableManager.getOutput(id);
        if (output.getState() == ExecutableState.RUNNING) {
            AbstractExecutable executable = executableManager.getJob(id);
            if (executable instanceof DefaultChainedExecutable
                    && executable.getParams().get(SEGMENT_ID).equalsIgnoreCase(segmentId)
                    && !nodeData.equalsIgnoreCase(serverName)) {
                try {
                    logger.warn(nodeData + " has released the lock for: " + segmentId
                            + " but the job still running. so " + serverName + " resume the job");
                    if (!jobLock.isLocked(getLockPath(segmentId))) {
                        executableManager.resumeRunningJobForce(executable.getId());
                        fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS);
                        break;
                    }
                } catch (Exception e) {
                    logger.error("resume the job but fail in server: " + serverName, e);
                }
            }
        }
    }
}
 

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
@Test
public void testExceptionOnLostJobOutput() {
    ExecutableManager manager = ExecutableManager.getInstance(jobService.getConfig());
    AbstractExecutable executable = new TestJob();
    manager.addJob(executable);
    List<CubingJob> jobs = jobService.innerSearchCubingJobs("cube", "jobName",
            Collections.<ExecutableState> emptySet(), 0, Long.MAX_VALUE, Collections.<String, Output> emptyMap(),
            true, "project");
    Assert.assertEquals(0, jobs.size());
}
 

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
public List<JobInstance> innerSearchCubingJobs(final String cubeName, final String jobName,
        final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
    if (null == projectName) {
        aclEvaluate.checkIsGlobalAdmin();
    } else {
        aclEvaluate.checkProjectOperationPermission(projectName);
    }
    // prepare time range
    Calendar calendar = Calendar.getInstance();
    calendar.setTime(new Date());
    long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
    long timeEndInMillis = Long.MAX_VALUE;
    Set<ExecutableState> states = convertStatusEnumToStates(statusList);
    final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs(timeStartInMillis, timeEndInMillis);

    return Lists
            .newArrayList(
                    FluentIterable
                            .from(innerSearchCubingJobs(cubeName, jobName, states, timeStartInMillis,
                                    timeEndInMillis, allOutputs, false, projectName))
                            .transform(new Function<CubingJob, JobInstance>() {
                                @Override
                                public JobInstance apply(CubingJob cubingJob) {
                                    return JobInfoConverter.parseToJobInstanceQuietly(cubingJob, allOutputs);
                                }
                            }).filter(new Predicate<JobInstance>() {
                                @Override
                                public boolean apply(@Nullable JobInstance input) {
                                    return input != null;
                                }
                            }));
}
 

import org.apache.kylin.job.execution.Output; //导入依赖的package包/类
public Output getOutput(String uuid) {
    try {
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(uuid);
        Preconditions.checkArgument(jobOutput != null, "there is no related output for job id:" + uuid);
        final DefaultOutput result = new DefaultOutput();
        result.setExtra(jobOutput.getInfo());
        result.setState(ExecutableState.valueOf(jobOutput.getStatus()));
        result.setVerboseMsg(jobOutput.getContent());
        result.setLastModified(jobOutput.getLastModified());
        return result;
    } catch (PersistentException e) {
        logger.error("fail to get job output:" + uuid, e);
        throw new RuntimeException(e);
    }
}