Java Executable类代码示例

import org.apache.kylin.job.execution.Executable; //导入依赖的package包/类
private static void assertJobEqual(Executable one, Executable another) {
    assertEquals(one.getClass(), another.getClass());
    assertEquals(one.getId(), another.getId());
    assertEquals(one.getStatus(), another.getStatus());
    assertEquals(one.isRunnable(), another.isRunnable());
    assertEquals(one.getOutput(), another.getOutput());
    assertTrue((one.getParams() == null && another.getParams() == null) || (one.getParams() != null && another.getParams() != null));
    if (one.getParams() != null) {
        assertEquals(one.getParams().size(), another.getParams().size());
        for (String key : one.getParams().keySet()) {
            assertEquals(one.getParams().get(key), another.getParams().get(key));
        }
    }
    if (one instanceof ChainedExecutable) {
        assertTrue(another instanceof ChainedExecutable);
        List<? extends Executable> onesSubs = ((ChainedExecutable) one).getTasks();
        List<? extends Executable> anotherSubs = ((ChainedExecutable) another).getTasks();
        assertTrue((onesSubs == null && anotherSubs == null) || (onesSubs != null && anotherSubs != null));
        if (onesSubs != null) {
            assertEquals(onesSubs.size(), anotherSubs.size());
            for (int i = 0; i < onesSubs.size(); ++i) {
                assertJobEqual(onesSubs.get(i), anotherSubs.get(i));
            }
        }
    }
}
 

import org.apache.kylin.job.execution.Executable; //导入依赖的package包/类
private boolean isJobPoolFull() {
    Map<String, Executable> runningJobs = context.getRunningJobs();
    if (runningJobs.size() >= jobEngineConfig.getMaxConcurrentJobLimit()) {
        logger.warn("There are too many jobs running, Job Fetch will wait until next schedule time");
        return true;
    }

    return false;
}
 

import org.apache.kylin.job.execution.Executable; //导入依赖的package包/类
@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock lock) throws SchedulerException {
    jobLock = lock;

    String serverMode = jobEngineConfig.getConfig().getServerMode();
    if (!("job".equals(serverMode.toLowerCase()) || "all".equals(serverMode.toLowerCase()))) {
        logger.info("server mode: " + serverMode + ", no need to run job scheduler");
        return;
    }
    logger.info("Initializing Job Engine ....");

    if (!initialized) {
        initialized = true;
    } else {
        return;
    }

    this.jobEngineConfig = jobEngineConfig;

    if (jobLock.lockJobEngine() == false) {
        throw new IllegalStateException("Cannot start job scheduler due to lack of job lock");
    }

    executableManager = ExecutableManager.getInstance(jobEngineConfig.getConfig());
    //load all executable, set them to a consistent status
    fetcherPool = Executors.newScheduledThreadPool(1);
    int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
    jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
            new SynchronousQueue<Runnable>());
    context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());

    executableManager.resumeAllRunningJobs();

    int pollSecond = jobEngineConfig.getPollIntervalSecond();
    logger.info("Fetching jobs every {} seconds", pollSecond);
    fetcher = jobEngineConfig.getJobPriorityConsidered() ? new FetcherRunnerWithPriority() : new FetcherRunner();
    fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
    hasStarted = true;
}
 

import org.apache.kylin.job.execution.Executable; //导入依赖的package包/类
@Override
public void run() {
    // logger.debug("Job Fetcher is running...");
    Map<String, Executable> runningJobs = context.getRunningJobs();
    if (runningJobs.size() >= jobEngineConfig.getMaxConcurrentJobLimit()) {
        logger.warn("There are too many jobs running, Job Fetch will wait until next schedule time");
        return;
    }

    int nRunning = 0, nReady = 0, nOthers = 0;
    for (final String id : executableManager.getAllJobIds()) {
        if (runningJobs.containsKey(id)) {
            // logger.debug("Job id:" + id + " is already running");
            nRunning++;
            continue;
        }
        final Output output = executableManager.getOutput(id);
        if ((output.getState() != ExecutableState.READY)) {
            // logger.debug("Job id:" + id + " not runnable");
            nOthers++;
            continue;
        }
        nReady++;
        AbstractExecutable executable = executableManager.getJob(id);
        String jobDesc = executable.toString();
        logger.info(jobDesc + " prepare to schedule");
        try {
            context.addRunningJob(executable);
            jobPool.execute(new JobRunner(executable));
            logger.info(jobDesc + " scheduled");
        } catch (Exception ex) {
            context.removeRunningJob(executable);
            logger.warn(jobDesc + " fail to schedule", ex);
        }
    }
    logger.info("Job Fetcher: " + nRunning + " running, " + runningJobs.size() + " actual running, " + nReady + " ready, " + nOthers + " others");
}
 

import org.apache.kylin.job.execution.Executable; //导入依赖的package包/类
@Override
synchronized public void run() {
    try {
        Map<String, Executable> runningJobs = context.getRunningJobs();
        if (runningJobs.size() >= jobEngineConfig.getMaxConcurrentJobLimit()) {
            logger.warn("There are too many jobs running, Job Fetch will wait until next schedule time");
            return;
        }

        int nRunning = 0, nOtherRunning = 0, nReady = 0, nOthers = 0;
        for (final String id : executableManager.getAllJobIds()) {
            if (runningJobs.containsKey(id)) {
                nRunning++;
                continue;
            }

            final Output output = executableManager.getOutput(id);

            if ((output.getState() != ExecutableState.READY)) {
                if (output.getState() == ExecutableState.RUNNING) {
                    nOtherRunning++;
                } else {
                    nOthers++;
                }
                continue;
            }

            nReady++;
            final AbstractExecutable executable = executableManager.getJob(id);
            try {
                jobPool.execute(new JobRunner(executable));
            } catch (Exception ex) {
                logger.warn(executable.toString() + " fail to schedule in server: " + serverName, ex);
            }
        }
        logger.info("Job Fetcher: " + nRunning + " should running, " + runningJobs.size() + " actual running, "
                + nOtherRunning + " running in other server, " + nReady + " ready, " + nOthers + " others");
    } catch (Exception e) {
        logger.warn("Job Fetcher caught a exception " + e);
    }
}
 

import org.apache.kylin.job.execution.Executable; //导入依赖的package包/类
@Override
public synchronized void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException {
    String serverMode = jobEngineConfig.getConfig().getServerMode();
    if (!("job".equals(serverMode.toLowerCase()) || "all".equals(serverMode.toLowerCase()))) {
        logger.info("server mode: " + serverMode + ", no need to run job scheduler");
        return;
    }
    logger.info("Initializing Job Engine ....");

    if (!initialized) {
        initialized = true;
    } else {
        return;
    }

    this.jobEngineConfig = jobEngineConfig;
    this.jobLock = (DistributedLock) jobLock;
    this.serverName = this.jobLock.getClient(); // the lock's client string contains node name of this server

    executableManager = ExecutableManager.getInstance(jobEngineConfig.getConfig());
    //load all executable, set them to a consistent status
    fetcherPool = Executors.newScheduledThreadPool(1);

    //watch the zookeeper node change, so that when one job server is down, other job servers can take over.
    watchPool = Executors.newFixedThreadPool(1);
    WatcherProcessImpl watcherProcess = new WatcherProcessImpl(this.serverName);
    lockWatch = this.jobLock.watchLocks(getWatchPath(), watchPool, watcherProcess);

    int corePoolSize = jobEngineConfig.getMaxConcurrentJobLimit();
    jobPool = new ThreadPoolExecutor(corePoolSize, corePoolSize, Long.MAX_VALUE, TimeUnit.DAYS,
            new SynchronousQueue<Runnable>());
    context = new DefaultContext(Maps.<String, Executable> newConcurrentMap(), jobEngineConfig.getConfig());

    int pollSecond = jobEngineConfig.getPollIntervalSecond();
    logger.info("Fetching jobs every {} seconds", pollSecond);
    fetcher = new FetcherRunner();
    fetcherPool.scheduleAtFixedRate(fetcher, pollSecond / 10, pollSecond, TimeUnit.SECONDS);
    hasStarted = true;

    resumeAllRunningJobs();
}
 

import org.apache.kylin.job.execution.Executable; //导入依赖的package包/类
public DefaultContext(ConcurrentMap<String, Executable> runningJobs, KylinConfig kylinConfig) {
    this.runningJobs = runningJobs;
    this.kylinConfig = kylinConfig;
}
 

import org.apache.kylin.job.execution.Executable; //导入依赖的package包/类
void addRunningJob(Executable executable) {
    runningJobs.put(executable.getId(), executable);
}
 

import org.apache.kylin.job.execution.Executable; //导入依赖的package包/类
void removeRunningJob(Executable executable) {
    runningJobs.remove(executable.getId());
}
 

import org.apache.kylin.job.execution.Executable; //导入依赖的package包/类
public Map<String, Executable> getRunningJobs() {
    return Collections.unmodifiableMap(runningJobs);
}
 

import org.apache.kylin.job.execution.Executable; //导入依赖的package包/类
@Override
synchronized public void run() {
    try (SetThreadName ignored = new SetThreadName("Scheduler %s FetcherRunner",
            System.identityHashCode(DefaultScheduler.this))) {
        // logger.debug("Job Fetcher is running...");
        Map<String, Executable> runningJobs = context.getRunningJobs();
        if (isJobPoolFull()) {
            return;
        }

        int nRunning = 0, nReady = 0, nStopped = 0, nOthers = 0, nError = 0, nDiscarded = 0, nSUCCEED = 0;
        for (final String id : executableManager.getAllJobIds()) {
            if (isJobPoolFull()) {
                return;
            }
            if (runningJobs.containsKey(id)) {
                // logger.debug("Job id:" + id + " is already running");
                nRunning++;
                continue;
            }
            final AbstractExecutable executable = executableManager.getJob(id);
            if (!executable.isReady()) {
                final Output output = executableManager.getOutput(id);
                // logger.debug("Job id:" + id + " not runnable");
                if (output.getState() == ExecutableState.DISCARDED) {
                    nDiscarded++;
                } else if (output.getState() == ExecutableState.ERROR) {
                    nError++;
                } else if (output.getState() == ExecutableState.SUCCEED) {
                    nSUCCEED++;
                } else if (output.getState() == ExecutableState.STOPPED) {
                    nStopped++;
                } else {
                    if (fetchFailed) {
                        executableManager.forceKillJob(id);
                        nError++;
                    } else {
                        nOthers++;
                    }
                }
                continue;
            }
            nReady++;
            String jobDesc = null;
            try {
                jobDesc = executable.toString();
                logger.info(jobDesc + " prepare to schedule");
                context.addRunningJob(executable);
                jobPool.execute(new JobRunner(executable));
                logger.info(jobDesc + " scheduled");
            } catch (Exception ex) {
                if (executable != null)
                    context.removeRunningJob(executable);
                logger.warn(jobDesc + " fail to schedule", ex);
            }
        }

        fetchFailed = false;
        logger.info("Job Fetcher: " + nRunning + " should running, " + runningJobs.size() + " actual running, "
                + nStopped + " stopped, " + nReady + " ready, " + nSUCCEED + " already succeed, " + nError
                + " error, " + nDiscarded + " discarded, " + nOthers + " others");
    } catch (Exception e) {
        fetchFailed = true; // this could happen when resource store is unavailable
        logger.warn("Job Fetcher caught a exception ", e);
    }
}