C++ JobQueue类代码示例

Continuation callbackOnJobQueue (
    JobQueue& jobQueue, std::string const& name, JobType jobType)
{
    return Continuation ([name, jobType, &jobQueue] (Callback const& cb) {
        jobQueue.addJob (jobType, name, [cb] (Job&) { cb(); });
    });
}

void boss( ) {
   for (;;) {
      // Get the request from somewhere
      Request req;
      myJobQueue.submitJob(req);
   }
}

bool JobManager::processJobs()
{
    m_jobCount = 0;
    m_jobsNotIdle = 0;
    std::list<JobQueue*>::const_iterator it = m_queues.begin();
    for (; it != m_queues.end(); ++it)
        m_jobCount += (*it)->jobCount();

    m_jobsProcessed = 0;
    while (!m_errorOccured && m_jobsProcessed < m_jobCount) {
        // Select a valid queue
        std::list<JobQueue*>::iterator queueIt = m_queues.begin();
        JobQueue* queue = *queueIt;
        while (queue->hasShowStoppers() || queue->isEmpty()) {
            queue = *(++queueIt);
            if (queueIt == m_queues.end())
                goto finish;
        }

        MutexLocker locker(&m_jobsRunningMutex);
        if (m_jobsRunning >= m_maxJobsRunning)
            pthread_cond_wait(&m_needJobsCond, &m_jobsRunningMutex);

        if (m_errorOccured)
            break;

        // Now select a valid job
        if (Job* job = queue->getNextJob()) {
            job->addJobListenner(this);
            job->run();
            m_jobsRunning++;
            m_jobsNotIdle++;
            printReportLine(job);
        }
    }

    finish:

    MutexLocker locker(&m_jobsRunningMutex);
    if (m_jobsRunning) {
        Notice() << "Waiting for unfinished jobs...";
        pthread_cond_wait(&m_allDoneCond, &m_jobsRunningMutex);
    }

    return !m_errorOccured;
}

static inline void
enqueueStandardSplittingJob(JobQueue& jobQueue, const LcpCacheStringPtr* inputs, unsigned numInputs, string* output, size_t jobLength)
{
    if (numInputs == 1)
        jobQueue.enqueue(new CopyDataJob(inputs[0], output));

    else if (numInputs <= 2)
        jobQueue.enqueue(new BinaryMergeJob(inputs[0], inputs[1], 0, output));

    else if (numInputs <= 4)
        jobQueue.enqueue(new MergeJobStandardSplitting<4>(inputs, numInputs, output, jobLength));

    else if (numInputs <= 8)
        jobQueue.enqueue(new MergeJobStandardSplitting<8>(inputs, numInputs, output, jobLength));

    else if (numInputs <= 16)
        jobQueue.enqueue(new MergeJobStandardSplitting<16>(inputs, numInputs, output, jobLength));

    else if (numInputs <= 32)
        jobQueue.enqueue(new MergeJobStandardSplitting<32>(inputs, numInputs, output, jobLength));

    else if (numInputs <= 64)
        jobQueue.enqueue(new MergeJobStandardSplitting<64>(inputs, numInputs, output, jobLength));

    else
    {
        DBG(1, "Can't create job with that many streams. Add more cases.");
        abort();
    }
}

 virtual size_t run( void )
 {
   for(;;)
   {
     CountPtr< JobQueue::Job > job = q->getJob();
     if( size_t(-1) == job->work( q ) ) return 0;
     Thread::yield();
   }
 }

/*!	Basic system initialization that must happen before any jobs are launched.
*/
void
LaunchDaemon::_InitSystem()
{
	_AddInitJob(new InitRealTimeClockJob());
	_AddInitJob(new InitSharedMemoryDirectoryJob());
	_AddInitJob(new InitTemporaryDirectoryJob());

	fJobQueue.AddJob(fInitTarget);
}

/*!	Adds the specified \a job to the launch queue
	queue, except those that are triggered by events.

	Unless \c FORCE_NOW is set, the target is only launched if its events,
	if any, have been triggered already.

	Calling this method will trigger a demand event if \c TRIGGER_DEMAND has
	been set.
*/
bool
LaunchDaemon::_LaunchJob(Job* job, uint32 options)
{
	if (job != NULL && (job->IsLaunching() || job->IsRunning()))
		return true;

	if (!_CanLaunchJob(job, options))
		return false;

	// Test if we can launch all requirements
	if (!_CanLaunchJobRequirements(job, options | TRIGGER_DEMAND))
		return false;

	// Actually launch the requirements
	int32 count = job->Requirements().CountStrings();
	for (int32 index = 0; index < count; index++) {
		Job* requirement = FindJob(job->Requirements().StringAt(index));
		if (requirement != NULL) {
			// TODO: For jobs that have their communication channels set up,
			// we would not need to trigger demand at this point
			if (!_LaunchJob(requirement, options | TRIGGER_DEMAND)) {
				// Failed to put a requirement into the launch queue
				return false;
			}
		}
	}

	if (job->Target() != NULL)
		job->Target()->ResolveSourceFiles();
	if (job->Event() != NULL)
		job->Event()->ResetTrigger();

	job->SetLaunching(true);

	status_t status = fJobQueue.AddJob(job);
	if (status != B_OK) {
		debug_printf("Adding job %s to queue failed: %s\n", job->Name(),
			strerror(status));
		return false;
	}

	// Try to launch pending jobs as well
	count = job->Pending().CountStrings();
	for (int32 index = 0; index < count; index++) {
		Job* pending = FindJob(job->Pending().StringAt(index));
		if (pending != NULL && _LaunchJob(pending, 0)) {
			// Remove the job from the pending list once its in the launch
			// queue, so that is not being launched again next time.
			index--;
			count--;
		}
	}

	return true;
}

int main (int argc, char * argv[]) {
    QCoreApplication app(argc, argv);

    JobQueue jq;

    auto a = jq.setTimeout(1000, [&](const qtael::Await & await)->void {
        qDebug() << "A: run callback";
        await(1000);
        qDebug() << "A: first";
        await(1000);
        qDebug() << "A: second";
        await(1000);
        qDebug() << "A: third";
    });

    auto b = jq.setTimeout(1500, [&](const qtael::Await & await)->void {
        qDebug() << "B: run callback";
        await(1000);
        qDebug() << "B: first";
        await(1000);
        qDebug() << "B: second";
        await(1000);
        qDebug() << "B: third";
    });

    auto c = jq.setTimeout(2000, [&](const qtael::Await & /*await*/)->void {
        qDebug() << "C: run callback";
        app.quit();
    });

    auto d = new qtael::Async([&](const qtael::Await & await)->void {
        await(2500);
        qDebug() << "D: run callback";
        jq.clear(b);
    });
    d->connect(d, SIGNAL(finished()), SLOT(deleteLater()));
    d->start();

    return app.exec();
}

static inline void
parallelLcpMergeStandardSplitting(const LcpCacheStringPtr* input, unsigned numInputs, string* output, size_t length)
{
    g_outputBase = output;
    g_splittingsExecuted = 0;
    g_mergeJobsCreated = 0;
    g_splittingTime = 0;

    ClockTimer timer;
    timer.start();

    g_outputBase = output;

    JobQueue jobQueue;
    DBG(debug_merge_start_message, "doing parallel lcp merge for " << numInputs << " input streams using " << omp_get_max_threads() << " threads with standard splitting");
    jobQueue.enqueue(new InitialJobStandardSplitting(input, numInputs, output, length));
    jobQueue.numaLoop(-1, omp_get_max_threads());

    g_stats >> "toplevelmerge_time" << timer.elapsed();
    g_stats >> "splittings_executed" << g_splittingsExecuted;
    g_stats >> "mergejobs_created" << g_mergeJobsCreated;
    g_stats >> "splitting_time" << g_splittingTime;
}

/*!	Adds the specified \a job to the launch queue
	queue, except those that are triggered by events.

	Unless \a forceNow is true, the target is only launched if its events,
	if any, have been triggered already.

	Calling this method will trigger a demand event.
*/
void
LaunchDaemon::_LaunchJob(Job* job, bool forceNow)
{
	if (job == NULL || job->IsLaunched() || (!forceNow
		&& (!job->EventHasTriggered() || !job->CheckCondition(*this)
			|| Events::TriggerDemand(job->Event())))) {
		return;
	}

	int32 count = job->Requirements().CountStrings();
	for (int32 index = 0; index < count; index++) {
		Job* requirement = FindJob(job->Requirements().StringAt(index));
		if (requirement != NULL)
			_LaunchJob(requirement);
	}

	if (job->Target() != NULL)
		job->Target()->ResolveSourceFiles();
	if (job->Event() != NULL)
		job->Event()->ResetTrigger();

	fJobQueue.AddJob(job);
}

    /*
     * returns true if all elements have been written to output
     * false if the merge has been stopped to free work.
     */
    inline bool
    mergeToOutput(JobQueue& jobQueue)
    {
        for (size_t lastLength = length; length >= MERGE_BULK_SIZE; length -= MERGE_BULK_SIZE, output += MERGE_BULK_SIZE)
        {
            if (g_lengthOfLongestJob == lastLength)
                g_lengthOfLongestJob = length;

            if (g_lengthOfLongestJob < length)
                g_lengthOfLongestJob = length; // else if to prevent work sharing when we just increased g_lengthOfLongestJob
            else if (USE_WORK_SHARING &&
                     jobQueue.has_idle() &&
                     length > SHARE_WORK_THRESHOLD &&
                     g_lengthOfLongestJob == length)
                return false;

            loserTree.writeElementsToStream(output, MERGE_BULK_SIZE);
            lastLength = length;
        }

        loserTree.writeElementsToStream(output, length);

        return true;
    }

    void JobScheduler::run(Ogre::Real time)
    {
        ///@todo use different buckets for jobs not yet started, instead of
        ///      iterating over those each time.

        ///@todo dynamically determine token threshold. Maybe make it work load depending.


        // Queue for finished jobs
        JobQueue notDone;

        for (JobQueue::iterator it = mJobQueue.begin(), end = mJobQueue.end(); it != end; ++it)
        {
            JobEntry entry = *it;

            TimeSource::TimeSourceType tst = entry.job->getTimeSource();
            TimeSource* ts = TimeSourceManager::getSingleton().getTimeSource(tst);
            Time clock = ts->getClock();
            if (tst != entry.timeSourceLastCall) // time source has changed, e.g. in a job queue
            {
                entry.timeLastCall = clock;
                entry.timeSourceLastCall = tst;
            }

            if (entry.markedToRemove)
            {
                // Notify listener, the job was removed
                if (entry.listener != NULL)
                {
                    entry.listener->jobRemoved(entry.ticket);
                }

                if (entry.job->destroyWhenDone() )
                {
                    delete entry.job;
                }
            }
            else if (entry.start <= clock && clock < entry.end)
            {
                // Is the token threshold reached?
                if (entry.tokens >= mTokenThreshold)
                {
                    // Yes, pay run fee and execute.
                    entry.tokens = 0;
                    bool runAgain = !entry.job->execute(clock - entry.timeLastCall);

                    if (!entry.called)
                    {
                        // Notify listener, the job started for the first time
                        if (entry.listener != NULL)
                        {
                            entry.listener->jobStarted(entry.ticket);
                        }
                        entry.called = true;
                    }

                    if (runAgain)
                    {
                        // Job is not done, reset token count and requeue.
                        entry.tokens = entry.priority;
                        entry.timeLastCall = clock;
                        notDone.push_back(entry);
                    }
                    else
                    {
                        // Notify listener, the job finished regularly.
                        if (entry.listener != NULL)
                        {
                            entry.listener->jobFinished(entry.ticket);
                        }

                        // If we are supposed to delete the Job, do so now.
                        if (entry.job->destroyWhenDone())
                        {
                            delete entry.job;
                        }
                    }
                }
                else
                {
                    // No, increase token count
                    entry.tokens += entry.priority;
                    notDone.push_back(entry);
                }
            }
            else if (clock < entry.end)
            {
                // Start time not yet reached. Queue again.
                notDone.push_back(entry);
            }
            else
            {
                // Job reached its end time and didn't want to finish itself, so we do it.
                if (entry.job->isDiscardable())
                {
                    entry.job->discard();
                    if (entry.listener != NULL)
                    {
                        entry.listener->jobDiscarded(entry.ticket);
                    }
//.........这里部分代码省略.........

    virtual bool run(JobQueue& job_queue)
    {
        size_t n = strptr.size();

        LOGC(debug_jobs)
            << "Process SmallsortJob8 " << this << " of size " << n;

        strptr = strptr.copy_back();

        if (n < g_inssort_threshold) {
            inssort::inssort_generic(strptr.copy_back().active(), depth);
            return true;
        }

        Char* charcache = new Char[n];

        // std::deque is much slower than std::vector, so we use an artificial
        // pop_front variable.
        size_t pop_front = 0;
        std::vector<RadixStep8_CI> radixstack;
        radixstack.emplace_back(strptr, depth, charcache);

        while (radixstack.size() > pop_front)
        {
            while (radixstack.back().idx < 255)
            {
                RadixStep8_CI& rs = radixstack.back();
                size_t b = ++rs.idx; // process the bucket rs.idx

                size_t bktsize = rs.bkt[b + 1] - rs.bkt[b];

                if (bktsize == 0)
                    continue;
                else if (bktsize < g_inssort_threshold)
                {
                    inssort::inssort_generic(
                        rs.strptr.sub(rs.bkt[b], bktsize).copy_back().active(),
                        depth + radixstack.size());
                }
                else
                {
                    radixstack.emplace_back(rs.strptr.sub(rs.bkt[b], bktsize),
                                            depth + radixstack.size(),
                                            charcache);
                }

                if (use_work_sharing && job_queue.has_idle())
                {
                    // convert top level of stack into independent jobs
                    LOGC(debug_jobs)
                        << "Freeing top level of SmallsortJob8's radixsort stack";

                    RadixStep8_CI& rt = radixstack[pop_front];

                    while (rt.idx < 255)
                    {
                        b = ++rt.idx; // enqueue the bucket rt.idx

                        size_t bktsize = rt.bkt[b + 1] - rt.bkt[b];

                        if (bktsize == 0) continue;
                        EnqueueSmallsortJob8(
                            job_queue, rt.strptr.sub(rt.bkt[b], bktsize),
                            depth + pop_front);
                    }

                    // shorten the current stack
                    ++pop_front;
                }
            }
            radixstack.pop_back();
        }

        delete[] charcache;

        return true;
    }

/**
 * Waits until there is a job in the queue, pops it off and executes
 * it, then waits for another.  Runs until jobs are completed and
 * main thread sets acceptingJobs=0.
 */
void *threadfun(void *arg) {
    DPRINTF(("%s %lu entry\n", __func__, (unsigned long) pthread_self()));

    //    struct timespec timeout;
    JobQueue *jq = (JobQueue *) arg;
    Job *job;
    int status;
    void *threadState = NULL;
    if(jq->ThreadState_new != NULL) {
        threadState = jq->ThreadState_new(jq->threadData);
        CHECKMEM(threadState);
    }

    for(;;) {
        //        clock_gettime(CLOCK_REALTIME, &timeout);
        //        timeout.tv_sec += 3;

        status = pthread_mutex_lock(&jq->lock); // LOCK
        if(status)
            ERR(status, "lock");
        else
            DPRINTF(("%s:%s:%d: locked\n", __FILE__, __func__, __LINE__));

        // Wait while the queue is empty and accepting jobs
        while(NULL == jq->todo && jq->acceptingJobs) {
            DPRINTF(("%s:%d:  awaiting work. todo=%p\n",
                     __func__, __LINE__, jq->todo));

            ++jq->idle;
            if(jq->idle == jq->nThreads) {
                status = pthread_cond_signal(&jq->wakeMain);
                if(status)
                    ERR(status, "signal wakeMain");
            }
            //status = pthread_cond_timedwait(&jq->wakeWorker, &jq->lock,
            //                                &timeout);
            status = pthread_cond_wait(&jq->wakeWorker, &jq->lock);
            --jq->idle;
            //if(status == ETIMEDOUT)
            //    continue;
            if(status)
                ERR(status, "wait wakeWorker");
        }

        /*
         * todo accepting
         *   0     0  <- exit
         *   0     1  <- stay in while loop
         *   1     0  <- do work
         *   1     1  <- do work
         */

        if(NULL == jq->todo) {  // shutting down
            assert(!jq->acceptingJobs);
            break;
        } else {                // do job
            DPRINTF(("%s %lu got work\n", __func__,
                     (unsigned long) pthread_self()));

            // remove job from queue
            assert(NULL != jq->todo);
            job = jq->todo;
            jq->todo = jq->todo->next;

#ifdef DPRINTF_ON
            printf("%s:%d:queue:", __func__, __LINE__);
            Job_print(jq->todo);
#endif

            status = pthread_mutex_unlock(&jq->lock);   // UNLOCK
            if(status)
                ERR(status, "unlock");
            else
                DPRINTF(("%s:%s:%d: unlocked\n", __FILE__, __func__,
                         __LINE__));

            DPRINTF(("%s %lu calling jobfun\n", __func__,
                     (unsigned long) pthread_self()));
            job->jobfun(job->param, threadState);
            DPRINTF(("%s %lu back fr jobfun\n", __func__,
                     (unsigned long) pthread_self()));
            free(job);
        }
    }
    // still have lock
    --jq->nThreads;

    status = pthread_cond_signal(&jq->wakeMain);
    if(status)
        ERR(status, "signal wakeMain");

    status = pthread_mutex_unlock(&jq->lock);   // UNLOCK
    if(status)
        ERR(status, "unlock");
    else
//.........这里部分代码省略.........

void worker( ) {
   for (;;) {
      Request r(myJobQueue.getJob( ));
      // Do something with the job...
   }
}