Python Queue.dequeue方法代碼示例

# 需要導入模塊: from retask.queue import Queue [as 別名]
# 或者: from retask.queue.Queue import dequeue [as 別名]
class Worker(object):
    """ Represents the worker process.  Waits for tasks to come in from the
    webapp and then acts on them.
    """

    def __init__(self):
        self.queue = Queue('commits')
        self.queue.connect()
        # TODO -- set both of these with the config file.
        # Use pyramid tools to load config.
        self.sleep_interval = 1
        self.scratch_dir = "/home/threebean/scratch/pep8bot-scratch"
        try:
            os.makedirs(self.scratch_dir)
        except OSError:
            pass  # Assume that the scratch_dir already exists.

    def run(self):
        while True:
            time.sleep(self.sleep_interval)
            print "Waking"
            if self.queue.length == 0:
                continue

            task = self.queue.dequeue()
            data = task.data
            url = data['repository']['url']

            # TODO -- don't clone this url.  But fork and clone our url.

            name = data['repository']['name']
            owner = data['repository']['owner']['name']
            self.working_dir = tempfile.mkdtemp(
                prefix=owner + '-' + name,
                dir=self.scratch_dir,
            )
            print "** Cloning to", self.working_dir
            print sh.git.clone(url, self.working_dir)
            print "** Processing files."
            for root, dirs, files in os.walk(self.working_dir):

                if '.git' in root:
                    continue

                for filename in files:
                    if filename.endswith(".py"):
                        infile = root + "/" + filename
                        print "** Tidying", infile
                        tmpfile = infile + ".bak"
                        script = os.path.expanduser(
                            "~/devel/PythonTidy/PythonTidy.py"
                        )
                        sh.python(script, infile, tmpfile)
                        shutil.move(tmpfile, infile)

            with directory(self.working_dir):
                print sh.pwd()
                print sh.git.status()

# 需要導入模塊: from retask.queue import Queue [as 別名]
# 或者: from retask.queue.Queue import dequeue [as 別名]
def monitor_buildqueue():
    """
    This function monitors the build queue.

    If the build is still on then it puts it back to the queue.
    If the build is finished then it goes to the job queue.
    """
    key = get_key('darkbuildqueue')
    config = get_redis_config()
    jobqueue = Queue('jobqueue', config)
    jobqueue.connect()
    buildqueue = Queue('buildqueue', config)
    buildqueue.connect()
    rdb = redis_connection()
    if not rdb:
        log(key, 'redis is missing', 'error')
        return None
    rdb.set('darkbuildqueue-status', '1')
    while True:
        if check_shutdown():
            break
        try:
            time.sleep(60)
            length = buildqueue.length
            if length == 0:
                log(key, "Sleeping, no buildqueue job", 'info')
                time.sleep(60)
                continue
            task = buildqueue.dequeue()
            kojiurl = task.data['kojiurl']
            idx = task.data['jobid']
            kc = koji.ClientSession(kojiurl, {'debug': False, 'password': None,\
                            'debug_xmlrpc': False, 'user': None})

            res = kc.getBuild(idx)
            if not res:
                #We reached to the new build yet to start
                #Time to sleep
                log(key, "build deleted %s" % idx, 'error')
                continue
            if res['state'] == 1:
                #completed build now push to our redis queue
                jobqueue.enqueue(task)
                log(key, "in job queue %s" % idx, 'info')
                continue

            if res['state'] == 0:
                #building state
                buildqueue.enqueue(task)
                log(key, "in build queue %s" % idx, 'info')
                continue

        except Exception, error:
            log(key, str(error), 'error')

# 需要導入模塊: from retask.queue import Queue [as 別名]
# 或者: from retask.queue.Queue import dequeue [as 別名]
class Channel(object):
    """
    Abstraction above retask (the set of "channels" between backend(s),
    jobgrabber and workers).  We could use multiple backends and/or diffferent
    "atomic" medium (other implemntation than Queue) in future.  But
    make sure nobody needs to touch the "medium" directly.
    """

    def __init__(self, opts, log=None):
        self.log = log
        self.opts = opts
        # channel for Backend <--> JobGrabber communication
        self.jg_start = Queue("jg_control_start")
        # channel for JobGrabber <--> [[Builders]] communication
        self.build_queues = dict()
        while not self.jg_start.connect():
            wait_log(self.log, "waiting for redis", 5)

    def _get_queue(self, bgroup):
        if not bgroup in self.build_queues:
            q_id = "copr-be-{0}".format(bgroup)
            q = Queue(q_id)
            if not q.connect():
                # As we already connected to jg_control_message, this should
                # be also OK.
                raise Exception("can't connect to redis, should never happen!")
            return q

        return self.build_queues[bgroup]

    def add_build(self, bgroup, build):
        """ this should be used by job_grab only for now """
        q = self._get_queue(bgroup)
        try:
            q.enqueue(Task(build))
        except Exception as err:
            # I've seen isses Task() was not able to jsonify urllib exceptions
            if not self.log:
                return False
            self.log.error("can't enqueue build {0}, reason:\n{1}".format(
                build, err
            ))

        return True

    # Builder's API
    def get_build(self, bgroup):
        """
        Return task from queue or return 0
        """
        q = self._get_queue(bgroup)
        t = q.dequeue()
        return t.data if t else None

    # JobGrab's API
    def backend_started(self):
        return self.jg_start.length

    def job_graber_initialized(self):
        while self.jg_start.dequeue():
            pass

    def remove_all_builds(self):
        for bgroup in self.build_queues:
            q = self._get_queue(bgroup)
            while q.dequeue():
                pass
        self.build_queues = dict()

    # Backend's API
    def backend_start(self):
        """ Notify jobgrab about service start. """
        if not self.jg_start.enqueue(Task("start")):
             raise Exception("can't append to retask queue, should never happen!")

        while self.jg_start.length:
            wait_log(self.log, "waiting until jobgrabber initializes queue")

# 需要導入模塊: from retask.queue import Queue [as 別名]
# 或者: from retask.queue.Queue import dequeue [as 別名]
 def runTest(self):
     queue = Queue('testqueue')
     queue.connect()
     task = queue.dequeue()
     i = task.data
     self.assertEqual(task.data['name'], u'kushal')

# 需要導入模塊: from retask.queue import Queue [as 別名]
# 或者: from retask.queue.Queue import dequeue [as 別名]

#.........這裏部分代碼省略.........
        (this assumes there are certs and a fedmsg config on disk)
        """

        if not (self.opts.fedmsg_enabled and fedmsg):
            return

        try:
            fedmsg.init(name="relay_inbound", cert_prefix="copr", active=True)
        except Exception as e:
            self.callback.log(
                "failed to initialize fedmsg: {0}".format(e))

    def on_pkg_skip(self, job):
        """
        Handle package skip
        """
        self._announce_start(job)
        self.callback.log(
            "Skipping: package {0} has been already built before.".format(job.pkg))
        job.status = BuildStatus.SKIPPED  # skipped
        self._announce_end(job)

    def obtain_job(self):
        """
        Retrieves new build task from queue.
        Checks if the new job can be started and not skipped.
        """
        self.update_process_title(suffix="No task")

        # this sometimes caused TypeError in random worker
        # when another one  picekd up a task to build
        # why?
        try:
            task = self.task_queue.dequeue()
        except TypeError:
            return
        if not task:
            return

        # import ipdb; ipdb.set_trace()
        job = BuildJob(task.data, self.opts)

        self.update_process_title(suffix="Task: {} chroot: {}".format(job.build_id, job.chroot))

        # Checking whether the build is not cancelled
        if not self.starting_build(job):
            return

        # Checking whether to build or skip
        if self.pkg_built_before(job.pkg, job.chroot, job.destdir):
            self.on_pkg_skip(job)
            return

        # FIXME
        # this is our best place to sanity check the job before starting
        # up any longer process

        return job

    def do_job(self, job):
        """
        Executes new job.

        :param job: :py:class:`~backend.job.BuildJob`
        """
        self._announce_start(job)

# 需要導入模塊: from retask.queue import Queue [as 別名]
# 或者: from retask.queue.Queue import dequeue [as 別名]

#.........這裏部分代碼省略.........
        Initialize Fedmsg
        (this assumes there are certs and a fedmsg config on disk)
        """

        if not (self.opts.fedmsg_enabled and fedmsg):
            return

        try:
            fedmsg.init(name="relay_inbound", cert_prefix="copr", active=True)
        except Exception as e:
            self.log.exception("Failed to initialize fedmsg: {}".format(e))

    # TODO: doing skip logic on fronted during @start_build query
    # def on_pkg_skip(self, job):
    #     """
    #     Handle package skip
    #     """
    #     self._announce_start(job)
    #     self.log.info("Skipping: package {} has been already built before.".format(job.pkg))
    #     job.status = BuildStatus.SKIPPED
    #     self.notify_job_grab_about_task_end(job)
    #     self._announce_end(job)

    def obtain_job(self):
        """
        Retrieves new build task from queue.
        Checks if the new job can be started and not skipped.
        """
        # ToDo: remove retask, use redis lua fsm logic similiar to VMM
        # this sometimes caused TypeError in random worker
        # when another one  picekd up a task to build
        # why?
        try:
            task = self.task_queue.dequeue()
        except TypeError:
            return
        if not task:
            return

        job = BuildJob(task.data, self.opts)
        self.update_process_title(suffix="Task: {} chroot: {}, obtained at {}"
                                  .format(job.build_id, job.chroot, str(datetime.now())))

        return job

    def do_job(self, job):
        """
        Executes new job.

        :param job: :py:class:`~backend.job.BuildJob`
        """

        self._announce_start(job)
        self.update_process_title(suffix="Task: {} chroot: {} build started"
                                  .format(job.build_id, job.chroot))
        status = BuildStatus.SUCCEEDED

        # setup our target dir locally
        if not os.path.exists(job.chroot_dir):
            try:
                os.makedirs(job.chroot_dir)
            except (OSError, IOError):
                self.log.exception("Could not make results dir for job: {}"
                                   .format(job.chroot_dir))
                status = BuildStatus.FAILURE

# 需要導入模塊: from retask.queue import Queue [as 別名]
# 或者: from retask.queue.Queue import dequeue [as 別名]
# coding: utf-8

NUM_QUEUES = 2

import sys
sys.path.append("/usr/share/copr/")

from retask.task import Task
from retask.queue import Queue
from backend.helpers import BackendConfigReader

opts = BackendConfigReader().read()
redis_config = {
    'host': opts['redis_host'],
    'port': opts['redis_port'],
    'db': opts['redis_db'],
}

for i in range(0, NUM_QUEUES):
    print("## Queue {}".format(i))
    q = Queue("copr-be-{}".format(i), config=redis_config)
    q.connect()
    save_q = []
    while q.length != 0:
    	task = q.dequeue()
        print task.data
        save_q.append(task)
    for t in save_q:
        q.enqueue(t)

# 需要導入模塊: from retask.queue import Queue [as 別名]
# 或者: from retask.queue.Queue import dequeue [as 別名]
from retask.task import Task
from retask.queue import Queue
queue = Queue('example')
queue.connect()
while queue.length != 0:
    task = queue.dequeue()
    print task.data