Python amqp.AMQPClient类代码示例

def drain_queue(queue):
    """Strip a (declared) queue from all outstanding messages"""
    if not queue:
        return

    if not queue in queues.QUEUES:
        print "Queue %s not configured" % queue
        return

    print "Queue to be drained: %s" % queue

    if not get_user_confirmation():
        return

    client = AMQPClient()
    client.connect()

    tag = client.basic_consume(queue=queue, callback=callbacks.dummy_proc)

    print "Queue draining about to start, hit Ctrl+c when done"
    time.sleep(2)
    print "Queue draining starting"

    num_processed = 0
    while True:
        client.basic_wait()
        num_processed += 1
        sys.stderr.write("Ignored %d messages\r" % num_processed)

    client.basic_cancel(tag)
    client.close()

def main():

    usage = "Usage: %s <instance_name>\n" % PROGNAME

    if len(sys.argv) != 2:
        sys.stderr.write(usage)
        return 1

    instance_name = sys.argv[1]

    # WARNING: This assumes that instance names
    # are of the form prefix-id, and uses prefix to
    # determine the routekey for AMPQ
    prefix = instance_name.split('-')[0]
    routekey = "ganeti.%s.event.progress" % prefix
    amqp_client = AMQPClient(confirm_buffer=10)
    amqp_client.connect()
    amqp_client.exchange_declare(settings.EXCHANGE_GANETI, "topic")

    for msg in jsonstream(sys.stdin):
        msg['event_time'] = split_time(time.time())
        msg['instance'] = instance_name

        # log to stderr
        sys.stderr.write("[MONITOR] %s\n" % json.dumps(msg))

        # then send it over AMQP
        amqp_client.basic_publish(exchange=settings.EXCHANGE_GANETI,
                                  routing_key=routekey,
                                  body=json.dumps(msg))

    amqp_client.close()
    return 0

class Queue(object):
    """Queue.
       Required constructor parameters: hosts, exchange, client_id.
    """

    def __init__(self, **params):
        hosts = params['hosts']
        self.exchange = params['exchange']
        self.client_id = params['client_id']

        self.client = AMQPClient(hosts=hosts)
        self.client.connect()

        self.client.exchange_declare(exchange=self.exchange,
                                     type='topic')

    def send(self, message_key, user, instance, resource, value, details):
        body = Message(
            self.client_id, user, instance, resource, value, details)
        self.client.basic_publish(exchange=self.exchange,
                                  routing_key=message_key,
                                  body=json.dumps(body.__dict__))

    def close(self):
        self.client.close()

    def _init(self):
        log.info("Initializing")

        # Set confirm buffer to 1 for heartbeat messages
        self.client = AMQPClient(logger=log_amqp, confirm_buffer=1)
        # Connect to AMQP host
        self.client.connect()

        # Declare queues and exchanges
        exchange = settings.EXCHANGE_GANETI
        exchange_dl = queues.convert_exchange_to_dead(exchange)
        self.client.exchange_declare(exchange=exchange,
                                     type="topic")
        self.client.exchange_declare(exchange=exchange_dl,
                                     type="topic")
        for queue in queues.QUEUES:
            # Queues are mirrored to all RabbitMQ brokers
            self.client.queue_declare(queue=queue, mirrored=True,
                                      dead_letter_exchange=exchange_dl)
            # Declare the corresponding dead-letter queue
            queue_dl = queues.convert_queue_to_dead(queue)
            self.client.queue_declare(queue=queue_dl, mirrored=True)

        # Bind queues to handler methods
        for binding in queues.BINDINGS:
            try:
                callback = getattr(callbacks, binding[3])
            except AttributeError:
                log.error("Cannot find callback %s", binding[3])
                raise SystemExit(1)
            queue = binding[0]
            exchange = binding[1]
            routing_key = binding[2]

            self.client.queue_bind(queue=queue, exchange=exchange,
                                   routing_key=routing_key)

            self.client.basic_consume(queue=binding[0],
                                      callback=callback,
                                      prefetch_count=5)

            queue_dl = queues.convert_queue_to_dead(queue)
            exchange_dl = queues.convert_exchange_to_dead(exchange)
            # Bind the corresponding dead-letter queue
            self.client.queue_bind(queue=queue_dl,
                                   exchange=exchange_dl,
                                   routing_key=routing_key)

            log.debug("Binding %s(%s) to queue %s with handler %s",
                      exchange, routing_key, queue, binding[3])

        # Declare the queue that will be used for receiving requests, e.g. a
        # status check request
        hostname, pid = get_hostname(), os.getpid()
        queue = queues.get_dispatcher_request_queue(hostname, pid)
        self.client.queue_declare(queue=queue, mirrored=True,
                                  ttl=REQUEST_QUEUE_TTL)
        self.client.basic_consume(queue=queue, callback=handle_request)
        log.debug("Binding %s(%s) to queue %s with handler 'hadle_request'",
                  exchange, routing_key, queue)

    def __init__(self, logger, cluster_name):
        pyinotify.ProcessEvent.__init__(self)
        self.logger = logger
        self.cluster_name = cluster_name

        # Set max_retries to 0 for unlimited retries.
        self.client = AMQPClient(hosts=settings.AMQP_HOSTS, confirm_buffer=25,
                                 max_retries=0, logger=logger)

        logger.info("Attempting to connect to RabbitMQ hosts")

        self.client.connect()
        logger.info("Connected successfully")

        self.ganeti_master = get_ganeti_master()
        logger.debug("Ganeti Master Node: %s", self.ganeti_master)

        self.ganeti_node = get_ganeti_node()
        logger.debug("Current Ganeti Node: %s", self.ganeti_node)

        # Check if this is the master node
        logger.info("Checking if this is Ganeti Master of %s cluster: %s",
                    self.cluster_name,
                    "YES" if self.ganeti_master == self.ganeti_node else "NO")

        self.client.exchange_declare(settings.EXCHANGE_GANETI, type='topic')

        self.op_handlers = {"INSTANCE": self.process_instance_op,
                            "NETWORK": self.process_network_op,
                            "CLUSTER": self.process_cluster_op,
                            # "GROUP": self.process_group_op}
                            "TAGS": self.process_tag_op}

    def __init__(self, **params):
        hosts = params["hosts"]
        self.exchange = params["exchange"]
        self.client_id = params["client_id"]

        self.client = AMQPClient(hosts=hosts)
        self.client.connect()

        self.client.exchange_declare(exchange=self.exchange, type="topic")

    def __init__(self, **params):
        hosts = params['hosts']
        self.exchange = params['exchange']
        self.client_id = params['client_id']

        self.client = AMQPClient(hosts=hosts)
        self.client.connect()

        self.client.exchange_declare(exchange=self.exchange,
                                     type='topic')

    def handle(self, *args, **options):
        if len(args) != 1:
            raise CommandError("Please provide a queue")

        queue = args[0]
        interactive = options['interactive']
        requeue = options['requeue']

        client = AMQPClient()
        client.connect()

        pp = pprint.PrettyPrinter(indent=4, width=4)

        more_msgs = True
        counter = 0
        sep = '-' * 80
        while more_msgs:
            msg = client.basic_get(queue=queue)
            if msg:
                counter += 1
                print sep
                print 'Message %d:' % counter
                print sep
                pp.pprint(msg)
                if not requeue or interactive:
                    if interactive and not get_user_confirmation():
                        continue
                    # Acknowledging the message will remove it from the queue
                    client.basic_ack(msg)
            else:
                more_msgs = False

    def _init(self):
        log.info("Initializing")

        self.client = AMQPClient(logger=log_amqp)
        # Connect to AMQP host
        self.client.connect()

        # Declare queues and exchanges
        exchange = settings.EXCHANGE_GANETI
        exchange_dl = queues.convert_exchange_to_dead(exchange)
        self.client.exchange_declare(exchange=exchange,
                                     type="topic")
        self.client.exchange_declare(exchange=exchange_dl,
                                     type="topic")

        for queue in queues.QUEUES:
            # Queues are mirrored to all RabbitMQ brokers
            self.client.queue_declare(queue=queue, mirrored=True,
                                      dead_letter_exchange=exchange_dl)
            # Declare the corresponding dead-letter queue
            queue_dl = queues.convert_queue_to_dead(queue)
            self.client.queue_declare(queue=queue_dl, mirrored=True)

        # Bind queues to handler methods
        for binding in queues.BINDINGS:
            try:
                callback = getattr(callbacks, binding[3])
            except AttributeError:
                log.error("Cannot find callback %s", binding[3])
                raise SystemExit(1)
            queue = binding[0]
            exchange = binding[1]
            routing_key = binding[2]

            self.client.queue_bind(queue=queue, exchange=exchange,
                                   routing_key=routing_key)

            self.client.basic_consume(queue=binding[0],
                                      callback=callback,
                                      prefetch_count=5)

            queue_dl = queues.convert_queue_to_dead(queue)
            exchange_dl = queues.convert_exchange_to_dead(exchange)
            # Bind the corresponding dead-letter queue
            self.client.queue_bind(queue=queue_dl,
                                   exchange=exchange_dl,
                                   routing_key=routing_key)

            log.debug("Binding %s(%s) to queue %s with handler %s",
                      exchange, routing_key, queue, binding[3])

    def __init__(self, logger, cluster_name):
        pyinotify.ProcessEvent.__init__(self)
        self.logger = logger
        self.cluster_name = cluster_name

        # Set max_retries to 0 for unlimited retries.
        self.client = AMQPClient(hosts=settings.AMQP_HOSTS, confirm_buffer=25,
                                 max_retries=0, logger=logger)

        handler_logger.info("Attempting to connect to RabbitMQ hosts")

        self.client.connect()
        handler_logger.info("Connected succesfully")

        self.client.exchange_declare(settings.EXCHANGE_GANETI, type='topic')

        self.op_handlers = {"INSTANCE": self.process_instance_op,
                            "NETWORK": self.process_network_op}

def purge_exchanges():
    """Delete declared exchanges from RabbitMQ, after removing all queues"""
    purge_queues()

    client = AMQPClient()
    client.connect()

    exchanges = queues.EXCHANGES
    print "Exchanges to be deleted: ", exchanges

    if not get_user_confirmation():
        return

    for exch in exchanges:
        result = client.exchange_delete(exchange=exch)
        print "Deleting exchange %s. Result: %s" % (exch, result)
    client.close()

def purge_queues():
    """
        Delete declared queues from RabbitMQ. Use with care!
    """
    client = AMQPClient(max_retries=120)
    client.connect()

    print "Queues to be deleted: ", queues.QUEUES

    if not get_user_confirmation():
        return

    for queue in queues.QUEUES:
        result = client.queue_delete(queue=queue)
        print "Deleting queue %s. Result: %s" % (queue, result)

    client.close()

    def handle(self, *args, **options):
        verbose = (options["verbosity"] == "2")
        self.keep_zombies = options["keep_zombies"]
        log_level = logging.DEBUG if verbose else logging.WARNING
        log.setLevel(log_level)

        client = AMQPClient(confirms=False)
        client.connect()

        self.client = client

        for queue in queues.QUEUES:
            dead_queue = queues.convert_queue_to_dead(queue)
            while 1:
                message = client.basic_get(dead_queue)
                if not message:
                    break
                log.debug("Received message %s", message)
                self.handle_message(message)
        client.close()
        return 0

class JobFileHandler(pyinotify.ProcessEvent):
    def __init__(self, logger, cluster_name):
        pyinotify.ProcessEvent.__init__(self)
        self.logger = logger
        self.cluster_name = cluster_name

        # Set max_retries to 0 for unlimited retries.
        self.client = AMQPClient(hosts=settings.AMQP_HOSTS, confirm_buffer=25,
                                 max_retries=0, logger=logger)

        logger.info("Attempting to connect to RabbitMQ hosts")

        self.client.connect()
        logger.info("Connected successfully")

        self.ganeti_master = get_ganeti_master()
        logger.debug("Ganeti Master Node: %s", self.ganeti_master)

        self.ganeti_node = get_ganeti_node()
        logger.debug("Current Ganeti Node: %s", self.ganeti_node)

        # Check if this is the master node
        logger.info("Checking if this is Ganeti Master of %s cluster: %s",
                    self.cluster_name,
                    "YES" if self.ganeti_master == self.ganeti_node else "NO")

        self.client.exchange_declare(settings.EXCHANGE_GANETI, type='topic')

        self.op_handlers = {"INSTANCE": self.process_instance_op,
                            "NETWORK": self.process_network_op,
                            "CLUSTER": self.process_cluster_op,
                            # "GROUP": self.process_group_op}
                            "TAGS": self.process_tag_op}

    def process_IN_CLOSE_WRITE(self, event):
        self.process_IN_MOVED_TO(event)

    def process_IN_MOVED_TO(self, event):
        jobfile = os.path.join(event.path, event.name)
        if not event.name.startswith("job-"):
            self.logger.debug("Not a job file: %s" % event.path)
            return

        try:
            data = utils.ReadFile(jobfile)
        except IOError:
            return

        data = serializer.LoadJson(data)
        job = jqueue._QueuedJob.Restore(None, data, False, False)

        job_id = int(job.id)

        for op in job.ops:
            op_id = op.input.OP_ID

            msg = None
            try:
                handler_fn = self.op_handlers[op_id.split('_')[1]]
                msg, routekey = handler_fn(op, job_id)
            except KeyError:
                pass

            if not msg:
                self.logger.debug("Ignoring job: %s: %s", job_id, op_id)
                continue

            # Generate a unique message identifier
            event_time = get_time_from_status(op, job)

            # Get the last line of the op log as message
            try:
                logmsg = op.log[-1][-1]
            except IndexError:
                logmsg = None

            # Add shared attributes for all operations
            msg.update({"event_time": event_time,
                        "operation": op_id,
                        "status": op.status,
                        "cluster": self.cluster_name,
                        "logmsg": logmsg,
                        "result": op.result,
                        "jobId": job_id})

            if op.status == "success":
                msg["result"] = op.result

            if op_id == "OP_INSTANCE_CREATE" and op.status == "error":
                # In case an instance creation fails send the job input
                # so that the job can be retried if needed.
                msg["job_fields"] = op.Serialize()["input"]

            # Check if this is the master node. Only the master node should
            # deliver messages to RabbitMQ.
            current_master = get_ganeti_master()
            if self.ganeti_master != current_master:
                self.logger.warning("Ganeti Master changed! New Master: %s",
                                    current_master)

#.........这里部分代码省略.........

class Dispatcher:
    debug = False

    def __init__(self, debug=False):
        self.debug = debug
        self._init()

    def wait(self):
        log.info("Waiting for messages..")
        timeout = 600
        while True:
            try:
                # Close the Django DB connection before processing
                # every incoming message. This plays nicely with
                # DB connection pooling, if enabled and allows
                # the dispatcher to recover from broken connections
                # gracefully.
                close_connection()
                msg = self.client.basic_wait(timeout=timeout)
                if not msg:
                    log.warning(
                        "Idle connection for %d seconds. Will connect"
                        " to a different host. Verify that"
                        " snf-ganeti-eventd is running!!",
                        timeout,
                    )
                    self.client.reconnect()
            except SystemExit:
                break
            except Exception as e:
                log.exception("Caught unexpected exception: %s", e)

        self.client.basic_cancel()
        self.client.close()

    def _init(self):
        log.info("Initializing")

        self.client = AMQPClient(logger=log_amqp)
        # Connect to AMQP host
        self.client.connect()

        # Declare queues and exchanges
        exchange = settings.EXCHANGE_GANETI
        exchange_dl = queues.convert_exchange_to_dead(exchange)
        self.client.exchange_declare(exchange=exchange, type="topic")
        self.client.exchange_declare(exchange=exchange_dl, type="topic")

        for queue in queues.QUEUES:
            # Queues are mirrored to all RabbitMQ brokers
            self.client.queue_declare(queue=queue, mirrored=True, dead_letter_exchange=exchange_dl)
            # Declare the corresponding dead-letter queue
            queue_dl = queues.convert_queue_to_dead(queue)
            self.client.queue_declare(queue=queue_dl, mirrored=True)

        # Bind queues to handler methods
        for binding in queues.BINDINGS:
            try:
                callback = getattr(callbacks, binding[3])
            except AttributeError:
                log.error("Cannot find callback %s", binding[3])
                raise SystemExit(1)
            queue = binding[0]
            exchange = binding[1]
            routing_key = binding[2]

            self.client.queue_bind(queue=queue, exchange=exchange, routing_key=routing_key)

            self.client.basic_consume(queue=binding[0], callback=callback, prefetch_count=5)

            queue_dl = queues.convert_queue_to_dead(queue)
            exchange_dl = queues.convert_exchange_to_dead(exchange)
            # Bind the corresponding dead-letter queue
            self.client.queue_bind(queue=queue_dl, exchange=exchange_dl, routing_key=routing_key)

            log.debug("Binding %s(%s) to queue %s with handler %s", exchange, routing_key, queue, binding[3])