Python http_client.HTTPClient类代码示例

class Stats(object):
    def __init__(self):
        self.name = ''
        self.type = ''
        self.app = None
        self.stats = {}
        # There are two modes that are not exclusive
        # first the kernel mode
        self.api_key = ''
        self.secret = ''
        self.http_proxy = ''
        self.con = HTTPClient(uri='http://kernel.shinken.io')
        # then the statsd one
        self.statsd_sock = None
        self.statsd_addr = None


    def launch_reaper_thread(self):
        self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper')
        self.reaper_thread.daemon = True
        self.reaper_thread.start()


    def register(self, app, name, _type, api_key='', secret='', http_proxy='',
                 statsd_host='localhost', statsd_port=8125, statsd_prefix='shinken',
                 statsd_enabled=False):
        self.app = app
        self.name = name
        self.type = _type
        # kernel.io part
        self.api_key = api_key
        self.secret = secret
        self.http_proxy = http_proxy
        # local statsd part
        self.statsd_host = statsd_host
        self.statsd_port = statsd_port
        self.statsd_prefix = statsd_prefix
        self.statsd_enabled = statsd_enabled

        if self.statsd_enabled:
            logger.debug('Loading statsd communication with %s:%s.%s',
                         self.statsd_host, self.statsd_port, self.statsd_prefix)
            self.load_statsd()

        # Also load the proxy if need
        self.con.set_proxy(self.http_proxy)


    # Let be crystal clear about why I don't use the statsd lib in python: it's crappy.
    # how guys did you fuck this up to this point? django by default for the conf?? really?...
    # So raw socket are far better here
    def load_statsd(self):
        try:
            self.statsd_addr = (socket.gethostbyname(self.statsd_host), self.statsd_port)
            self.statsd_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        except (socket.error, socket.gaierror), exp:
            logger.error('Cannot create statsd socket: %s' % exp)
            return

 def create_connection(self):
     self.con = HTTPClient(address=self.arb_satmap['address'], port=self.arb_satmap['port'],
                           timeout=self.timeout, data_timeout=self.data_timeout,
                           use_ssl=self.use_ssl,
                           strong_ssl=self.hard_ssl_name_check
                           )
     self.uri = self.con.uri

 def __init__(self):
     self.name = ''
     self.type = ''
     self.app = None
     self.stats = {}
     self.api_key = ''
     self.secret = ''
     self.con = HTTPClient(uri='http://kernel.shinken.io')

 def __init__(self):
     self.name = ''
     self.type = ''
     self.app = None
     self.stats = {}
     self.api_key = ''
     self.secret = ''
     self.cyph = None
     self.con = HTTPClient(uri='http://metrology')

 def __init__(self):
     self.name = ''
     self.type = ''
     self.app = None
     self.stats = {}
     # There are two modes that are not exclusive
     # first the kernel mode
     self.api_key = ''
     self.secret = ''
     self.http_proxy = ''
     self.con = HTTPClient(uri='http://kernel.shinken.io')
     # then the statsd one
     self.statsd_sock = None
     self.statsd_addr = None

 def __init__(self):
     self.name = ""
     self.type = ""
     self.app = None
     self.stats = {}
     # There are two modes that are not exclusive
     # first the kernel mode
     self.api_key = ""
     self.secret = ""
     self.http_proxy = ""
     self.con = HTTPClient(uri="http://kernel.shinken.io")
     # then the statsd one
     self.statsd_interval = 5
     self.statsd_types = ["system", "queue", "perf"]
     self.statsd_sock = None
     self.statsd_addr = None
     self.statsd_types = "system,object,queue,perf"
     self.statsd_pattern = None
     self.name_cache = {}

 def __init__(self):
     self.name = ''
     self.type = ''
     self.app = None
     self.stats = {}
     # There are two modes that are not exclusive
     # first the kernel mode
     self.api_key = ''
     self.secret = ''
     self.http_proxy = ''
     self.con = HTTPClient(uri='http://kernel.shinken.io')
     # then the statsd one
     self.statsd_interval = 5
     self.statsd_types = ['system', 'queue', 'perf']
     self.statsd_sock = None
     self.statsd_addr = None
     self.statsd_types = 'system,object,queue,perf'
     self.statsd_pattern = None
     self.name_cache = {}

class SatelliteLink(Item):
    """SatelliteLink is a common Class for link to satellite for
    Arbiter with Conf Dispatcher.

    """
    # id = 0 each Class will have it's own id

    properties = Item.properties.copy()
    properties.update({
        'address':         StringProp(fill_brok=['full_status']),
        'timeout':         IntegerProp(default='3', fill_brok=['full_status']),
        'data_timeout':    IntegerProp(default='120', fill_brok=['full_status']),
        'check_interval':  IntegerProp(default='60', fill_brok=['full_status']),
        'max_check_attempts': IntegerProp(default='3', fill_brok=['full_status']),
        'spare':              BoolProp(default='0', fill_brok=['full_status']),
        'manage_sub_realms':  BoolProp(default='1', fill_brok=['full_status']),
        'manage_arbiters':    BoolProp(default='0', fill_brok=['full_status'], to_send=True),
        'modules':            ListProp(default='', to_send=True),
        'polling_interval':   IntegerProp(default='1', fill_brok=['full_status'], to_send=True),
        'use_timezone':       StringProp(default='NOTSET', to_send=True),
        'realm':              StringProp(default='', fill_brok=['full_status'], brok_transformation=get_obj_name_two_args_and_void),
        'satellitemap':       DictProp(default=None, elts_prop=AddrProp, to_send=True, override=True),
        'use_ssl':            BoolProp(default='0', fill_brok=['full_status']),
    })

    running_properties = Item.running_properties.copy()
    running_properties.update({
        'con':                  StringProp(default=None),
        'alive':                StringProp(default=True, fill_brok=['full_status']),
        'broks':                StringProp(default=[]),
        'attempt':              StringProp(default=0, fill_brok=['full_status']), # the number of failed attempt
        'reachable':            StringProp(default=False, fill_brok=['full_status']), # can be network ask or not (dead or check in timeout or error)
        'last_check':           IntegerProp(default=0, fill_brok=['full_status']),
        'managed_confs':        StringProp(default={}),
    })
    
    def __init__(self, *args, **kwargs):
        super(SatelliteLink, self).__init__(*args, **kwargs)
        
        self.arb_satmap = {'address': '0.0.0.0', 'port': 0}
        if hasattr(self, 'address'):
            self.arb_satmap['address'] = self.address
        if hasattr(self, 'port'):
            try:
                self.arb_satmap['port'] = int(self.port)
            except:
                pass

    
    def set_arbiter_satellitemap(self, satellitemap):
        """
            arb_satmap is the satellitemap in current context:
                - A SatelliteLink is owned by an Arbiter
                - satellitemap attribute of SatelliteLink is the map defined IN THE satellite configuration
                  but for creating connections, we need the have the satellitemap of the Arbiter
        """
        self.arb_satmap = {'address': self.address, 'port': self.port}
        self.arb_satmap.update(satellitemap)


    def create_connection(self):
        self.con = HTTPClient(address=self.arb_satmap['address'], port=self.arb_satmap['port'],
                              timeout=self.timeout, data_timeout=self.data_timeout, use_ssl=self.use_ssl
                              )
        self.uri = self.con.uri
        

    def put_conf(self, conf):
        if self.con is None:
            self.create_connection()

        # Maybe the connexion was not ok, bail out
        if not self.con:
            return False

        try:
            #pyro.set_timeout(self.con, self.data_timeout)
            self.con.post('put_conf', {'conf':conf}, wait='long')
            #pyro.set_timeout(self.con, self.timeout)
            print "PUT CONF SUCESS", self.get_name()
            return True
        except HTTPExceptions, exp:
            self.con = None
            logger.error("Failed sending configuration for %s: %s" % (self.get_name(), str(exp)))
            return False

class Stats(object):
    def __init__(self):
        self.name = ""
        self.type = ""
        self.app = None
        self.stats = {}
        # There are two modes that are not exclusive
        # first the kernel mode
        self.api_key = ""
        self.secret = ""
        self.http_proxy = ""
        self.con = HTTPClient(uri="http://kernel.shinken.io")
        # then the statsd one
        self.statsd_interval = 5
        self.statsd_types = ["system", "queue", "perf"]
        self.statsd_sock = None
        self.statsd_addr = None
        self.statsd_types = "system,object,queue,perf"
        self.statsd_pattern = None
        self.name_cache = {}

    def launch_reaper_thread(self):
        self.reaper_thread = threading.Thread(None, target=self.reaper, name="stats-reaper")
        self.reaper_thread.daemon = True
        self.reaper_thread.start()

    def launch_harvester_thread(self):
        self.harvester_thread = threading.Thread(None, target=self.harvester, name="stats-harvester")
        self.harvester_thread.daemon = True
        self.harvester_thread.start()

    def register(
        self,
        app,
        name,
        _type,
        api_key="",
        secret="",
        http_proxy="",
        statsd_host="localhost",
        statsd_port=8125,
        statsd_prefix="shinken",
        statsd_enabled=False,
        statsd_interval=5,
        statsd_types=None,
        statsd_pattern=None,
    ):
        self.app = app
        self.name = name
        self.type = _type
        # kernel.io part
        self.api_key = api_key
        self.secret = secret
        self.http_proxy = http_proxy
        # local statsd part
        self.statsd_host = statsd_host
        self.statsd_port = statsd_port
        self.statsd_prefix = statsd_prefix
        self.statsd_enabled = statsd_enabled
        self.statsd_interval = statsd_interval
        if statsd_types is not None:
            self.statsd_types = [t.strip() for t in statsd_types.split(",") if t.strip()]
        if statsd_pattern is not None:
            self.statsd_pattern = statsd_pattern
        self.name_cache = {}

        if self.statsd_enabled:
            logger.debug(
                "Loading statsd communication with %s:%s.%s", self.statsd_host, self.statsd_port, self.statsd_prefix
            )
            self.load_statsd()

        # Also load the proxy if need
        self.con.set_proxy(self.http_proxy)

    # Tells whether statsd is enabled and stats should be sent
    def is_statsd_enabled(self):
        return self.statsd_sock is not None and self.name and self.app is not None

    # Tells whether kernel.status.io exporter is enabled
    def is_shinkenio_enabled(self):
        return self.name and self.api_key and self.secret

    # Let be crystal clear about why I don't use the statsd lib in python: it's crappy.
    # how guys did you fuck this up to this point? django by default for the conf?? really?...
    # So raw socket are far better here
    def load_statsd(self):
        try:
            self.statsd_addr = (socket.gethostbyname(self.statsd_host), self.statsd_port)
            self.statsd_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        except (socket.error, socket.gaierror), exp:
            logger.error("Cannot create statsd socket: %s" % exp)
            return

class SatelliteLink(Item):
    """SatelliteLink is a common Class for link to satellite for
    Arbiter with Conf Dispatcher.

    """

    # id = 0 each Class will have it's own id

    properties = Item.properties.copy()
    properties.update(
        {
            "address": StringProp(fill_brok=["full_status"]),
            "timeout": IntegerProp(default="3", fill_brok=["full_status"]),
            "data_timeout": IntegerProp(default="120", fill_brok=["full_status"]),
            "check_interval": IntegerProp(default="60", fill_brok=["full_status"]),
            "max_check_attempts": IntegerProp(default="3", fill_brok=["full_status"]),
            "spare": BoolProp(default="0", fill_brok=["full_status"]),
            "manage_sub_realms": BoolProp(default="1", fill_brok=["full_status"]),
            "manage_arbiters": BoolProp(default="0", fill_brok=["full_status"], to_send=True),
            "modules": ListProp(default="", to_send=True),
            "polling_interval": IntegerProp(default="1", fill_brok=["full_status"], to_send=True),
            "use_timezone": StringProp(default="NOTSET", to_send=True),
            "realm": StringProp(
                default="", fill_brok=["full_status"], brok_transformation=get_obj_name_two_args_and_void
            ),
            "satellitemap": DictProp(default=None, elts_prop=AddrProp, to_send=True, override=True),
            "use_ssl": BoolProp(default="0", fill_brok=["full_status"]),
            "hard_ssl_name_check": BoolProp(default="0", fill_brok=["full_status"]),
            "passive": BoolProp(default="0", fill_brok=["full_status"], to_send=True),
        }
    )

    running_properties = Item.running_properties.copy()
    running_properties.update(
        {
            "con": StringProp(default=None),
            "alive": StringProp(default=True, fill_brok=["full_status"]),
            "broks": StringProp(default=[]),
            "attempt": StringProp(default=0, fill_brok=["full_status"]),  # the number of failed attempt
            "reachable": StringProp(
                default=False, fill_brok=["full_status"]
            ),  # can be network ask or not (dead or check in timeout or error)
            "last_check": IntegerProp(default=0, fill_brok=["full_status"]),
            "managed_confs": StringProp(default={}),
        }
    )

    def __init__(self, *args, **kwargs):
        super(SatelliteLink, self).__init__(*args, **kwargs)

        self.arb_satmap = {"address": "0.0.0.0", "port": 0}
        if hasattr(self, "address"):
            self.arb_satmap["address"] = self.address
        if hasattr(self, "port"):
            try:
                self.arb_satmap["port"] = int(self.port)
            except:
                pass

    def set_arbiter_satellitemap(self, satellitemap):
        """
            arb_satmap is the satellitemap in current context:
                - A SatelliteLink is owned by an Arbiter
                - satellitemap attribute of SatelliteLink is the map defined IN THE satellite configuration
                  but for creating connections, we need the have the satellitemap of the Arbiter
        """
        self.arb_satmap = {
            "address": self.address,
            "port": self.port,
            "use_ssl": self.use_ssl,
            "hard_ssl_name_check": self.hard_ssl_name_check,
        }
        self.arb_satmap.update(satellitemap)

    def create_connection(self):
        self.con = HTTPClient(
            address=self.arb_satmap["address"],
            port=self.arb_satmap["port"],
            timeout=self.timeout,
            data_timeout=self.data_timeout,
            use_ssl=self.use_ssl,
            strong_ssl=self.hard_ssl_name_check,
        )
        self.uri = self.con.uri

    def put_conf(self, conf):
        if self.con is None:
            self.create_connection()

        # Maybe the connexion was not ok, bail out
        if not self.con:
            return False

        try:
            # pyro.set_timeout(self.con, self.data_timeout)
            self.con.get("ping")
            self.con.post("put_conf", {"conf": conf}, wait="long")
            # pyro.set_timeout(self.con, self.timeout)
            print "PUT CONF SUCESS", self.get_name()
            return True
#.........这里部分代码省略.........

class Stats(object):
    def __init__(self):
        self.name = ''
        self.type = ''
        self.app = None
        self.stats = {}
        self.api_key = ''
        self.secret = ''
        self.con = HTTPClient(uri='http://kernel.shinken.io')
        

    def launch_reaper_thread(self):
        self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper')
        self.reaper_thread.daemon = True
        self.reaper_thread.start()


    def register(self, app, name, _type, api_key='', secret=''):
        self.app = app
        self.name = name
        self.type = _type
        self.api_key = api_key
        self.secret = secret
        

    # Will increment a stat key, if None, start at 0
    def incr(self, k, v):
        _min, _max, nb, _sum = self.stats.get(k, (None, None, 0, 0))
        nb += 1
        _sum += v
        if _min is None or v < _min:
            _min = v
        if _max is None or v > _max:
            _max = v
        self.stats[k] = (_min, _max, nb, _sum)


    def _encrypt(self, data):
        m = hashlib.md5()
        m.update(self.secret)
        key = m.hexdigest()
        
        m = hashlib.md5()
        m.update(self.secret + key)
        iv = m.hexdigest()
        
        data = pad(data)
        
        aes = AES.new(key, AES.MODE_CBC, iv[:16])
        
        encrypted = aes.encrypt(data)
        return base64.urlsafe_b64encode(encrypted)



    def reaper(self):
        while True:
            now = int(time.time())
            stats = self.stats
            self.stats = {}

            if len(stats) != 0:
                s = ', '.join(['%s:%s' % (k,v) for (k,v) in stats.iteritems()])
            # If we are not in an initializer daemon we skip, we cannot have a real name, it sucks
            # to find the data after this
            if not self.name or not self.api_key or not self.secret:
                time.sleep(60)
                continue

            metrics = []
            for (k,e) in stats.iteritems():
                nk = '%s.%s.%s' % (self.type, self.name, k)
                _min, _max, nb, _sum = e
                _avg = float(_sum) / nb
                # nb can't be 0 here and _min_max can't be None too
                s = '%s.avg %f %d' % (nk, _avg, now)
                metrics.append(s)
                s = '%s.min %f %d' % (nk, _min, now)
                metrics.append(s)
                s = '%s.max %f %d' % (nk, _max, now)
                metrics.append(s)
                s = '%s.count %f %d' % (nk, nb, now)
                metrics.append(s)

            #logger.debug('REAPER metrics to send %s (%d)' % (metrics, len(str(metrics))) )
            # get the inner data for the daemon
            struct = self.app.get_stats_struct()
            struct['metrics'].extend(metrics)
            #logger.debug('REAPER whole struct %s' % struct)
            j = json.dumps(struct)
            if AES is not None and self.secret != '':
                # RESET AFTER EACH calls!
                logger.debug('Stats PUT to kernel.shinken.io/api/v1/put/ with %s %s' % (self.api_key, self.secret))

                # assume a %16 length messagexs
                encrypted_text = self._encrypt(j)
                try:
                    r = self.con.put('/api/v1/put/?api_key=%s' % (self.api_key),  encrypted_text)
                except HTTPException, exp:
                    logger.debug('Stats REAPER cannot put to the metric server %s' % exp)
#.........这里部分代码省略.........

daemon=options.daemon

if daemon not in daemon_types:
    print 'CRITICAL - ', daemon, 'is not a Shinken daemon!'
    parser.print_help()
    raise SystemExit(CRITICAL)

port=options.portnum
if port==0:
    port=daemon_types[daemon]

con = None
try:
    con = HTTPClient(address=options.hostname, port=port,  timeout=options.timeout, data_timeout=options.data_timeout, use_ssl=options.ssl)
    result=con.get('ping')
except Exception, exp:
    print "CRITICAL : the %s is not reachable : (%s)." % (daemon,exp)
    raise SystemExit(CRITICAL)

if result:
   if result=='pong':
       if daemon != 'arbiter':
           try:
               result=con.get('have_conf')
           except Exception, exp:
               print "CRITICAL : the have_conf call to the %s failed : (%s)." % (daemon,exp)
               raise SystemExit(CRITICAL)

           if result:

class Stats(object):
    def __init__(self):
        self.name = ''
        self.type = ''
        self.app = None
        self.stats = {}
        self.api_key = ''
        self.secret = ''
        self.cyph = None
        self.con = HTTPClient(uri='http://metrology')
        

    def launch_reaper_thread(self):
        self.reaper_thread = threading.Thread(None, target=self.reaper, name='stats-reaper')
        self.reaper_thread.daemon = True
        self.reaper_thread.start()


    def register(self, app, name, _type, api_key='', secret=''):
        self.app = app
        self.name = name
        self.type = _type
        self.api_key = api_key
        self.secret = secret

        # Assumea 16 len secret, but should be alreayd ok
        self.secret += '\0' * (-len(self.secret) % 16)
        if AES is not None and self.secret != '':
            self.cyph = AES.new(self.secret, AES.MODE_ECB)

        

    # Will increment a stat key, if None, start at 0
    def incr(self, k, v):
        _min, _max, nb, _sum = self.stats.get(k, (None, None, 0, 0))
        nb += 1
        _sum += v
        if _min is None or v < _min:
            _min = v
        if _max is None or v > _max:
            _max = v
        self.stats[k] = (_min, _max, nb, _sum)



    def reaper(self):
        while True:
            now = int(time.time())
            logger.debug('REAPER loop')
            stats = self.stats
            self.stats = {}

            if len(stats) != 0:
                s = ', '.join(['%s:%s' % (k,v) for (k,v) in stats.iteritems()])
                logger.debug("REAPER: %s ", s)
            # If we are not in an initializer daemon we skip, we cannot have a real name, it sucks
            # to find the data after this
            if not self.name:
                time.sleep(10)
                continue

            logger.debug('REAPER we got a name')
            metrics = []
            for (k,e) in stats.iteritems():
                nk = '%s.%s.%s' % (self.type, self.name, k)
                logger.debug('REAP %s:%s', nk, e)
                _min, _max, nb, _sum = e
                _avg = float(_sum) / nb
                # nb can't be 0 here and _min_max can't be None too
                s = '%s.avg %f %d' % (nk, _avg, now)
                metrics.append(s)
                s = '%s.min %f %d' % (nk, _min, now)
                metrics.append(s)
                s = '%s.max %f %d' % (nk, _max, now)
                metrics.append(s)
                s = '%s.count %f %d' % (nk, nb, now)
                metrics.append(s)

            logger.debug('REAPER metrics to send %s (%d)', metrics, len(str(metrics)) )
            # get the inner data for the daemon
            struct = self.app.get_stats_struct()
            struct['metrics'].extend(metrics)
            logger.debug('REAPER whole struct %s', struct)
            j = json.dumps(struct)
            if self.cyph is not None:
                logger.debug('PUT to /api/v1/put/ with %s %s', self.api_key, self.secret)
                # assume a %16 length messagexs
                j += '\0' * (-len(j) % 16)
                encrypted_text = self.cyph.encrypt(j)
                try:
                    self.con.put('/api/v1/put/', encrypted_text)
                except HTTPException, exp:
                    logger.debug('REAPER cannot put to the metric server %s', exp)
            time.sleep(10)