Python xlog.warn函数代码示例

    def test2(self):
        work_ciphers = ["AES128-SHA"]
        for cipher in self.cipher_list:
            if cipher in work_ciphers:
                continue
            else:
                work_ciphers.append(cipher)

            xlog.debug("%s", cipher)
            cipher_suites = (work_ciphers)

            openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile, cipher_suites=cipher_suites)
            try:
                ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context)
                server_type = test_server_type(ssl, self.ip)
                xlog.debug("%s", server_type)
                if "gws" not in server_type:
                    work_ciphers.remove(cipher)
            except Exception as e:
                xlog.warn("err:%s", e)
                try:
                    work_ciphers.remove(cipher)
                except:
                    pass

        work_str = ""
        for cipher in work_ciphers:
            work_str += cipher + ":"
        xlog.info("work ciphers:%s", work_str)

    def save(self):
        CONFIG_USER_FILENAME = os.path.abspath( os.path.join(root_path, 'data', 'gae_proxy', 'config.ini'))
        try:
            f = open(CONFIG_USER_FILENAME, 'w')
            if self.user_special.appid != "":
                f.write("[gae]\n")
                f.write("appid = %s\n" % self.user_special.appid)
                f.write("password = %s\n\n" % self.user_special.password)

            f.write("[proxy]\n")
            f.write("enable = %s\n" % self.user_special.proxy_enable)
            f.write("type = %s\n" % self.user_special.proxy_type)
            f.write("host = %s\n" % self.user_special.proxy_host)
            f.write("port = %s\n" % self.user_special.proxy_port)
            f.write("user = %s\n" % self.user_special.proxy_user)
            f.write("passwd = %s\n\n" % self.user_special.proxy_passwd)

            if self.user_special.host_appengine_mode != "gae":
                f.write("[hosts]\n")
                f.write("appengine.google.com = %s\n" % self.user_special.host_appengine_mode)
                f.write("www.google.com = %s\n\n" % self.user_special.host_appengine_mode)

            f.write("[google_ip]\n")

            if int(self.user_special.auto_adjust_scan_ip_thread_num) != self.DEFAULT_CONFIG.getint('google_ip', 'auto_adjust_scan_ip_thread_num'):
                f.write("auto_adjust_scan_ip_thread_num = %d\n\n" % int(self.user_special.auto_adjust_scan_ip_thread_num))
            if int(self.user_special.scan_ip_thread_num) != self.DEFAULT_CONFIG.getint('google_ip', 'max_scan_ip_thread_num'):
                f.write("max_scan_ip_thread_num = %d\n\n" % int(self.user_special.scan_ip_thread_num))

            if int(self.user_special.use_ipv6) != self.DEFAULT_CONFIG.getint('google_ip', 'use_ipv6'):
                f.write("use_ipv6 = %d\n\n" % int(self.user_special.use_ipv6))

            f.close()
        except:
            xlog.warn("launcher.config save user config fail:%s", CONFIG_USER_FILENAME)

    def do_POST(self):
        try:
            refer = self.headers.getheader('Referer')
            netloc = urlparse.urlparse(refer).netloc
            if not netloc.startswith("127.0.0.1") and not netloc.startswitch("localhost"):
                xlog.warn("web control ref:%s refuse", netloc)
                return
        except:
            pass
        xlog.debug ('GAEProxy web_control %s %s %s ', self.address_string(), self.command, self.path)
        try:
            ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
            if ctype == 'multipart/form-data':
                self.postvars = cgi.parse_multipart(self.rfile, pdict)
            elif ctype == 'application/x-www-form-urlencoded':
                length = int(self.headers.getheader('content-length'))
                self.postvars = urlparse.parse_qs(self.rfile.read(length), keep_blank_values=1)
            else:
                self.postvars = {}
        except:
            self.postvars = {}

        path = urlparse.urlparse(self.path).path
        if path == '/deploy':
            return self.req_deploy_handler()
        elif path == "/config":
            return self.req_config_handler()
        elif path == "/scan_ip":
            return self.req_scan_ip_handler()
        elif path.startswith("/importip"):
            return self.req_importip_handler()
        else:
            self.wfile.write(b'HTTP/1.1 404\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n404 Not Found')
            xlog.info('%s "%s %s HTTP/1.1" 404 -', self.address_string(), self.command, self.path)

def check_win10():
    if sys.platform != "win32":
        return False

    import ctypes

    class OSVERSIONINFOEXW(ctypes.Structure):
        _fields_ = [
            ("dwOSVersionInfoSize", ctypes.c_ulong),
            ("dwMajorVersion", ctypes.c_ulong),
            ("dwMinorVersion", ctypes.c_ulong),
            ("dwBuildNumber", ctypes.c_ulong),
            ("dwPlatformId", ctypes.c_ulong),
            ("szCSDVersion", ctypes.c_wchar * 128),
            ("wServicePackMajor", ctypes.c_ushort),
            ("wServicePackMinor", ctypes.c_ushort),
            ("wSuiteMask", ctypes.c_ushort),
            ("wProductType", ctypes.c_byte),
            ("wReserved", ctypes.c_byte),
        ]

    os_version = OSVERSIONINFOEXW()
    os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version)
    retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version))
    if retcode != 0:
        xlog.warn("Failed to get win32 OS version")
        return False

    if os_version.dwMajorVersion == 10:
        xlog.info("detect Win10, enable connect concurent control.")
        return True

    return False

def fetch(method, host, path, headers, payload, bufsize=8192):
    request_data = "%s %s HTTP/1.1\r\n" % (method, path)
    request_data += "".join("%s: %s\r\n" % (k, v) for k, v in headers.items())
    request_data += "\r\n"

    ssl_sock = https_manager.get_ssl_connection(host)
    if not ssl_sock:
        return

    ssl_sock.send(request_data.encode())
    payload_len = len(payload)
    start = 0
    while start < payload_len:
        send_size = min(payload_len - start, 65535)
        sended = ssl_sock.send(payload[start : start + send_size])
        start += sended

    response = httplib.HTTPResponse(ssl_sock, buffering=True)

    response.ssl_sock = ssl_sock
    try:
        orig_timeout = ssl_sock.gettimeout()
        ssl_sock.settimeout(90)
        response.begin()
        ssl_sock.settimeout(orig_timeout)
    except httplib.BadStatusLine as e:
        xlog.warn("direct_handler.fetch bad status line:%r", e)
        response = None
    except Exception as e:
        xlog.warn("direct_handler.fetch:%r", e)
    return response

    def remove_ip_process(self):
        try:
            while True:

                try:
                    ip_str = self.to_remove_ip_list.get_nowait()
                except:
                    break

                result = check_ip.test(ip_str)
                if result and result.appspot_ok:
                    self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type)
                    xlog.debug("remove ip process, restore ip:%s", ip_str)
                    continue

                if not check_ip.network_is_ok():
                    self.to_remove_ip_list.put(ip_str)
                    xlog.warn("network is unreachable. check your network connection.")
                    return

                xlog.info("real remove ip:%s ", ip_str)
                self.iplist_need_save = 1
        finally:
            self.remove_ip_thread_num_lock.acquire()
            self.remove_ip_thread_num -= 1
            self.remove_ip_thread_num_lock.release()

    def keep_alive_thread(self):
        while self.keep_alive:
            time.sleep(1)
            try:
                sock_list = self.new_conn_pool.get_need_keep_alive(maxtime=self.keep_alive-3)
                for ssl_sock in sock_list:
                    ssl_sock.close()

                sock_list = self.gae_conn_pool.get_need_keep_alive(maxtime=self.keep_alive-3)
                for ssl_sock in sock_list:
                    # only keep little alive link.
                    # if you have 25 appid, you can keep 5 alive link.
                    if self.gae_conn_pool.qsize() > max(1, len(appid_manager.working_appid_list)/2):
                        ssl_sock.close()
                        continue

                    #inactive_time = time.time() -ssl_sock.last_use_time
                    #logging.debug("inactive_time:%d", inactive_time)
                    if self.head_request(ssl_sock):
                        self.save_ssl_connection_for_reuse(ssl_sock)
                    else:
                        ssl_sock.close()

                self.create_more_connection()
            except Exception as e:
                xlog.warn("keep alive except:%r", e)

    def load(self):
        ConfigParser.RawConfigParser.OPTCRE = re.compile(r'(?P<option>[^=\s][^=]*)\s*(?P<vi>[=])\s*(?P<value>.*)$')

        self.USER_CONFIG = ConfigParser.ConfigParser()
        CONFIG_USER_FILENAME = os.path.abspath( os.path.join(root_path, 'data', 'gae_proxy', 'config.ini'))

        self.DEFAULT_CONFIG = ConfigParser.ConfigParser()
        DEFAULT_CONFIG_FILENAME = os.path.abspath( os.path.join(current_path, 'proxy.ini'))

        try:
            if os.path.isfile(CONFIG_USER_FILENAME):
                self.USER_CONFIG.read(CONFIG_USER_FILENAME)
            else:
                return

            if os.path.isfile(DEFAULT_CONFIG_FILENAME):
                self.DEFAULT_CONFIG.read(DEFAULT_CONFIG_FILENAME)
            else:
                return

            try:
                self.user_special.appid = self.USER_CONFIG.get('gae', 'appid')
                self.user_special.password = self.USER_CONFIG.get('gae', 'password')
            except:
                pass

            try:
                self.user_special.host_appengine_mode = self.USER_CONFIG.get('hosts', 'appengine.google.com')
            except:
                pass

            try:
                self.user_special.ip_connect_interval = config.CONFIG.getint('google_ip', 'ip_connect_interval')
            except:
                pass

            try:
                self.user_special.scan_ip_thread_num = config.CONFIG.getint('google_ip', 'max_scan_ip_thread_num')
            except:
                self.user_special.scan_ip_thread_num = self.DEFAULT_CONFIG.getint('google_ip', 'max_scan_ip_thread_num')

            try:
                self.user_special.auto_adjust_scan_ip_thread_num = config.CONFIG.getint('google_ip', 'auto_adjust_scan_ip_thread_num')
            except:
                pass

            try:
                self.user_special.use_ipv6 = config.CONFIG.getint('google_ip', 'use_ipv6')
            except:
                pass

            self.user_special.proxy_enable = self.USER_CONFIG.get('proxy', 'enable')
            self.user_special.proxy_type = self.USER_CONFIG.get('proxy', 'type')
            self.user_special.proxy_host = self.USER_CONFIG.get('proxy', 'host')
            self.user_special.proxy_port = self.USER_CONFIG.get('proxy', 'port')
            self.user_special.proxy_user = self.USER_CONFIG.get('proxy', 'user')
            self.user_special.proxy_passwd = self.USER_CONFIG.get('proxy', 'passwd')

        except Exception as e:
            xlog.warn("User_config.load except:%s", e)

def load_tasks():
    tasks = []
    intervals = []

    import imp
    tasks_path = config.tasks_path
    for fileName in os.listdir(tasks_path):
        if fileName.lower().endswith('.py'):
            xlog.info('start load %s' % fileName)
            try:
                with open(os.path.join(tasks_path, fileName), 'rb') as fpy:
                    task = imp.load_source('Auto-tasks_%d_%s' % (len(tasks), fileName), tasks_path, fpy)

                    name = task.name
                    interval = task.run_interval

                    task.init()

                    tasks.append(task)
                    intervals.append(interval)

                    xlog.info('load task %s success.' % name)
            except:
                xlog.warn('load %s fail.' % fileName)
    del imp

    return (tasks, intervals)

    def load_ip_range(self):
        self.ip_range_map = {}
        self.ip_range_list = []
        self.ip_range_index = []
        self.candidate_amount_ip = 0

        content = self.load_range_content()
        lines = content.splitlines()
        for line in lines:
            if len(line) == 0 or line[0] == '#':
                continue

            try:
                begin, end = ip_utils.split_ip(line)
                nbegin = ip_utils.ip_string_to_num(begin)
                nend = ip_utils.ip_string_to_num(end)
                if not nbegin or not nend or nend < nbegin:
                    xlog.warn("load ip range:%s fail", line)
                    continue
            except Exception as e:
                xlog.exception("load ip range:%s fail:%r", line, e)
                continue

            self.ip_range_map[self.candidate_amount_ip] = [nbegin, nend]
            self.ip_range_list.append( [nbegin, nend] )
            self.ip_range_index.append(self.candidate_amount_ip)
            num = nend - nbegin
            self.candidate_amount_ip += num
            # print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num

        self.ip_range_index.sort()

def test(ip_str, loop=1):
    xlog.info("==>%s", ip_str)
    check = Check_frame(ip_str, check_cert=False)

    for i in range(loop):

        result = check.check(callback=test_app_head)
        if not result:
            if "gws" in check.result.server_type:
                xlog.warn("ip:%s server_type:%s but appengine check fail.", ip_str, check.result.server_type)

            xlog.warn("check fail")
            #continue
        else:
            xlog.debug("=======app check ok: %s", ip_str)
            check.result.appspot_ok = result


        result = check.check(callback=test_server_type, check_ca=True)
        if not result:
            xlog.debug("test server type fail")
            continue

        check.result.server_type = result
        xlog.info("========== %s type:%s domain:%s handshake:%d", ip_str, check.result.server_type,
                     check.result.domain, check.result.handshake_time)

    return check.result

def request(headers={}, payload=None):
    max_retry = 3
    for i in range(max_retry):
        ssl_sock = None
        try:
            ssl_sock = https_manager.get_ssl_connection()
            if not ssl_sock:
                xlog.debug("create_ssl_connection fail")
                continue

            if ssl_sock.host == "":
                ssl_sock.appid = appid_manager.get_appid()
                if not ssl_sock.appid:
                    raise GAE_Exception(1, "no appid can use")
                headers["Host"] = ssl_sock.appid + ".appspot.com"
                ssl_sock.host = headers["Host"]
            else:
                headers["Host"] = ssl_sock.host

            response = _request(ssl_sock, headers, payload)
            if not response:
                ssl_sock.close()
                continue

            response.ssl_sock = ssl_sock
            return response

        except Exception as e:
            xlog.warn("request failed:%s", e)
            if ssl_sock:
                ssl_sock.close()
    raise GAE_Exception(2, "try max times")

    def load(self):
        ConfigParser.RawConfigParser.OPTCRE = re.compile(r"(?P<option>[^=\s][^=]*)\s*(?P<vi>[=])\s*(?P<value>.*)$")

        self.DEFAULT_CONFIG = ConfigParser.ConfigParser()
        DEFAULT_CONFIG_FILENAME = os.path.abspath(os.path.join(current_path, "proxy.ini"))

        self.USER_CONFIG = ConfigParser.ConfigParser()
        CONFIG_USER_FILENAME = os.path.abspath(os.path.join(root_path, "data", "gae_proxy", "config.ini"))

        try:
            if os.path.isfile(DEFAULT_CONFIG_FILENAME):
                self.DEFAULT_CONFIG.read(DEFAULT_CONFIG_FILENAME)
                self.user_special.scan_ip_thread_num = self.DEFAULT_CONFIG.getint("google_ip", "max_scan_ip_thread_num")
            else:
                return

            if os.path.isfile(CONFIG_USER_FILENAME):
                self.USER_CONFIG.read(CONFIG_USER_FILENAME)
            else:
                return

            try:
                self.user_special.appid = self.USER_CONFIG.get("gae", "appid")
                self.user_special.password = self.USER_CONFIG.get("gae", "password")
            except:
                pass

            try:
                self.user_special.host_appengine_mode = self.USER_CONFIG.get("hosts", "appengine.google.com")
            except:
                pass

            try:
                self.user_special.scan_ip_thread_num = config.CONFIG.getint("google_ip", "max_scan_ip_thread_num")
            except:
                self.user_special.scan_ip_thread_num = self.DEFAULT_CONFIG.getint("google_ip", "max_scan_ip_thread_num")

            try:
                self.user_special.auto_adjust_scan_ip_thread_num = config.CONFIG.getint(
                    "google_ip", "auto_adjust_scan_ip_thread_num"
                )
            except:
                pass

            try:
                self.user_special.use_ipv6 = config.CONFIG.getint("google_ip", "use_ipv6")
            except:
                pass

            self.user_special.proxy_enable = self.USER_CONFIG.get("proxy", "enable")
            self.user_special.proxy_type = self.USER_CONFIG.get("proxy", "type")
            self.user_special.proxy_host = self.USER_CONFIG.get("proxy", "host")
            self.user_special.proxy_port = self.USER_CONFIG.get("proxy", "port")
            self.user_special.proxy_user = self.USER_CONFIG.get("proxy", "user")
            self.user_special.proxy_passwd = self.USER_CONFIG.get("proxy", "passwd")

        except Exception as e:
            xlog.warn("User_config.load except:%s", e)

    def request(self, method, host, schema="http", path="/", headers={}, data="", timeout=40):
        # change top domain to xx-net.net
        # this domain bypass the cloudflare front for ipv4
        #p = host.find(".")
        #host_sub = host[:p]
        #host = host_sub + ".xx-net.net"

        schema = "http"
        # force schema to http, avoid cert fail on heroku curl.
        # and all x-server provide ipv4 access

        url = schema + "://" + host + path
        payloads = ['%s %s HTTP/1.1\r\n' % (method, url)]
        for k in headers:
            v = headers[k]
            payloads.append('%s: %s\r\n' % (k, v))
        head_payload = "".join(payloads)

        request_body = '%s%s%s%s' % \
                       ((struct.pack('!H', len(head_payload)),  head_payload,
                         struct.pack('!I', len(data)), data))
        request_headers = {'Content-Length': len(data), 'Content-Type': 'application/octet-stream'}

        heroku_host = ""
        content, status, response = self._request(
                                            "POST", heroku_host, "/2/",
                                            request_headers, request_body, timeout)

        # xlog.info('%s "PHP %s %s %s" %s %s', handler.address_string(), handler.command, url, handler.protocol_version, response.status, response.getheader('Content-Length', '-'))
        # xlog.debug("status:%d", status)
        if status == 200:
            xlog.debug("%s %s%s trace:%s", method, host, path, response.task.get_trace())
            self.last_success_time = time.time()
            self.continue_fail_num = 0
            self.success_num += 1
        else:
            if status == 404:
                heroku_host = response.ssl_sock.host
                xlog.warn("heroku:%s fail", heroku_host)
                try:
                    self.host_manager.remove(heroku_host)
                except:
                    pass

            self.last_fail_time = time.time()
            self.continue_fail_num += 1
            self.fail_num += 1

        try:
            res = simple_http_client.TxtResponse(content)
        except:
            return "", 501, {}

        res.worker = response.worker
        res.task = response.task
        return res.body, res.status, res

 def runJob(self):
     while self.check_num < 1000000:
         try:
             time.sleep(1)
             ip_int = ip_range.get_ip()
             #ip_int = ip_range.random_get_ip()
             ip_str = ip_utils.ip_num_to_string(ip_int)
             self.check_ip(ip_str)
         except Exception as e:
             xlog.warn("google_ip.runJob fail:%s", e)