Python requesocks.session函数代码示例

def get_page(url, port, timeout, max_retries=1):
    if max_retries == 1:
        session = requesocks.session()
    elif max_retries > 1:
        session = requesocks.session(config={'max_retries': 10})
    else:
        raise
    session.proxies = {'http': 'socks5://127.0.0.1:{}'.format(port),
                       'https': 'socks5://127.0.0.1:{}'.format(port)}
    return session.get(url, timeout=timeout)

def runtest():
	if target.tor is None and target.proxy is None and target.socks is None:
		try:
			request = requests.get(target.host)
			checkparam(request)
		except requests.ConnectionError as error_message:
			print "============================"
			print "HTTP connection failed with status message:"
			print error_message
			print "============================"

	elif target.tor is not None and target.proxy is None and target.socks is None:
		try:		
			session = requesocks.session()
			session.proxies = {
				"socks5": "socks5://127.0.0.1:9050"
			}
			request = session.get(target.host, auth=('user','pass'))
			checkparam(request)
		except requesocks.exceptions.ConnectionError as error_message:
			print "============================"
			print "HTTP connection failed with status message:"
			print error_message
			print "============================"

	elif target.proxy is not None and target.tor is None and target.socks is None:
		proxy = {
			"http": "http://"+target.proxy,
			"https": "https://"+target.proxy
		}
		try:
			request = requests.get(target.host, proxies=proxy)
			checkparam(request)
		except requests.ConnectionError as error_message:
			print "============================"
			print "HTTP connection failed with status message:"
			print error_message
			print "============================"		 	

	elif target.socks is not None and target.proxy is None and target.tor is None:
		try:		
			session = requesocks.session()
			session.proxies = {
				"socks4": "socks4://"+target.socks,
				"socks5": "socks5://"+target.socks
			}
			request = session.get(target.host, auth=('user','pass'))
			checkparam(request)
		except requesocks.exceptions.ConnectionError as error_message:
			print "============================"
			print "HTTP connection failed with status message:"
			print error_message
			print "============================"

def getUrl(animelist):
    s = requesocks.session()
    s.proxies = {'http':'socks5://127.0.0.1:1080'}
    urllist = []
    for item in animelist:
        res = s.get('http://www.dilidili.com/anime/'+item).content
        soup = BeautifulSoup(res, "lxml")
        try:
            u = soup.find('div', 'download area').a['href']
        except:
            continue
        if re.search('pan.baidu', u):
            urllist.append(u)
            print "Get Url:"+u
            continue
        res = s.get(u).content
        soup = BeautifulSoup(res, "lxml")
        for it in soup('a', href=True):
            if re.search('pan.baidu', it['href']):
                urllist.append(it['href'])
                print "Get url:"+it['href']
                break
    for item in urllist:
        os.system('echo %s >> urllist.txt' % item)
    return urllist

 def __init__(self):
     self.cookies = {}
     self.browser = requesocks.session()
     self.browser.cookies.clear()
     self.browser.cookies = self.cookies
     self.sock5 = None
     self.proxy = None

    def get_metadata(self):
        if self.script_alias:
            archive_url = '{0}/{1}/perfsonar/archive/'.format(self.api_url, self.script_alias)
        else:
            archive_url = '{0}/perfsonar/archive/'.format(self.api_url)
        
        session = requesocks.session()
        if os.getenv('SOCKS5'):
            session.proxies = {'http': os.getenv('SOCKS5'), 'https': os.getenv('SOCKS5')}
        session.verify=False
        r = session.get(archive_url, 
           params=dict(self.filters.metadata_filters, **self.filters.time_filters),
           headers = self.request_headers)

        self.inspect_request(r)

        if r.status_code == 200 and \
            r.headers['content-type'] == 'application/json':
            data = json.loads(r.text)
            for i in data:
                yield Metadata(i, self.api_url, self.filters)
        else:
            self.http_alert(r)
            return
            yield

def main(stochastic=False, resolve_classes=False):
	session = requesocks.session()
	session.proxies = {'http':'54.183.147.66:3128',
									   'https':'54.183.147.66:3128'}
	resolver = ClassResolver(session, resolve_classes)
	timeschedules = TimeSchedules(session)
	print('loaded')
	for index, term in enumerate(timeschedules):
		print(term)
		has_entries = False
		# ignore with probability 1 - (1 / index)
		if random.random() * index >= 1 and stochastic:
			continue
		dataset = TimeSchedulesDataset()
		for dept in term:
			print(dept)
			dataset.add(dept.get_entries())
			has_entries = True
		if has_entries:
			base = os.path.dirname(os.path.realpath(__file__))
			dataset.notify_changes('/var/www/canigraduate/data/timeschedules/%s.json' % str(term))
			dataset.write_timeschedules('/var/www/canigraduate/data/timeschedules/%s.json' % str(term))
			dataset.write_timeschedules_compressed('/var/www/canigraduate/data/timeschedules-compressed/%s.json' % str(term))
			dataset.write_timeschedules_data_compressed('/var/www/canigraduate/data/timeschedules-compressed/%s.data.json' % str(term))
		if resolve_classes:
			for id in dataset.data.keys():
				resolver.add(id, term, dataset.names[id])
	print('resolving classes')
	if resolve_classes:
		resolver.write(resolve_classes)

def getip_requesocks(url):
    print "(+) Sending request with requesocks..."
    session = requesocks.session()
    session.proxies = {'http': 'socks5://127.0.0.1:9050',
    'https': 'socks5://127.0.0.1:9050'}
    r = session.get(url)
    print "(+) IP is: " + r.text.replace("\n", "")

    def login(self):
        self.s = requests.session()

        #self.s.proxies = {'http': 'http://127.0.0.1:8087','https': 'http://127.0.0.1:8087'}
                           
        soup = BeautifulSoup(self.get_source(self.login_url))
        authenticity_token =  soup.find_all(attrs={"name": "authenticity_token"})[0]['value']
        utf8 =  soup.find_all(attrs={"name": "utf8"})[0]['value']
        data = {
            'utf8':utf8,
            'session[login]':self.username,
            'session[password]':self.password,
            'authenticity_token' : authenticity_token
        }

        self.s.post(soup.select('.sign-in__form')[0]['action'], data = data)

        soup = BeautifulSoup(self.get_source('https://tutsplus.com/account/courses'))
        account_name = soup.select('.account-header__name')[0].string

        if not account_name :
            return False

        print 'Logined success, account name: '+account_name
        return True

def get_by_rnp(number, TIMEOUT):
    payload = {
        #"action": "enviar",
        "txtRuc": "",
        "txtRnp": str(number),
        "cmbCapitulo": "",
        "cmbTipoPersona": "",
    }

    kargs = {
        "data": payload,
        "headers": _headers,
        "timeout": TIMEOUT,
    }

    url = "http://www.osce.gob.pe/consultasenlinea/rnp_consulta/ProveedoresInscritos.asp?action=enviar"
    tor_req = req_socks.session()
    tor_req.proxies = _tor_proxies
    try:
        r = tor_req.post(url, **kargs)
        name = extract_name(r.text)
        if name is not None:
            with codecs.open("out_osce.tsv", "a") as myfile:
                myfile.write(number.encode("utf-8") + "\t")
                myfile.write(name.encode("utf-8") + "\n")
    except req_socks.exceptions.Timeout:
        with codecs.open("out_osce.tsv", "a") as myfile:
            out = "Timeout error %s" % number.encode("utf-8")
            myfile.write(out + "\n")
    except socket.timeout:
        with codecs.open("out_osce.tsv", "a") as myfile:
            out = "Timeout error %s" % number.encode("utf-8")
            myfile.write(out + "\n")

    def __init__(self, args):
        super(HttpScan, self).__init__(args)
        self.session = requesocks.session()

        adapters.DEFAULT_RETRIES = self.args.max_retries
        self.tor = None
        if self.args.tor:
            self.out.log("Enabling TOR")
            self.tor = Torify()
            self.session.proxies = {'http': 'socks5://127.0.0.1:9050',
                                    'https': 'socks5://127.0.0.1:9050'}
            if self.args.check_tor:
                # Check TOR
                self.out.log("Checking IP via TOR")
                rip, tip = self.tor.check_ip(verbose=True)
                if tip is None:
                    self.out.log('TOR is not working properly!', logging.ERROR)
                    exit(-1)

        if self.args.cookies is not None:
            if path.exists(self.args.cookies) and path.isfile(self.args.cookies):
                self.cookies = MozillaCookieJar(self.args.cookies)
                self.cookies.load()
            else:
                # self.out.log('Could not find cookie file: %s' % self.args.load_cookies, logging.ERROR)
                self.cookies = Cookies.from_request(self.args.cookies)
        else:
            self.cookies = None

        self.ua = UserAgent() if self.args.user_agent is None else self.args.user_agent

def downloadNewImages(bucket, new_dog_img_dict, d_id):
	try:
		dfile = new_dog_img_dict[d_id]
		fname= d_id +'.jpg'
		if not bucket.get_key(fname):
			session = requesocks.session()
			session.proxies = {'http': 'socks5://127.0.0.1:9050','https': 'socks5://127.0.0.1:9050'}
	
			file_object = bucket.new_key(fname)
			r = session.get(dfile)
			if r.status_code == 200:
				with open(fname, 'wb') as f:
					r.raw.decode_content = True
					shutil.copyfileobj(r.raw, f)
				file_object.set_contents_from_filename('./'+fname,policy='public-read')
				os.remove(fname)
			else:
				client = MongoClient()
				db = client.pet
				err_coll = db.errs
				err_coll.insert_one({'id':d_id, 'err':r.status_code})
	except:
		client = MongoClient()
		db = client.pet
		err_coll = db.errs
		e = sys.exc_info()[1]
		err_coll.insert_one({'id':record['id'], 'err':str(e)})

		def downloadNewImages():
			tor_process = stem.process.launch_tor_with_config(config = {'SocksPort': str(self.socks_port)})
			self.success = []
			for x in self.new_dog_img_dict.keys():
				try:
					dfile = new_dog_img_dict[x]
					fname= x +'.jpg'
					if not self.bucket.get_key(fname):
						session = requesocks.session()
						session.proxies = {'http': 'socks5://127.0.0.1:9050','https': 'socks5://127.0.0.1:9050'}
				
						file_object = self.bucket.new_key(fname)
						r = session.get(dfile)
						if r.status_code == 200:
							with open(fname, 'wb') as f:
								r.raw.decode_content = True
								shutil.copyfileobj(r.raw, f)
							file_object.set_contents_from_filename('./'+fname,policy='public-read')
							self.success.append(x)
							os.remove(fname)
						else:
							self.err_coll.insert_one({'id':record['id'], 'err':r.status_code})
				except:
					e = sys.exc_info()[1]
					self.err_coll.insert_one({'id':record['id'], 'err':str(e)})

    def __init__(self, cookies_dict: dict=None):
        self.xnova_url = 'uni4.xnova.su'
        self.user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:48.0) Gecko/20100101 Firefox/48.0'
        self.error_str = None
        self.proxy = None
        # load user-agent from config/net.ini
        cfg = configparser.ConfigParser()
        cfg.read('config/net.ini', encoding='utf-8')
        if 'net' in cfg:
            self.user_agent = cfg['net']['user_agent']
            self.xnova_url = cfg['net']['xnova_url']
            self.proxy = cfg['net']['proxy']
            if self.proxy == '':
                self.proxy = None
        # construct requests HTTP session
        if self.proxy is not None:
            if self.proxy.startswith('socks5://'):
                # for SOCKS5 proxy create requesocks session
                self.sess = requesocks.session()
                logger.info('Using SOCKS5 proxy session (requesocks)')
        else:
            self.sess = requests.Session()  # else normal session
        if self.proxy is not None:
            self.sess.proxies = {'http': self.proxy, 'https': self.proxy}
            logger.info('Set HTTP/HTTPS proxy to: {0}'.format(self.proxy))

        # Some default headers for a page downloader
        self.sess.headers.update({'User-Agent': self.user_agent})
        self.sess.headers.update({'Referer': 'https://{0}/'.format(self.xnova_url)})
        self.sess.headers.update({'Accept': '*/*'})
        self.sess.headers.update({'Accept-Language': 'ru-RU,ru;q=0.8,en-US;q=0.6,en;q=0.4'})
        self.sess.headers.update({'Accept-Encoding': 'gzip, deflate'})
        
        if cookies_dict:
            self.set_cookies_from_dict(cookies_dict)

def get_http_client():
    if config['use_tor_proxy']:
        session = requesocks.session()
        session.proxies = {'http': 'socks5://127.0.0.1:%d' % config['tor_proxy_port'],
                           'https': 'socks5://127.0.0.1:%d' % config['tor_proxy_port']}
        return session
    else:
        return requests.session()

def doGet(*args, **kwargs):
	url        = args[0]
	doVhosts   = kwargs.pop('vhosts'    ,None)
	urlQueue   = kwargs.pop('urlQueue'  ,None)
	subs       = kwargs.pop('subs'      ,None)
	extraHosts = kwargs.pop('extraHosts',None)
	proxy = kwargs.pop('proxy',None)

	kwargs['allow_redirects'] = False
	session = requests.session()
	if(proxy is not None):
		session.proxies={'http':'socks5://'+proxy,'https':'socks5://'+proxy}
	resp = session.get(url[0],**kwargs)

	#If we have an https URL and we are configured to scrape hosts from the cert...
	if(url[0].find('https') != -1 and url[1] == True):
		#Pull hostnames from cert, add as additional URLs and flag as not to pull certs
		host = urlparse(url[0]).hostname
		port = urlparse(url[0]).port
		if(port is None):
			port = 443
		names = []
		try:
			cert     = ssl.get_server_certificate((host,port),ssl_version=ssl.PROTOCOL_SSLv23)
			x509     = M2Crypto.X509.load_cert_string(cert.decode('string_escape'))
			subjText = x509.get_subject().as_text()
			names    = re.findall("CN=([^\s]+)",subjText)
			altNames = x509.get_ext('subjectAltName').get_value()
			names.extend(re.findall("DNS:([^,]*)",altNames))
		except:
			pass

		for name in names:
			if(name.find('*.') != -1):
				for sub in subs:
					try:
						sub = sub.strip()
						hostname = name.replace('*.',sub+'.')
						if(hostname not in extraHosts):
							extraHosts[hostname] = 1
							address = socket.gethostbyname(hostname)
							urlQueue.put(['https://'+hostname+':'+str(port),False,url[2]])
							print '[+] Discovered subdomain '+address
					except:
						pass
				name = name.replace('*.','')
				if(name not in extraHosts):
					extraHosts[name] = 1
					urlQueue.put(['https://'+name+':'+str(port),False,url[2]])
					print '[+] Added host '+name
			else:
				if (name not in extraHosts):
					extraHosts[name] = 1
					urlQueue.put(['https://'+name+':'+str(port),False,url[2]])
					print '[+] Added host '+name
		return resp
	else:	
		return resp