Python fake_useragent.UserAgent方法代码示例

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def get_aqi_result(idx: str) -> Optional[AQIRecord]:
    url = f'https://api.waqi.info/api/feed/@{idx}/obs.en.json'
    headers = {
        'User-Agent': UserAgent().chrome,
        'accept-language': 'ko',
    }
    async with aiohttp.ClientSession() as session:
        async with session.get(url, headers=headers) as res:
            d2 = await res.json(loads=json.loads)

    if d2['rxs']['obs'][0]['status'] != 'ok':
        return None

    data = d2['rxs']['obs'][0]['msg']

    return AQIRecord(
        name=data['i18n']['name']['ko'],
        aqi=data['aqi'],
        time=data['time']['utc']['v'],
        **{
            x['p']: Field(*x['v'])
            for x in data['iaqi']
            if x['p'] in ['pm25', 'pm10', 'o3', 'no2', 'so2', 'co']
        },
    ) 

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def headless(self, path: str, proxy: str = "") -> None:
        ua = UserAgent()
        userAgent = ua.random
        options = webdriver.ChromeOptions()
        options.add_argument("headless")
        options.add_argument("window-size=1500,1200")
        options.add_argument("no-sandbox")
        options.add_argument("disable-dev-shm-usage")
        options.add_argument("disable-gpu")
        options.add_argument("log-level=3")
        options.add_argument(f"user-agent={userAgent}")

        if proxy != "":
            self.proxy = True
            options.add_argument("proxy-server={}".format(proxy))

        self.driver = webdriver.Chrome(path, chrome_options=options)
        self.set_config()
        self._headless = True 

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def get_page(url, options={}):
    try:
        ua = UserAgent()
    except FakeUserAgentError:
        pass
    base_headers = {
        'User-Agent':  ua.random,
        'Accept-Encoding': 'gzip, deflate, sdch',
        'Accept-Language': 'zh-CN,zh;q=0.8'
    }
    headers = dict(base_headers, **options)
    print('Getting', url)
    try:
        r = requests.get(url, headers=headers)
        print('Getting result', url, r.status_code)
        if r.status_code == 200:
            return r.text
    except ConnectionError:
        print('Crawling Failed', url)
        return None 

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def main(args=None):
    print(datetime.now())

    with requests.Session() as session:
        if UserAgent:
            session.headers['User-Agent'] = UserAgent(
                path=ROOT + "/res/user-agent.json"
            ).random
        else:
            print("Random User-Agent disabled. Please install 'fake-useragent'.")

        p = Provider.find(session)

        if p is True or p is False:
            sys.exit(p)

        if p.connect():
            print("Connected successfully! :3")
            sys.exit(0)
        else:
            print("Connection failed :(")
            sys.exit(1) 

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def download(url, output=None, num_tasks=16, max_tries=10,
                   fake_user_agent=False, quiet=False, *, loop=None):
    if loop is None:
        loop = asyncio.get_event_loop()
    if fake_user_agent:
        user_agent = UserAgent().random
    else:
        user_agent = None
    d = Download(
        url=url,
        output_fname=output,
        num_tasks=num_tasks,
        max_tries=max_tries,
        user_agent=user_agent,
        quiet=quiet,
        loop=loop
    )
    try:
        return await d.download()
    finally:
        await d.close() 

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def getUrl(url,num_retries = 5):
    ua = UserAgent()
    headers = {'User-Agent':ua.random}
    try:
        response = requests.get(url,headers = headers)
        response.encoding = response.apparent_encoding
        data = response.text
        return data
    except Exception as e:
        if num_retries > 0:
            time.sleep(10)
            print(url)
            print("requests fail, retry!")
            return getUrl(url,num_retries-1) #递归调用
        else:
            print("retry fail!")
            print("error: %s" % e + " " + url)
            return #返回空值，程序运行报错 

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def __init__(self, crawler):
        super(RandomUserAgentMiddleware, self).__init__()

        fallback = crawler.settings.get('FAKEUSERAGENT_FALLBACK', None)

        self.ua = UserAgent(fallback=fallback)
        self.ua_type = crawler.settings.get('RANDOM_UA_TYPE', 'random')

        self.per_proxy = crawler.settings.get('RANDOM_UA_PER_PROXY', False)
        self.proxy2ua = {} 

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def get_ua():
    ua = UserAgent()
    key = random.random() * 20
    referer = ''.join([random.choice(string.ascii_letters + string.digits) for _ in range(int(key))])
    referer = 'www.' + referer.lower() + '.com'
    ip = socket.inet_ntoa(struct.pack('>I', random.randint(1, 0xffffffff)))
    HEADERS["User-Agent"] = ua.random
    HEADERS["Referer"] = referer
    HEADERS["X-Forwarded-For"] = HEADERS["X-Real-IP"] = ip

    return HEADERS 

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def testUserAgent(self):
        """
        测试UserAgent
        :return:
        """
        from fake_useragent import UserAgent
        for i in range(10000):
            ua = UserAgent(verify_ssl=False)
            print(ua.random) 

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def _set_user_agent():
    # try:
    #     user_agent = UserAgent(verify_ssl=False).random
    #     return user_agent
    # except:
    #     print("请求头设置失败，使用默认请求头")
    #     return 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.' + str(
    #         random.randint(5000, 7000)) + '.0 Safari/537.36'
    return "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36" 

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def get_http_banner(self, url):
		#http/https请求获取banner
		try:
			r = requests.get(url, headers={'UserAgent':UserAgent().random}, 
				timeout=2, verify=False, allow_redirects=True)
			soup = BeautifulSoup(r.content,'lxml')
			return soup.title.text.strip('\n').strip()
		except Exception as e:
			pass 

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def genUA():
    """returns a fake random user-agent"""
    return str(UserAgent().random) 

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def download_image(link, image_data, query):
    download_image.delta += 1
    # Use a random user agent header for bot id
    ua = UserAgent()
    headers = {"User-Agent": ua.random}

    # Get the image link
    try:
        # Get the file name and type
        file_name = link.split("/")[-1]
        type = file_name.split(".")[-1]
        type = (type[:3]) if len(type) > 3 else type
        if type.lower() == "jpe":
            type = "jpeg"
        if type.lower() not in ["jpeg", "jfif", "exif", "tiff", "gif", "bmp", "png", "webp", "jpg"]:
            type = "jpg"

        # Download the image
        print("[%] Downloading Image #{} from {}".format(download_image.delta, link))
        try:
            urllib.request.urlretrieve(link,
                                       "data/raw/{}/".format(query) + "{}.{}".format(str(download_image.delta),
                                                                                             type))
            print("[%] Downloaded File\n")
        except Exception as e:
            download_image.delta -= 1
            print("[!] Issue Downloading: {}\n[!] Error: {}".format(link, e))
    except Exception as e:
        download_image.delta -= 1
        print("[!] Issue getting: {}\n[!] Error:: {}".format(link, e)) 

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def get_useragent():
    agent = fake_useragent.UserAgent(fallback='Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0')
    return agent.random 

# 需要导入模块: import fake_useragent [as 别名]
# 或者: from fake_useragent import UserAgent [as 别名]
def get_random_ua(): #随机UA
    ua = UserAgent()
    return ua.random