Python ProxyList.get_random_proxy方法代码示例

# 需要导入模块: from grab.proxylist import ProxyList [as 别名]
# 或者: from grab.proxylist.ProxyList import get_random_proxy [as 别名]
def main(**kwargs):
    logging.basicConfig(level=logging.DEBUG)

    pl = ProxyList()
    pl.load_file('/web/proxy-us.txt')
    proxy = pl.get_random_proxy()

    urls = [
        'http://jabbim.com',
    ]

    pool = []
    for url in urls:
        pro = Process(target=click, kwargs={'url': url,
                                            'proxy': proxy.get_address()})
        pro._start_time = time.time()
        pro.daemon = True
        pro.start()
        pool.append(pro)

    TIMEOUT = None
    while True:
        if not pro.is_alive():
            break
        else:
            time.sleep(0.5)
            if TIMEOUT and time.time() - pro._start_time > TIMEOUT:
                pro.terminate()

# 需要导入模块: from grab.proxylist import ProxyList [as 别名]
# 或者: from grab.proxylist.ProxyList import get_random_proxy [as 别名]

#.........这里部分代码省略.........

        # TODO: keep original task priority if it was set explicitly
        self.task_queue.put(task, task.priority, schedule_time=task.schedule_time)
        return True

    def stop(self):
        """
        This method set internal flag which signal spider
        to stop processing new task and shuts down.
        """

        logger_verbose.debug("Method `stop` was called")
        self.work_allowed = False

    def load_proxylist(self, source, source_type=None, proxy_type="http", auto_init=True, auto_change=True, **kwargs):
        self.proxylist = ProxyList()
        if isinstance(source, BaseProxySource):
            self.proxylist.set_source(source)
        elif isinstance(source, six.string_types):
            if source_type == "text_file":
                self.proxylist.load_file(source, proxy_type=proxy_type)
            elif source_type == "url":
                self.proxylist.load_url(source, proxy_type=proxy_type)
            else:
                raise SpiderMisuseError(
                    "Method `load_proxylist` received " "invalid `source_type` argument: %s" % source_type
                )
        else:
            raise SpiderMisuseError("Method `load_proxylist` received " "invalid `source` argument: %s" % source)

        self.proxylist_enabled = True
        self.proxy = None
        if not auto_change and auto_init:
            self.proxy = self.proxylist.get_random_proxy()
        self.proxy_auto_change = auto_change

    def process_next_page(self, grab, task, xpath, resolve_base=False, **kwargs):
        """
        Generate task for next page.

        :param grab: Grab instance
        :param task: Task object which should be assigned to next page url
        :param xpath: xpath expression which calculates list of URLS
        :param **kwargs: extra settings for new task object

        Example::

            self.follow_links(grab, 'topic', '//div[@class="topic"]/a/@href')
        """
        try:
            # next_url = grab.xpath_text(xpath)
            next_url = grab.doc.select(xpath).text()
        except IndexError:
            return False
        else:
            url = grab.make_url_absolute(next_url, resolve_base=resolve_base)
            page = task.get("page", 1) + 1
            grab2 = grab.clone()
            grab2.setup(url=url)
            task2 = task.clone(task_try_count=1, grab=grab2, page=page, **kwargs)
            self.add_task(task2)
            return True

    def render_stats(self, timing=True):
        out = ["------------ Stats: ------------"]
        out.append("Counters:")

# 需要导入模块: from grab.proxylist import ProxyList [as 别名]
# 或者: from grab.proxylist.ProxyList import get_random_proxy [as 别名]

#.........这里部分代码省略.........

        All ``**kwargs`` will be passed to `Document` constructor.
        """

        self.reset()
        if isinstance(content, six.text_type):
            raise error.GrabMisuseError('Method `setup_document` accepts only '
                                        'byte string in `content` argument.')

        # Configure Document instance
        doc = Document(grab=self)
        doc.body = content
        doc.status = ''
        doc.head = b'HTTP/1.1 200 OK\r\n\r\n'
        doc.parse(charset=kwargs.get('document_charset'))
        doc.code = 200
        doc.total_time = 0
        doc.connect_time = 0
        doc.name_lookup_time = 0
        doc.url = ''

        for key, value in kwargs.items():
            setattr(doc, key, value)

        self.doc = doc

    def change_proxy(self, random=True):
        """
        Set random proxy from proxylist.
        """

        if self.proxylist.size():
            if random:
                proxy = self.proxylist.get_random_proxy()
            else:
                proxy = self.proxylist.get_next_proxy()
            self.setup(proxy=proxy.get_address(),
                       proxy_userpwd=proxy.get_userpwd(),
                       proxy_type=proxy.proxy_type)
        else:
            logger.debug('Proxy list is empty')

    #
    # Private methods
    #

    @classmethod
    def common_headers(cls):
        """
        Build headers which sends typical browser.
        """

        return {
            'Accept': 'text/xml,application/xml,application/xhtml+xml'
                      ',text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.%d'
                      % randint(2, 5),
            'Accept-Language': 'en-us,en;q=0.%d' % (randint(5, 9)),
            'Accept-Charset': 'utf-8,windows-1251;q=0.7,*;q=0.%d'
                              % randint(5, 7),
            'Keep-Alive': '300',
        }

    def save_dumps(self):
        # pylint: disable=no-member
        thread_name = threading.currentThread().getName().lower()
        # pylint: enable=no-member

# 需要导入模块: from grab.proxylist import ProxyList [as 别名]
# 或者: from grab.proxylist.ProxyList import get_random_proxy [as 别名]

#.........这里部分代码省略.........

        All ``**kwargs`` will be passed to `Document` constructor.
        """

        self.reset()

        if isinstance(content, six.text_type):
            raise error.GrabMisuseError('Method `setup_document` accepts only '
                                        'byte string in `content` argument.')

        # Configure Document instance
        doc = Document(grab=self)
        doc.body = content
        doc.status = ''
        doc.head = ''
        doc.parse(charset=kwargs.get('document_charset'))
        doc.code = 200
        doc.total_time = 0
        doc.connect_time = 0
        doc.name_lookup_time = 0
        doc.url = ''

        for key, value in kwargs.items():
            setattr(doc, key, value)

        self.doc = doc

    def change_proxy(self):
        """
        Set random proxy from proxylist.
        """

        if self.proxylist.size():
            proxy = self.proxylist.get_random_proxy()
            self.setup(proxy=proxy.get_address(),
                       proxy_userpwd=proxy.get_userpwd(),
                       proxy_type=proxy.proxy_type)
        else:
            logger.debug('Proxy list is empty')

    def use_next_proxy(self):
        """
        Set next proxy from proxylist.
        """

        if self.proxylist.size():
            proxy = self.proxylist.get_next_proxy()
            self.setup(proxy=proxy.get_address(),
                       proxy_userpwd=proxy.get_userpwd(),
                       proxy_type=proxy.proxy_type)
        else:
            logger.debug('Proxy list is empty')

    """
    Private methods
    """

    def common_headers(self):
        """
        Build headers which sends typical browser.
        """

        return {
            'Accept': 'text/xml,application/xml,application/xhtml+xml'
                      ',text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.%d'
                      % randint(2, 5),

# 需要导入模块: from grab.proxylist import ProxyList [as 别名]
# 或者: from grab.proxylist.ProxyList import get_random_proxy [as 别名]

#.........这里部分代码省略.........

    def stop(self):
        """
        This method set internal flag which signal spider
        to stop processing new task and shuts down.
        """

        logger_verbose.debug('Method `stop` was called')
        self.work_allowed = False

    def load_proxylist(self, source, source_type=None, proxy_type='http',
                       auto_init=True, auto_change=True,
                       **kwargs):
        self.proxylist = ProxyList()
        if isinstance(source, BaseProxySource):
            self.proxylist.set_source(source)
        elif isinstance(source, six.string_types):
            if source_type == 'text_file':
                self.proxylist.load_file(source, proxy_type=proxy_type)
            elif source_type == 'url':
                self.proxylist.load_url(source, proxy_type=proxy_type)
            else:
                raise SpiderMisuseError('Method `load_proxylist` received '
                                        'invalid `source_type` argument: %s'
                                        % source_type) 
        else:
            raise SpiderMisuseError('Method `load_proxylist` received '
                                    'invalid `source` argument: %s'
                                    % source) 

        self.proxylist_enabled = True
        self.proxy = None
        if not auto_change and auto_init:
            self.proxy = self.proxylist.get_random_proxy()
        self.proxy_auto_change = auto_change

    def process_next_page(self, grab, task, xpath,
                          resolve_base=False, **kwargs):
        """
        Generate task for next page.

        :param grab: Grab instance
        :param task: Task object which should be assigned to next page url
        :param xpath: xpath expression which calculates list of URLS
        :param **kwargs: extra settings for new task object

        Example::

            self.follow_links(grab, 'topic', '//div[@class="topic"]/a/@href')
        """
        try:
            # next_url = grab.xpath_text(xpath)
            next_url = grab.doc.select(xpath).text()
        except IndexError:
            return False
        else:
            url = grab.make_url_absolute(next_url, resolve_base=resolve_base)
            page = task.get('page', 1) + 1
            grab2 = grab.clone()
            grab2.setup(url=url)
            task2 = task.clone(task_try_count=1, grab=grab2,
                               page=page, **kwargs)
            self.add_task(task2)
            return True

    def render_stats(self, timing=True):

# 需要导入模块: from grab.proxylist import ProxyList [as 别名]
# 或者: from grab.proxylist.ProxyList import get_random_proxy [as 别名]

#.........这里部分代码省略.........
            Should be one of the following: 'text_file' or 'url'.
        :param proxy_type:
            Should be one of the following: 'socks4', 'socks5' or'http'.
        :param auto_change:
            If set to `True` then automatical random proxy rotation
            will be used.


        Proxy source format should be one of the following (for each line):
            - ip:port
            - ip:port:login:password

        """
        self.proxylist = ProxyList()
        if isinstance(source, BaseProxySource):
            self.proxylist.set_source(source)
        elif isinstance(source, six.string_types):
            if source_type == 'text_file':
                self.proxylist.load_file(source, proxy_type=proxy_type)
            elif source_type == 'url':
                self.proxylist.load_url(source, proxy_type=proxy_type)
            else:
                raise SpiderMisuseError('Method `load_proxylist` received '
                                        'invalid `source_type` argument: %s'
                                        % source_type)
        else:
            raise SpiderMisuseError('Method `load_proxylist` received '
                                    'invalid `source` argument: %s'
                                    % source)

        self.proxylist_enabled = True
        self.proxy = None
        if not auto_change and auto_init:
            self.proxy = self.proxylist.get_random_proxy()
        self.proxy_auto_change = auto_change

    def process_next_page(self, grab, task, xpath,
                          resolve_base=False, **kwargs):
        """
        Generate task for next page.

        :param grab: Grab instance
        :param task: Task object which should be assigned to next page url
        :param xpath: xpath expression which calculates list of URLS
        :param **kwargs: extra settings for new task object

        Example::

            self.follow_links(grab, 'topic', '//div[@class="topic"]/a/@href')
        """
        try:
            # next_url = grab.xpath_text(xpath)
            next_url = grab.doc.select(xpath).text()
        except IndexError:
            return False
        else:
            url = grab.make_url_absolute(next_url, resolve_base=resolve_base)
            page = task.get('page', 1) + 1
            grab2 = grab.clone()
            grab2.setup(url=url)
            task2 = task.clone(task_try_count=1, grab=grab2,
                               page=page, **kwargs)
            self.add_task(task2)
            return True

    def render_stats(self, timing=None):

# 需要导入模块: from grab.proxylist import ProxyList [as 别名]
# 或者: from grab.proxylist.ProxyList import get_random_proxy [as 别名]

#.........这里部分代码省略.........
            self.stat.inc('spider:task-%s-initial' % res['task'].name)

        # Update traffic statistics
        if res['grab'] and res['grab'].response:
            self.stat.timers['network-name-lookup'] +=\
                res['grab'].response.name_lookup_time
            self.stat.timers['network-connect'] +=\
                res['grab'].response .connect_time
            self.stat.timers['network-total'] +=\
                res['grab'].response .total_time
            if from_cache:
                self.stat.inc('spider:download-size-with-cache',
                              res['grab'].response.download_size)
                self.stat.inc('spider:upload-size-with-cache',
                              res['grab'].response.upload_size)
            else:
                self.stat.inc('spider:download-size',
                              res['grab'].response.download_size)
                self.stat.inc('spider:upload-size',
                              res['grab'].response.upload_size)

        handler = self.find_task_handler(res['task'])
        self.execute_task_handler(res, handler)

    def process_grab_proxy(self, task, grab):
        "Assign new proxy from proxylist to the task"

        if task.use_proxylist:
            if self.proxylist_enabled:
                if self.proxy_auto_change:
                    self.proxy = self.change_proxy(task, grab)

    def change_proxy(self, task, grab):
        proxy = self.proxylist.get_random_proxy()
        grab.setup(proxy=proxy.get_address(),
                   proxy_userpwd=proxy.get_userpwd(),
                   proxy_type=proxy.proxy_type)
        return proxy

    def process_new_task(self, task):
        """
        Handle new task.

        1) Setup Grab object for that task
        2) Try to load task from the cache
        3) If no cached data then submit task to network transport
        """

        grab = self.setup_grab_for_task(task)
        grab_config_backup = grab.dump_config()

        cache_result = None
        if self.is_task_cacheable(task, grab):
            with self.stat.log_time('cache'):
                with self.stat.log_time('cache.read'):
                    cache_result = self.load_task_from_cache(
                        self.transport, task, grab, grab_config_backup)

        if cache_result:
            logger_verbose.debug('Task data is loaded from the cache. '
                                 'Yielding task result.')
            self.process_network_result(cache_result, from_cache=True)
            self.stat.inc('spider:task-%s-cache' % task.name)
        else:
            if self.only_cache:
                logger.debug('Skipping network request to %s' %