Python utils.md5string函数代码示例

    def crawl_list_page(self, response):
        
        db_cookie = self.get_cookie() or {}
        r_cookie = response.cookies
        
        print db_cookie
        print r_cookie
        
        db_ctime = int(db_cookie.get('ctime', 0))
        r_ctime = int(r_cookie.get('ctime', 0))
        
        if self.check_captcha(response):
            if db_ctime <= r_ctime:
                db_cookie = self.verify_vcode(response)
                if not db_cookie:
                    raise Exception('sougou_weixin refresh cookies fail!')
            
            #self.crawl(response.url, callback=self.crawl_list_page, cookies=db_cookie, save=response.save, force_update=True)
        else:
            # response.cookies.update(cookies)
            # 更新cookies 会导致无法转跳到 detail页面

            for each in response.doc(self.LIST_ANCHOR_SEL).items():
                taskid = md5string(each.text())
                self.crawl(each.attr.href, taskid=taskid, callback=self.detail_page, save=response.save, cookies=response.cookies)

    def _update_project(self, project):
        """update one project"""
        if project["name"] not in self.projects:
            self.projects[project["name"]] = {}
        self.projects[project["name"]].update(project)
        self.projects[project["name"]]["md5sum"] = utils.md5string(project["script"])
        if not self.projects[project["name"]].get("active_tasks", None):
            self.projects[project["name"]]["active_tasks"] = deque(maxlen=self.ACTIVE_TASKS)

        # load task queue when project is running and delete task_queue when project is stoped
        if project["status"] in ("RUNNING", "DEBUG"):
            if project["name"] not in self.task_queue:
                self._load_tasks(project["name"])
            self.task_queue[project["name"]].rate = project["rate"]
            self.task_queue[project["name"]].burst = project["burst"]

            # update project runtime info from processor by sending a _on_get_info
            # request, result is in status_page.track.save
            self.on_select_task(
                {
                    "taskid": "_on_get_info",
                    "project": project["name"],
                    "url": "data:,_on_get_info",
                    "status": self.taskdb.SUCCESS,
                    "fetch": {"save": ["min_tick"]},
                    "process": {"callback": "_on_get_info"},
                }
            )
        else:
            if project["name"] in self.task_queue:
                self.task_queue[project["name"]].rate = 0
                self.task_queue[project["name"]].burst = 0
                del self.task_queue[project["name"]]

    def _update_project(self, project):
        '''update one project'''
        if project['name'] not in self.projects:
            self.projects[project['name']] = {}
        self.projects[project['name']].update(project)
        self.projects[project['name']]['md5sum'] = utils.md5string(project['script'])
        if not self.projects[project['name']].get('active_tasks', None):
            self.projects[project['name']]['active_tasks'] = deque(maxlen=self.ACTIVE_TASKS)

        # load task queue when project is running and delete task_queue when project is stoped
        if project['status'] in ('RUNNING', 'DEBUG'):
            if project['name'] not in self.task_queue:
                self._load_tasks(project['name'])
            self.task_queue[project['name']].rate = project['rate']
            self.task_queue[project['name']].burst = project['burst']

            # update project runtime info from processor by sending a _on_get_info
            # request, result is in status_page.track.save
            self.on_select_task({
                'taskid': '_on_get_info',
                'project': project['name'],
                'url': 'data:,_on_get_info',
                'status': self.taskdb.SUCCESS,
                'fetch': {
                    'save': ['min_tick', ],
                },
                'process': {
                    'callback': '_on_get_info',
                },
            })
        else:
            if project['name'] in self.task_queue:
                self.task_queue[project['name']].rate = 0
                self.task_queue[project['name']].burst = 0
                del self.task_queue[project['name']]

    def _crawl(self, url, **kwargs):
        task = {}

        if kwargs.get('callback'):
            callback = kwargs['callback']
            if isinstance(callback, basestring) and hasattr(self, callback):
                func = getattr(self, callback)
            elif hasattr(callback, 'im_self') and callback.im_self is self:
                func = callback
                kwargs['callback'] = func.__name__
            else:
                raise NotImplementedError("self.%s() not implemented!" % callback)
            if hasattr(func, '_config'):
                for k, v in func._config.iteritems():
                    kwargs.setdefault(k, v)

        if hasattr(self, 'crawl_config'):
            for k, v in self.crawl_config.iteritems():
                kwargs.setdefault(k, v)

        url = quote_chinese(_build_url(url.strip(), kwargs.get('params')))
        if kwargs.get('files'):
            assert isinstance(kwargs.get('data', {}), dict), "data must be a dict when using with files!"
            content_type, data = _encode_multipart_formdata(kwargs.get('data', {}),
                                                            kwargs.get('files', {}))
            kwargs.setdefault('headers', {})
            kwargs['headers']['Content-Type'] = content_type
            kwargs['data'] = data
        if kwargs.get('data'):
            kwargs['data'] = _encode_params(kwargs['data'])
        if kwargs.get('data'):
            kwargs.setdefault('method', 'POST')

        schedule = {}
        for key in ('priority', 'retries', 'exetime', 'age', 'itag', 'force_update'):
            if key in kwargs and kwargs[key] is not None:
                schedule[key] = kwargs[key]
        if schedule:
            task['schedule'] = schedule

        fetch = {}
        for key in ('method', 'headers', 'data', 'timeout', 'allow_redirects', 'cookies', 'proxy', 'etag', 'last_modifed', 'save', 'js_run_at', 'js_script', 'load_images', 'fetch_type'):
            if key in kwargs and kwargs[key] is not None:
                fetch[key] = kwargs[key]
        if fetch:
            task['fetch'] = fetch

        process = {}
        for key in ('callback', ):
            if key in kwargs and kwargs[key] is not None:
                process[key] = kwargs[key]
        if process:
            task['process'] = process

        task['project'] = self.project_name
        task['url'] = url
        task['taskid'] = task.get('taskid') or md5string(url)

        self._follows.append(task)
        return task

 def test_put(n):
     logger.info("message queue put %d", n)
     start_time = time.time()
     for i in range(n):
         task['url'] = 'http://bench.pyspider.org/?l=%d' % i
         task['taskid'] = md5string(task['url'])
         queue.put(task, block=True, timeout=1)
     end_time = time.time()
     cost_time = end_time - start_time
     logger.info("cost %.2fs, %.2f/s %.2fms",
                 cost_time, n * 1.0 / cost_time, cost_time / n * 1000)

 def test_update(n, start=0):
     logger.info("taskdb update %d" % n)
     start_time = time.time()
     for i in range(n):
         task['url'] = 'http://bench.pyspider.org/?l=%d' % (i + start)
         task['taskid'] = md5string(task['url'])
         task['track'] = track
         taskdb.update(task['project'], task['taskid'], task)
     end_time = time.time()
     cost_time = end_time - start_time
     logger.info("cost %.2fs, %.2f/s %.2fms",
                 cost_time, n * 1.0 / cost_time, cost_time / n * 1000)

 def test_insert(n, start=0):
     logger.info("taskdb insert %d", n)
     start_time = time.time()
     for i in range(n):
         task['url'] = 'http://bench.pyspider.org/?l={0:d}'.format((i + start))
         task['taskid'] = md5string(task['url'])
         task['track'] = {}
         taskdb.insert(task['project'], task['taskid'], task)
     end_time = time.time()
     cost_time = end_time - start_time
     logger.info("cost %.2fs, %.2f/s %.2fms",
                 cost_time, n * 1.0 / cost_time, cost_time / n * 1000)

 def test_get(n, start=0, random=True, fields=request_task_fields):
     logger.info("taskdb get %d %s" % (n, "randomly" if random else ""))
     range_n = list(range(n))
     if random:
         from random import shuffle
         shuffle(range_n)
     start_time = time.time()
     for i in range_n:
         task['url'] = 'http://bench.pyspider.org/?l=%d' % (i + start)
         task['taskid'] = md5string(task['url'])
         task['track'] = track
         taskdb.get_task(task['project'], task['taskid'], fields=fields)
     end_time = time.time()
     cost_time = end_time - start_time
     logger.info("cost %.2fs, %.2f/s %.2fms",
                 cost_time, n * 1.0 / cost_time, cost_time / n * 1000)

def send_message(ctx, scheduler_rpc, project, message):
    """
    Send Message to project from command line
    """
    if isinstance(scheduler_rpc, six.string_types):
        scheduler_rpc = connect_rpc(ctx, None, scheduler_rpc)
    if scheduler_rpc is None and os.environ.get("SCHEDULER_NAME"):
        scheduler_rpc = connect_rpc(ctx, None, "http://%s/" % (os.environ["SCHEDULER_PORT_23333_TCP"][len("tcp://") :]))
    if scheduler_rpc is None:
        scheduler_rpc = connect_rpc(ctx, None, "http://127.0.0.1:23333/")

    return scheduler_rpc.send_task(
        {
            "taskid": utils.md5string("data:,on_message"),
            "project": project,
            "url": "data:,on_message",
            "fetch": {"save": ("__command__", message)},
            "process": {"callback": "_on_message"},
        }
    )

    def update(self, project_info):
        self.project_info = project_info

        self.name = project_info['name']
        self.group = project_info['group']
        self.db_status = project_info['status']
        self.updatetime = project_info['updatetime']

        md5sum = utils.md5string(project_info['script'])
        if (self.md5sum != md5sum or self.waiting_get_info) and self.active:
            self._send_on_get_info = True
            self.waiting_get_info = True
        self.md5sum = md5sum

        if self.active:
            self.task_queue.rate = project_info['rate']
            self.task_queue.burst = project_info['burst']
        else:
            self.task_queue.rate = 0
            self.task_queue.burst = 0

def send_message(ctx, scheduler_rpc, project, message):
    if isinstance(scheduler_rpc, six.string_types):
        scheduler_rpc = connect_rpc(ctx, None, scheduler_rpc)
    if scheduler_rpc is None and os.environ.get('SCHEDULER_NAME'):
        scheduler_rpc = connect_rpc(ctx, None, 'http://%s/' % (
            os.environ['SCHEDULER_PORT_23333_TCP'][len('tcp://'):]))
    if scheduler_rpc is None:
        scheduler_rpc = connect_rpc(ctx, None, 'http://localhost:23333/')

    return scheduler_rpc.send_task({
        'taskid': utils.md5string('data:,on_message'),
        'project': project,
        'url': 'data:,on_message',
        'fetch': {
            'save': ('__command__', message),
        },
        'process': {
            'callback': '_on_message',
        }
    })

 def _load_project(self, project):
     '''Load project into self.projects from project info dict'''
     try:
         project['md5sum'] = utils.md5string(project['script'])
         ret = self.build_module(project, self.env)
         self.projects[project['name']] = ret
     except Exception as e:
         logger.exception("load project %s error", project.get('name', None))
         ret = {
             'loader': None,
             'module': None,
             'class': None,
             'instance': None,
             'exception': e,
             'exception_log': traceback.format_exc(),
             'info': project,
             'load_time': time.time(),
         }
         self.projects[project['name']] = ret
         return False
     logger.debug('project: %s updated.', project.get('name', None))
     return True

    def update(self, project_info):
        self.project_info = project_info

        self.name = project_info['name']
        self.group = project_info['group']
        self.db_status = project_info['status']
        self.updatetime = project_info['updatetime']

        md5sum = utils.md5string(project_info['script'])
        if (self.md5sum != md5sum or self.waiting_get_info) and self.active:
            self._send_on_get_info = True
            self.waiting_get_info = True
        self.md5sum = md5sum

        if self.active:
            self.task_queue.rate = project_info['rate']
            self.task_queue.burst = project_info['burst']
        else:
            self.task_queue.rate = 0
            self.task_queue.burst = 0

        logger.info('project %s updated, status:%s, paused:%s, %d tasks',
                    self.name, self.db_status, self.paused, len(self.task_queue))

 def get_taskid(self, task):
     '''Generate taskid by information of task md5(url) by default, override me'''
     return md5string(task['url'])

 def get_taskid(self, task):
     """Generate taskid by information of task md5(url) by default, override me"""
     return md5string(task["url"])