Python utils.log方法代码示例

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def test_connect(proxy,operator,mode=None):
    conn = aiohttp.TCPConnector(verify_ssl=False)
    async with ClientSession(connector=conn) as s:
        try:
            async with s.get(url=TEST_URL,proxy=proxy[2],
                             timeout=10,allow_redirects=False) as resp:
                page = await resp.text()
                if (resp.status != 200 or str(resp.url) != TEST_URL):
                    utils.log(('[INFO]#proxy:{ip} has been dropped\n'
                               '      #Reason:Abnormal url or return Code').format(ip=proxy[1]))
                    operator.del_proxy_with_id(config.free_ipproxy_table,proxy[0])
                    operator.del_proxy_with_id(config.httpbin_table,proxy[0])
                elif mode == 'add':
                    operator.insert_valid_proxy(id=proxy[0])
                else:
                    operator.update_valid_proxy(id=proxy[0])
                   
        except Exception as e:
            utils.log(('[INFO]#proxy:{ip} has been dropped\n'
                       '      #Reason:{msg}').format(ip=proxy[1],msg=str(e)))
            operator.del_proxy_with_id(config.free_ipproxy_table,proxy[0])
            operator.del_proxy_with_id(config.httpbin_table,proxy[0])
        finally:
            operator.commit() 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def success_parse(self, response):
        proxy = response.meta.get('proxy_info')
        table = response.meta.get('table')

        self.save_page(proxy.ip, response.body)
        self.log('success_parse speed:%s meta:%s' % (time.time() - response.meta.get('cur_time'), response.meta))

        proxy.vali_count += 1
        proxy.speed = time.time() - response.meta.get('cur_time')
        if self.success_content_parse(response):
            if table == self.name:
                if proxy.speed > self.timeout:
                    self.sql.del_proxy_with_id(table, proxy.id)
                else:
                    self.sql.update_proxy(table, proxy)
            else:
                if proxy.speed < self.timeout:
                    self.sql.insert_proxy(table_name = self.name, proxy = proxy)
        else:
            if table == self.name:
                self.sql.del_proxy_with_id(table_name = table, id = proxy.id)

        self.sql.commit() 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def insert_json(self, data = {}, table_name = None, commit = False):
        try:
            keys = []
            vals = []
            for k, v in data.items():
                keys.append(k)
                vals.append(v)
            val_str = ','.join(['%s'] * len(vals))
            key_str = ','.join(keys)

            command = "INSERT IGNORE INTO {table} ({keys}) VALUES({values})". \
                format(keys = key_str, values = val_str, table = table_name)
            # utils.log('insert_json command:%s' % command)
            self.cursor.execute(command, tuple(vals))

            if commit:
                self.conn.commit()
        except Exception, e:
            utils.log('sql helper insert_json exception msg:%s' % e, logging.WARNING) 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def query(self, command, commit = False, cursor_type = 'tuple'):
        try:
            utils.log('sql helper execute command:%s' % command)

            cursor = None
            if cursor_type == 'dict':
                cursor = self.conn.cursor(pymysql.cursors.DictCursor)
            else:
                cursor = self.cursor

            cursor.execute(command)
            data = cursor.fetchall()
            if commit:
                self.conn.commit()
            return data
        except Exception, e:
            utils.log('sql helper execute exception msg:%s' % str(e))
            return None 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def query_one(self, command, commit = False, cursor_type = 'tuple'):
        try:
            utils.log('sql helper execute command:%s' % command)

            cursor = None
            if cursor_type == 'dict':
                cursor = self.conn.cursor(pymysql.cursors.DictCursor)
            else:
                cursor = self.cursor

            cursor.execute(command)
            data = cursor.fetchone()
            if commit:
                self.conn.commit()
            return data
        except Exception, e:
            utils.log('sql helper execute exception msg:%s' % str(e))
            return None 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def analysis(request):
    data = {
        'status': 'failure'
    }

    try:
        status = request.COOKIES.get('status', '')
        guid = request.COOKIES.get('guid', '0')
        if status == 'success' and guid != '0':
            msg = red.lpop(guid)
            if msg != None:
                data = json.loads(msg)
                data['status'] = status
                utils.log('info:%s' % data.get('info'))
                response = HttpResponse(json.dumps(data), content_type = "application/json")
                return response
    except Exception, e:
        logging.error('analysis data exception:%s' % e) 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def runspider(spargs):
    url = spargs.get('url')
    name = spargs.get('name', 'jd')

    if not os.path.exists('log'):
        os.makedirs('log')

    configure_logging(install_root_handler = False)
    logging.basicConfig(
            filename = 'log/%s.log' % name,
            format = '%(levelname)s %(asctime)s: %(message)s',
            level = logging.ERROR
    )
    print "get_project_settings().attributes:", get_project_settings().attributes['SPIDER_MODULES']
    process = CrawlerProcess(get_project_settings())
    start_time = time.time()
    try:
        logging.info('进入爬虫')
        process.crawl(name, **spargs)
        process.start()
    except Exception, e:
        process.stop()
        logging.error("url:%s, errorMsg:%s" % (url, e.message)) 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def runspider(spargs):
    url = spargs.get('url')
    name = spargs.get('name', 'jd')
    guid = spargs.get('guid')
    product_id = spargs.get('product_id')

    if not os.path.exists('log'):
        os.makedirs('log')

    configure_logging(install_root_handler = False)
    logging.basicConfig(
            filename = 'log/%s.log' % name,
            format = '%(levelname)s %(asctime)s: %(message)s',
            level = logging.ERROR
    )
    print "get_project_settings().attributes:", get_project_settings().attributes['SPIDER_MODULES']
    process = CrawlerProcess(get_project_settings())
    start_time = time.time()
    try:
        logging.info('进入爬虫')
        process.crawl(name, **spargs)
        process.start()
    except Exception, e:
        process.stop()
        logging.error("url:%s, errorMsg:%s" % (url, e.message)) 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def _process_syn_scan(self, pkt):
        """
        Receives SYN scan response from devices.

        """
        src_mac = pkt[sc.Ether].src
        device_id = utils.get_device_id(src_mac, self._host_state)
        device_port = pkt[sc.TCP].sport

        with self._host_state.lock:
            port_list = self._host_state.pending_syn_scan_dict.setdefault(device_id, [])
            if device_port not in port_list:
                port_list.append(device_port)
                utils.log('[SYN Scan Debug] Device {} ({}): Port {}'.format(
                    pkt[sc.IP].src, device_id, device_port
                )) 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def _process_http_host(self, pkt, device_id, remote_ip):

        try:
            http_host = pkt[http.HTTPRequest].fields['Host'].decode('utf-8')
        except Exception as e:
            return
        
        device_port = pkt[sc.TCP].sport

        with self._host_state.lock:
            self._host_state \
                .pending_dns_dict \
                .setdefault(
                    (device_id, http_host, 'http-host', device_port), set()) \
                .add(remote_ip)

        utils.log('[UPLOAD] HTTP host:', http_host) 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def _upload_initialization(self):
        """Returns True if successfully initialized."""

        # Send client's timezone to server
        ts = time.time()

        utc_offset = int(
            (datetime.datetime.fromtimestamp(ts) -
                datetime.datetime.utcfromtimestamp(ts)).total_seconds()
        )

        utc_offset_url = server_config.UTC_OFFSET_URL.format(
            user_key=self._host_state.user_key,
            offset_seconds=utc_offset
        )

        utils.log('[DATA] Update UTC offset:', utc_offset_url)
        status = requests.get(utc_offset_url).text.strip()
        utils.log('[DATA] Update UTC offset status:', status)

        return 'SUCCESS' == status 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def start(self):
        url = self.monitor.get_overrideURL(self.camera.number)  #Request to test by @mrjd in forum
        stream_type = self.camera.getStreamType(2)
        
        if url == '':
            url = self.camera.getStreamUrl(2, stream_type)

        utils.log(2, 'Camera %s :: Preview Window Opened - Manual: %s;  Stream Type: %d;  URL: %s' %(self.camera.number, self.monitor.openRequest_manual(self.camera.number), stream_type, url))
        
        if stream_type == 0:
            t = threading.Thread(target = self.getImagesMjpeg, args = (url,))         
        elif stream_type == 1:
            t = threading.Thread(target = self.getImagesSnapshot, args = (url,)) 
        elif stream_type == 2:
            t = threading.Thread(target = self.getImagesMjpegInterlace, args = (url,))
            
        t.daemon = True
        
        self.monitor.openPreview(self.camera.number)
        t.start()
        self.show()
        self.wait_closeRequest() 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def stop(self, playFullscreen = False):
        self.monitor.closePreview(self.camera.number)
        self.close()
        utils.log(2, 'Camera %s :: Preview Window Closed' %self.camera.number)
        
        if not self.monitor.abortRequested() and not self.monitor.stopped():

            if playFullscreen:
                self.monitor.maybe_stop_current()
                fullscreenplayer = settings.getSetting_int('fullscreen_player')

                if fullscreenplayer == CAMERAWITHCONTROLS:
                    cameraplayer.play(self.camera.number)
                    
                elif fullscreenplayer == CAMERAWITHOUTCONTROLS:
                    cameraplayer.play(self.camera.number, show_controls = False)
                
                elif fullscreenplayer == ALLCAMERAPLAYER:
                    allcameraplayer.play()
                
            self.wait_openRequest() 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def getImagesSnapshot(self, url, *args, **kwargs):
        """ Update camera position with snapshots """
        
        x = 0
        while not self.monitor.abortRequested() and not self.monitor.stopped() and self.monitor.previewOpened(self.camera.number):
            
            try:
                filename = os.path.join(_datapath, '%s_%s.%d.jpg') %(self.prefix, self.camera.number, x)
                urlretrieve(url, filename)
                
                if os.path.exists(filename): 
                    self.img1.setImage(filename, useCache = False)                
                    xbmcvfs.delete(os.path.join(_datapath, '%s_%s.%d.jpg') %(self.prefix, self.camera.number, x - 1))
                    self.img2.setImage(filename, useCache = False)
                    x += 1
                    
            except Exception, e:
                utils.log(3, 'Camera %s :: Error updating preview image on Snapshot: %s' %(self.camera.number, e))
                self.img1.setImage(_error, useCache = False)
                break 

# 需要导入模块: import utils [as 别名]
# 或者: from utils import log [as 别名]
def getImagesSnapshot(self, camera, url, control, prefix):
        """ Update camera position with snapshots """
        
        x = 0
        while not monitor.abortRequested() and self.isRunning:
            
            try:
                filename = os.path.join(_datapath, '%s_%s.%d.jpg') %(prefix, camera.number, x)
                urlretrieve(url, filename)
                
                if os.path.exists(filename): 
                    control[0].setImage(filename, useCache = False)                
                    xbmcvfs.delete(os.path.join(_datapath, '%s_%s.%d.jpg') %(prefix, camera.number, x - 1))
                    control[1].setImage(filename, useCache = False)
                    x += 1
                    
            except Exception, e:
                utils.log(3, 'Camera %s - Error on MJPEG: %s' %(camera.number, e))
                control[0].setImage(_error, useCache = False)
                return