Python adbapi.ConnectionPool方法代码示例

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def process_item(self, item, spider):
        mysql_config = item.pop('__mysql__', None) # 存储时自动删除配置
        if mysql_config and item:
            if type(mysql_config) is dict:
                table = mysql_config.pop('table', None)
                db = mysql_config.get('db', None) or 'vrequest'
                mysql_config.setdefault('charset','utf8mb4')
                mysql_config.setdefault('db', db)
                dbk = hmac.new(b'',json.dumps(mysql_config, sort_keys=True).encode(),'md5').hexdigest()
                if dbk not in self.dbn:
                    self.dbn[dbk] = adbapi.ConnectionPool('pymysql', **mysql_config)
                    self.init_database(self.dbn[dbk], mysql_config, db, table, item)
                self.dbn[dbk].runInteraction(self.insert_item, db, table, item)
                return item
            else:
                raise TypeError('Unable Parse mysql_config type:{}'.format(type(mysql_config)))
        else:
            return item 

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def from_settings(cls, settings):
        """
        自定义组件或扩展很有用的方法: 这个方法名字固定, 是会被scrapy调用的。
        这里传入的cls是指当前的class
        """
        db_parms = dict(
            host=settings["MYSQL_HOST"],
            db=settings["MYSQL_DBNAME"],
            user=settings["MYSQL_USER"],
            passwd=settings["MYSQL_PASSWORD"],
            charset='utf8mb4',
            cursorclass=MySQLdb.cursors.DictCursor,
            use_unicode=True,
        )
        # 连接池ConnectionPool
        dbpool = adbapi.ConnectionPool("MySQLdb", **db_parms)

        # 此处相当于实例化pipeline, 要在init中接收。
        return cls(dbpool) 

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def from_settings(cls,settings):
        '''1、@classmethod声明一个类方法，而对于平常我们见到的则叫做实例方法。 
           2、类方法的第一个参数cls（class的缩写，指这个类本身），而实例方法的第一个参数是self，表示该类的一个实例
           3、可以通过类来调用，就像C.f()，相当于java中的静态方法'''
        dbparams=dict(
            host=settings['MYSQL_HOST'],#读取settings中的配置
            db=settings['MYSQL_DBNAME'],
            user=settings['MYSQL_USER'],
            passwd=settings['MYSQL_PASSWD'],
            charset='utf8',#编码要加上，否则可能出现中文乱码问题
            cursorclass=MySQLdb.cursors.DictCursor,
            use_unicode=False,
        )
        dbpool=adbapi.ConnectionPool('MySQLdb',**dbparams)#**表示将字典扩展为关键字参数,相当于host=xxx,db=yyy....
        return cls(dbpool)#相当于dbpool付给了这个类，self中可以得到

    #pipeline默认调用 

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def __init__(self, info):
        self.info = info
        self.dbpool = adbapi.ConnectionPool(**self.info) 

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def __setstate__(self, state):
        self.__dict__ = state
        self.info['password'] = getpass.getpass('Database password for %s: ' % (self.info['user'],))
        self.dbpool = adbapi.ConnectionPool(**self.info)
        del self.info['password'] 

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def from_settings(cls, settings):
        dbargs = dict(
            host=settings['MYSQL_HOST'],
            db=settings['MYSQL_DBNAME'],
            user=settings['MYSQL_USER'],
            passwd=settings['MYSQL_PASSWD'],
            charset='utf8',
            use_unicode=True,
        )
        dbpool = adbapi.ConnectionPool('MySQLdb', **dbargs)
        return cls(dbpool) 

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def from_settings(cls, settings):
        dbparams = dict(
            host=settings['MYSQL_HOST'],
            db=settings['MYSQL_DBNAME'],
            user=settings['MYSQL_USER'],
            passwd=settings['MYSQL_PASSWD'],
            charset='utf8',
            cursorclass=pymysql.cursors.DictCursor,
            use_unicode=False,
        )
        dbpool = adbapi.ConnectionPool('pymysql', **dbparams)
        return cls(dbpool) 

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def remove_all_deleted_files(dbpool):
    """Removes file entries from database that are deleted from the file system
    :param dbpool: twisted.enterprise.adbapi.ConnectionPool object
    """
    filenames = yield dbpool.runQuery('select filename from indexer')
    for filename in filenames:
        if not os.path.exists(filename[0]):
            print '[Indexer][Removing]: {0}'.format(filename[0])
            yield dbpool.runQuery('delete from indexer where filename=?', (filename[0],))
    resume_table_filenames = yield dbpool.runQuery('select filename from resume')
    for filename in resume_table_filenames:
        if not os.path.exists(filename[0]):
            print '[Resume][Removing]: {0}'.format(filename[0])
            yield dbpool.runQuery('delete from resume where filename=?', (filename[0],)) 

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def unshare(files, dbpool):
    """Sets the share value of the files to 0 in the database
    :param files: list of absolute file paths
    :param dbool: twisted.enterprise.adbapi.ConnectionPool object
    """
    for f in files:
        yield dbpool.runQuery('update indexer set share=0 where filename=?', (f,))
    defer.returnValue('unshared') 

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def share(files, dbpool):
    """Sets the share value of the files to 1 in the database
    :param files: list of absolute file paths
    :param dbool: twisted.enterprise.adbapi.ConnectionPool object
    """
    for f in files:
        yield dbpool.runQuery('update indexer set share=1 where filename=?', (f,))
    defer.returnValue('shared') 

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def check_hash_present_in_resume(filename, dbpool):
    """Checks if hash value is present in the resume table
    :param filename: absolute filepath
    :param dbpool: twisted.enterprise.adbapi.ConnectionPool object
    """
    response = yield dbpool.runQuery('select filename from resume where filename=?', (filename,))
    if len(response) == 0:
        defer.returnValue(False)
    else:
        filename = response[0][0]
        if not os.path.exists(filename):
            yield remove_resume_entry(filename, dbpool)
            defer.returnValue(False)
        defer.returnValue(True) 

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def remove_resume_entry(filename, dbpool):
    """Removes file entry from the resume table
    :param filename: absolute filepath
    :param dbpool: twisted.enterprise.adbapi.ConnectionPool object
    """
    filename_response = yield dbpool.runQuery('select filename from indexer where filename=?', (filename,))
    filename = filename_response[0][0]
    # print 'everything deleted from indexer and resume'
    yield dbpool.runQuery('delete from resume where filename=?', (filename,))
    yield dbpool.runQuery('delete from indexer where filename=?', (filename,)) 

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def add_new_file_entry_resume(file_entry, dbpool):
    """Add file entry to the resume table
    :param filename: absolute filepath
    :param dbpool: twisted.enterprise.adbapi.ConnectionPool object
    """
    # print 'new entry added to resume table'
    # filename, checksum = file_entry[0], file_entry[3]
    # checksum = file_entry[3]
    filename = file_entry[0]
    yield dbpool.runQuery('insert into resume values (?)', (filename,))
    yield dbpool.runQuery('insert into indexer values (?,?,?,?,?,?,?)', (file_entry)) 

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def get_file(file_hash, dbpool):
    """Returns the corresponding file object
    :param file_hash: hash of the file
    :param dbpool: twisted.enterprise.adbapi.ConnectionPool object
    """
    file_query_response = yield dbpool.runQuery('select filename from indexer where hash=?', (file_hash,))
    # print file_query_response[0][0]
    defer.returnValue(open(file_query_response[0][0], 'rb')) 

# 需要导入模块: from twisted.enterprise import adbapi [as 别名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 别名]
def get_piecehashes_of(file_hash, dbpool):
    """Returns concatenated hash of all chunks belonging to a particular file
    :param file_hash: hash of the file
    :param dbpool: twisted.enterprise.adbapi.ConnectionPool object
    """
    file_pieces_response = yield dbpool.runQuery('select piecehashes from indexer where hash=?', (file_hash,))
    defer.returnValue(file_pieces_response[0][0])