Python adbapi.ConnectionPool方法代碼示例

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def process_item(self, item, spider):
        mysql_config = item.pop('__mysql__', None) # 存儲時自動刪除配置
        if mysql_config and item:
            if type(mysql_config) is dict:
                table = mysql_config.pop('table', None)
                db = mysql_config.get('db', None) or 'vrequest'
                mysql_config.setdefault('charset','utf8mb4')
                mysql_config.setdefault('db', db)
                dbk = hmac.new(b'',json.dumps(mysql_config, sort_keys=True).encode(),'md5').hexdigest()
                if dbk not in self.dbn:
                    self.dbn[dbk] = adbapi.ConnectionPool('pymysql', **mysql_config)
                    self.init_database(self.dbn[dbk], mysql_config, db, table, item)
                self.dbn[dbk].runInteraction(self.insert_item, db, table, item)
                return item
            else:
                raise TypeError('Unable Parse mysql_config type:{}'.format(type(mysql_config)))
        else:
            return item 

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def from_settings(cls, settings):
        """
        自定義組件或擴展很有用的方法: 這個方法名字固定, 是會被scrapy調用的。
        這裏傳入的cls是指當前的class
        """
        db_parms = dict(
            host=settings["MYSQL_HOST"],
            db=settings["MYSQL_DBNAME"],
            user=settings["MYSQL_USER"],
            passwd=settings["MYSQL_PASSWORD"],
            charset='utf8mb4',
            cursorclass=MySQLdb.cursors.DictCursor,
            use_unicode=True,
        )
        # 連接池ConnectionPool
        dbpool = adbapi.ConnectionPool("MySQLdb", **db_parms)

        # 此處相當於實例化pipeline, 要在init中接收。
        return cls(dbpool) 

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def from_settings(cls,settings):
        '''1、@classmethod聲明一個類方法，而對於平常我們見到的則叫做實例方法。 
           2、類方法的第一個參數cls（class的縮寫，指這個類本身），而實例方法的第一個參數是self，表示該類的一個實例
           3、可以通過類來調用，就像C.f()，相當於java中的靜態方法'''
        dbparams=dict(
            host=settings['MYSQL_HOST'],#讀取settings中的配置
            db=settings['MYSQL_DBNAME'],
            user=settings['MYSQL_USER'],
            passwd=settings['MYSQL_PASSWD'],
            charset='utf8',#編碼要加上，否則可能出現中文亂碼問題
            cursorclass=MySQLdb.cursors.DictCursor,
            use_unicode=False,
        )
        dbpool=adbapi.ConnectionPool('MySQLdb',**dbparams)#**表示將字典擴展為關鍵字參數,相當於host=xxx,db=yyy....
        return cls(dbpool)#相當於dbpool付給了這個類，self中可以得到

    #pipeline默認調用 

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def __init__(self, info):
        self.info = info
        self.dbpool = adbapi.ConnectionPool(**self.info) 

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def __setstate__(self, state):
        self.__dict__ = state
        self.info['password'] = getpass.getpass('Database password for %s: ' % (self.info['user'],))
        self.dbpool = adbapi.ConnectionPool(**self.info)
        del self.info['password'] 

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def from_settings(cls, settings):
        dbargs = dict(
            host=settings['MYSQL_HOST'],
            db=settings['MYSQL_DBNAME'],
            user=settings['MYSQL_USER'],
            passwd=settings['MYSQL_PASSWD'],
            charset='utf8',
            use_unicode=True,
        )
        dbpool = adbapi.ConnectionPool('MySQLdb', **dbargs)
        return cls(dbpool) 

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def from_settings(cls, settings):
        dbparams = dict(
            host=settings['MYSQL_HOST'],
            db=settings['MYSQL_DBNAME'],
            user=settings['MYSQL_USER'],
            passwd=settings['MYSQL_PASSWD'],
            charset='utf8',
            cursorclass=pymysql.cursors.DictCursor,
            use_unicode=False,
        )
        dbpool = adbapi.ConnectionPool('pymysql', **dbparams)
        return cls(dbpool) 

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def remove_all_deleted_files(dbpool):
    """Removes file entries from database that are deleted from the file system
    :param dbpool: twisted.enterprise.adbapi.ConnectionPool object
    """
    filenames = yield dbpool.runQuery('select filename from indexer')
    for filename in filenames:
        if not os.path.exists(filename[0]):
            print '[Indexer][Removing]: {0}'.format(filename[0])
            yield dbpool.runQuery('delete from indexer where filename=?', (filename[0],))
    resume_table_filenames = yield dbpool.runQuery('select filename from resume')
    for filename in resume_table_filenames:
        if not os.path.exists(filename[0]):
            print '[Resume][Removing]: {0}'.format(filename[0])
            yield dbpool.runQuery('delete from resume where filename=?', (filename[0],)) 

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def unshare(files, dbpool):
    """Sets the share value of the files to 0 in the database
    :param files: list of absolute file paths
    :param dbool: twisted.enterprise.adbapi.ConnectionPool object
    """
    for f in files:
        yield dbpool.runQuery('update indexer set share=0 where filename=?', (f,))
    defer.returnValue('unshared') 

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def share(files, dbpool):
    """Sets the share value of the files to 1 in the database
    :param files: list of absolute file paths
    :param dbool: twisted.enterprise.adbapi.ConnectionPool object
    """
    for f in files:
        yield dbpool.runQuery('update indexer set share=1 where filename=?', (f,))
    defer.returnValue('shared') 

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def check_hash_present_in_resume(filename, dbpool):
    """Checks if hash value is present in the resume table
    :param filename: absolute filepath
    :param dbpool: twisted.enterprise.adbapi.ConnectionPool object
    """
    response = yield dbpool.runQuery('select filename from resume where filename=?', (filename,))
    if len(response) == 0:
        defer.returnValue(False)
    else:
        filename = response[0][0]
        if not os.path.exists(filename):
            yield remove_resume_entry(filename, dbpool)
            defer.returnValue(False)
        defer.returnValue(True) 

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def remove_resume_entry(filename, dbpool):
    """Removes file entry from the resume table
    :param filename: absolute filepath
    :param dbpool: twisted.enterprise.adbapi.ConnectionPool object
    """
    filename_response = yield dbpool.runQuery('select filename from indexer where filename=?', (filename,))
    filename = filename_response[0][0]
    # print 'everything deleted from indexer and resume'
    yield dbpool.runQuery('delete from resume where filename=?', (filename,))
    yield dbpool.runQuery('delete from indexer where filename=?', (filename,)) 

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def add_new_file_entry_resume(file_entry, dbpool):
    """Add file entry to the resume table
    :param filename: absolute filepath
    :param dbpool: twisted.enterprise.adbapi.ConnectionPool object
    """
    # print 'new entry added to resume table'
    # filename, checksum = file_entry[0], file_entry[3]
    # checksum = file_entry[3]
    filename = file_entry[0]
    yield dbpool.runQuery('insert into resume values (?)', (filename,))
    yield dbpool.runQuery('insert into indexer values (?,?,?,?,?,?,?)', (file_entry)) 

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def get_file(file_hash, dbpool):
    """Returns the corresponding file object
    :param file_hash: hash of the file
    :param dbpool: twisted.enterprise.adbapi.ConnectionPool object
    """
    file_query_response = yield dbpool.runQuery('select filename from indexer where hash=?', (file_hash,))
    # print file_query_response[0][0]
    defer.returnValue(open(file_query_response[0][0], 'rb')) 

# 需要導入模塊: from twisted.enterprise import adbapi [as 別名]
# 或者: from twisted.enterprise.adbapi import ConnectionPool [as 別名]
def get_piecehashes_of(file_hash, dbpool):
    """Returns concatenated hash of all chunks belonging to a particular file
    :param file_hash: hash of the file
    :param dbpool: twisted.enterprise.adbapi.ConnectionPool object
    """
    file_pieces_response = yield dbpool.runQuery('select piecehashes from indexer where hash=?', (file_hash,))
    defer.returnValue(file_pieces_response[0][0])