Python Database.add_index方法代码示例

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
class BenchCodernityDB(BenchBase):
    
    ID_FIELD = "_id"
    
    def __init__(self, *args, **kwargs):
        super(BenchCodernityDB, self).__init__(*args, **kwargs)
    
    def create_database(self):
        self.db = Database(self.db_name)
        self.db.create()
        self.db.add_index(WithSmallNumberIndex(self.db.path, "small_number"))

    def delete_database(self):
        self.db.close()
        shutil.rmtree(self.db_name)
            
    def create(self, record):
        self.db.insert(record)
    
    def get(self, key):
        return self.db.get("id", key, with_doc=True)
        
    def query(self, **kwargs):
        key, val = kwargs.items()[0]
        return list(self.db.get_many(key, val, limit=-1, with_doc=True))

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
def main():
    db = Database('/tmp/tut_update')
    db.create()
    x_ind = WithXIndex(db.path, 'x')
    db.add_index(x_ind)

    # full examples so we had to add first the data
    # the same code as in previous step

    for x in xrange(100):
        db.insert(dict(x=x))

    for y in xrange(100):
        db.insert(dict(y=y))

    # end of insert part

    print db.count(db.all, 'x')

    for curr in db.all('x', with_doc=True):
        doc = curr['doc']
        if curr['key'] % 7 == 0:
            db.delete(doc)
        elif curr['key'] % 5 == 0:
            doc['updated'] = True
            db.update(doc)

    print db.count(db.all, 'x')

    for curr in db.all('x', with_doc=True):
        print curr

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
 def test_to_many_shards(self, tmpdir):
     db = Database(str(tmpdir) + '/db')
     db.create(with_id_index=False)
     # it's ok to use sharded directly there
     with pytest.raises(IndexPreconditionsException):
         db.add_index(ShardedUniqueHashIndex(db.path, 'id', sh_nums=300))
     with pytest.raises(IndexPreconditionsException):
         db.add_index(ShardedUniqueHashIndex(db.path, 'id', sh_nums=256))

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
 def __init__(self):
     db = Database('db')
     if db.exists():
         db.open()
     else:
         db.create()
         index = UrlIndex(db.path, 'urlidx')
         db.add_index(index)
     self._db = db

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
def main():
    db = Database("/tmp/tut5_2")
    db.create()
    x_ind = WithXIndex(db.path, "x")
    db.add_index(x_ind)

    for x in xrange(100):
        db.insert(dict(x=x, t=random.random()))

    print db.run("x", "avg", start=10, end=30)

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
    def test_compact_shards(self, tmpdir):
        db = Database(str(tmpdir) + '/db')
        db.create(with_id_index=False)
        db.add_index(ShardedUniqueHashIndex5(db.path, 'id'))

        for x in xrange(100):
            db.insert({'x': x})

        db.compact()
        assert db.count(db.all, 'id') == 100

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
    def test_insert_get(self, tmpdir, sh_nums):
        db = Database(str(tmpdir) + '/db')
        db.create(with_id_index=False)
        n = globals()['ShardedUniqueHashIndex%d' % sh_nums]
        db.add_index(n(db.path, 'id'))
        l = []
        for x in xrange(10000):
            l.append(db.insert(dict(x=x))['_id'])

        for curr in l:
            assert db.get('id', curr)['_id'] == curr

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
class cache :
	"""
		cache for word morphological analysis
	"""
	def __init__(self,):
		"""
		Create Analex Cache
		"""
		self.cache={'checkedWords':{},
			    'FreqWords':{'noun':{}, 'verb':{},'stopword':{}},
			};
		self.db = Database('/tmp/qalsadiCache')
		if not self.db.exists():
			self.db.create();
			x_ind = WithAIndex(self.db.path, 'a')
			self.db.add_index(x_ind)        
		else:
			self.db.open();

	def __del__(self):
		"""
		Delete instance and clear cache
		
		"""
		self.cache=None;
		self.db.close();

	def isAlreadyChecked(self, word):
		try:
			return bool(self.db.get('a', word))
		except: return False
		#~ except: return False;

	def getChecked(self, word):
		x = self.db.get('a', word, with_doc=True)
		y= x.get('doc',False);
		if y: return y.get('d',[])
		else: return []
	
	def addChecked(self, word, data):
		idata = {"a":word,'d':data}
		self.db.insert(idata)

	
	def existsCacheFreq(self, word, wordtype):
		return word in self.cache['FreqWords'];
	
	def getFreq(self, originalword, wordtype):
		return self.cache['FreqWords'][wordtype].get(originalword,0);
	
	def addFreq(self, original, wordtype, freq):
		self.cache['FreqWords'][wordtype][original]=freq;

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
def main():
    db = Database('/tmp/tut2')
    db.create()
    x_ind = WithXIndex(db.path, 'x')
    db.add_index(x_ind)

    for x in xrange(100):
        db.insert(dict(x=x))

    for y in xrange(100):
        db.insert(dict(y=y))

    print db.get('x', 10, with_doc=True)

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
def main():
    db = Database('/tmp/tut5_1')
    db.create()
    x_ind = WithXIndex(db.path, 'x')
    db.add_index(x_ind)

    for x in xrange(100):
        db.insert(dict(x=x, t=random.random()))

    l = []
    for curr in db.get_many('x', start=10, end=30, limit=-1, with_doc=True):
        l.append(curr['doc']['t'])
    print sum(l) / len(l)

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
def main():
    db = Database('/tmp/tut4')
    db.create()
    x_ind = WithXIndex(db.path, 'x')
    db.add_index(x_ind)

    for x in xrange(100):
        db.insert(dict(x=x))

    for y in xrange(100):
        db.insert(dict(y=y))

    print db.get('x', 10, with_doc=True)

    for curr in db.get_many('x', start=15, end=25, limit=-1, with_doc=True):
        print curr

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
def main():
    db = Database('db/tut2')
    if db.exists():
    	db.open()
    else:
    	db.create()
    x_ind = WithXIndex(db.path, 'y')
    db.add_index(x_ind)

    for x in xrange(100):
        db.insert(dict(x=x))

    for y in xrange(100):
        db.insert(dict(y=y))

    print db.get('x', 10, with_doc=True)

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
        super(TaskWithIntiator, self).__init__(*args, **kwargs)

    def make_key_value(self, data):
        if data['t'] == constants.TASK_TYPE_CODE and data['tag'] == 'TRAVEL':
            return md5(data['extra_params']['username']).digest(), None

    def make_key(self, key):
        return md5(key.username).digest()

class PermitTravelIndex(HashIndex):

    def __init__(self, *args, **kwargs):
        kwargs['key_format'] = 'I'
        super(PermitTravelIndex, self).__init__(*args, **kwargs)
    
    def make_key_value(self, data):
        if data['t'] == constants.TASK_TYPE_CODE and data['tag'] == 'PERMIT_TRAVEL':
            return 0, None
   
    def make_key(self, key):
        return 0

codernity_db = Database('db')
codernity_db.create()
codernity_db.add_index(TaskWithIntiator(codernity_db.path, 'task_with_initiator'))
codernity_db.add_index(PermitTravelIndex(codernity_db.path, 'permit_travel'))
from lite_task_flow.indexes import add_index
add_index(codernity_db)

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
class CodernityDB(BaseService):

    """A service providing a codernity db interface."""

    name = 'db'
    default_config = dict(db=dict(path=''), app=dict(dir=''))

    def __init__(self, app):
        super(CodernityDB, self).__init__(app)
        self.dbfile = os.path.join(self.app.config['app']['dir'],
                                   self.app.config['db']['path'])
        self.db = None
        self.uncommitted = dict()
        self.stop_event = Event()
        self.db = Database(self.dbfile)
        try:
            log.info('opening db', path=self.dbfile)
            self.db.open()
        except DatabasePathException:
            log.info('db does not exist, creating it', path=self.dbfile)
            self.db.create()
            self.db.add_index(MD5Index(self.dbfile, 'key'))

    def _run(self):
        self.stop_event.wait()

    def stop(self):
        # commit?
        log.info('closing db')
        if self.started:
            self.db.close()
            self.stop_event.set()

    def get(self, key):
        log.debug('getting entry', key=key)
        if key in self.uncommitted:
            if self.uncommitted[key] is None:
                raise KeyError("key not in db")
            return self.uncommitted[key]
        try:
            value = self.db.get('key', key, with_doc=True)['doc']['value']
        except RecordNotFound:
            raise KeyError("key not in db")
        return compress.decompress(value)

    def put(self, key, value):
        log.debug('putting entry', key=key, value=value)
        self.uncommitted[key] = value

    def commit(self):
        log.debug('committing', db=self)
        for k, v in self.uncommitted.items():
            if v is None:
                doc = self.db.get('key', k, with_doc=True)['doc']
                self.db.delete(doc)
            else:
                self.db.insert({'key': k, 'value': compress.compress(v)})
        self.uncommitted.clear()

    def delete(self, key):
        log.debug('deleting entry', key=key)
        self.uncommitted[key] = None

    def __contains__(self, key):
        try:
            self.get(key)
        except KeyError:
            return False
        return True

    def __eq__(self, other):
        return isinstance(other, self.__class__) and self.db == other.db

    def __repr__(self):
        return '<DB at %d uncommitted=%d>' % (id(self.db), len(self.uncommitted))

    def inc_refcount(self, key, value):
        self.put(key, value)

    def dec_refcount(self, key):
        pass

    def revert_refcount_changes(self, epoch):
        pass

    def commit_refcount_changes(self, epoch):
        pass

    def cleanup(self, epoch):
        pass

    def put_temporarily(self, key, value):
        self.inc_refcount(key, value)
        self.dec_refcount(key)

# 需要导入模块: from CodernityDB.database import Database [as 别名]
# 或者: from CodernityDB.database.Database import add_index [as 别名]
    custom_header = 'from CodernityDB.sharded_hash import ShardedHashIndex'

    def __init__(self, *args, **kwargs):
        kwargs['sh_nums'] = 10
        kwargs['key_format'] = 'I'
        kwargs['use_make_keys'] = True
        super(MySharded, self).__init__(*args, **kwargs)

    def make_key_value(self, data):
        return data['x'], None

    def calculate_shard(self, key):
        return key % self.sh_nums


y = 1500 * 'y'

db = Database('/tmp/shard')

db.create(with_id_index=False)
db.add_index(CustomIdSharded(db.path, 'id'))
db.add_index(MySharded(db.path, 'x'))


# it makes non sense to use sharding with such small number of records
for x in xrange(10 ** 4):
    db.insert({'x': x, 'y': y})


print db.get('x', random.randint(0, 10 ** 4))['_id']