Python ColumnFamily.get_range方法代码示例

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
class TestSuperColumnFamily:
    def setUp(self):
        credentials = {'username': 'jsmith', 'password': 'havebadpass'}
        self.pool = ConnectionPool(keyspace='Keyspace1', credentials=credentials)
        self.cf = ColumnFamily(self.pool, 'Super2')

    def tearDown(self):
        for key, columns in self.cf.get_range():
            self.cf.remove(key)

    def test_super(self):
        key = 'TestSuperColumnFamily.test_super'
        columns = {'1': {'sub1': 'val1', 'sub2': 'val2'}, '2': {'sub3': 'val3', 'sub4': 'val4'}}
        assert_raises(NotFoundException, self.cf.get, key)
        self.cf.insert(key, columns)
        assert_equal(self.cf.get(key), columns)
        assert_equal(self.cf.multiget([key]), {key: columns})
        assert_equal(list(self.cf.get_range(start=key, finish=key)), [(key, columns)])

    def test_super_column_argument(self):
        key = 'TestSuperColumnFamily.test_super_columns_argument'
        sub12 = {'sub1': 'val1', 'sub2': 'val2'}
        sub34 = {'sub3': 'val3', 'sub4': 'val4'}
        columns = {'1': sub12, '2': sub34}
        self.cf.insert(key, columns)
        assert_equal(self.cf.get(key, super_column='1'), sub12)
        assert_raises(NotFoundException, self.cf.get, key, super_column='3')
        assert_equal(self.cf.multiget([key], super_column='1'), {key: sub12})
        assert_equal(list(self.cf.get_range(start=key, finish=key, super_column='1')), [(key, sub12)])

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
class TestSuperColumnFamily:
    def setUp(self):
        credentials = {'username': 'jsmith', 'password': 'havebadpass'}
        self.client = connect_thread_local('Keyspace1', credentials=credentials)
        self.cf = ColumnFamily(self.client, 'Super2',
                               write_consistency_level=ConsistencyLevel.ONE,
                               buffer_size=2, timestamp=self.timestamp,
                               super=True)

        try:
            self.timestamp_n = int(self.cf.get('meta')['meta']['timestamp'])
        except NotFoundException:
            self.timestamp_n = 0
        self.clear()

    def tearDown(self):
        self.cf.insert('meta', {'meta': {'timestamp': str(self.timestamp_n)}})

    # Since the timestamp passed to Cassandra will be in the same second
    # with the default timestamp function, causing problems with removing
    # and inserting (Cassandra doesn't know which is later), we supply our own
    def timestamp(self):
        self.timestamp_n += 1
        return self.timestamp_n

    def clear(self):
        for key, columns in self.cf.get_range(include_timestamp=True):
            for subcolumns in columns.itervalues():
                for value, timestamp in subcolumns.itervalues():
                    self.timestamp_n = max(self.timestamp_n, timestamp)
            self.cf.remove(key)

    def test_super(self):
        key = 'TestSuperColumnFamily.test_super'
        columns = {'1': {'sub1': 'val1', 'sub2': 'val2'}, '2': {'sub3': 'val3', 'sub4': 'val4'}}
        assert_raises(NotFoundException, self.cf.get, key)
        self.cf.insert(key, columns)
        assert self.cf.get(key) == columns
        assert self.cf.multiget([key]) == {key: columns}
        assert list(self.cf.get_range(start=key, finish=key)) == [(key, columns)]

    def test_super_column_argument(self):
        key = 'TestSuperColumnFamily.test_super_columns_argument'
        sub12 = {'sub1': 'val1', 'sub2': 'val2'}
        sub34 = {'sub3': 'val3', 'sub4': 'val4'}
        columns = {'1': sub12, '2': sub34}
        self.cf.insert(key, columns)
        assert self.cf.get(key, super_column='1') == sub12
        assert_raises(NotFoundException, self.cf.get, key, super_column='3')
        assert self.cf.multiget([key], super_column='1') == {key: sub12}
        assert list(self.cf.get_range(start=key, finish=key, super_column='1')) == [(key, sub12)]

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
class TestSuperColumnFamilyMap:
    def setUp(self):
        credentials = {'username': 'jsmith', 'password': 'havebadpass'}
        self.pool = ConnectionPool(keyspace='Keyspace1', credentials=credentials)
        self.cf = ColumnFamily(self.pool, 'Super2')
        self.map = ColumnFamilyMap(TestUTF8, self.cf)

    def tearDown(self):
        for key, columns in self.cf.get_range():
            self.cf.remove(key)

    def instance(self, key, super_column):
        instance = TestUTF8()
        instance.key = key
        instance.super_column = super_column
        instance.strcol = '1'
        instance.intcol = 2
        instance.floatcol = 3.5
        instance.datetimecol = datetime.now().replace(microsecond=0)
        instance.intstrcol = 8
        instance.floatstrcol = 4.6
        instance.datetimestrcol = datetime.now().replace(microsecond=0)

        return instance

    def test_super(self):
        instance = self.instance('TestSuperColumnFamilyMap.test_super', 'super1')
        assert_raises(NotFoundException, self.map.get, instance.key)
        self.map.insert(instance)
        res = self.map.get(instance.key)[instance.super_column]
        assert_equal(res, instance)
        assert_equal(self.map.multiget([instance.key])[instance.key][instance.super_column], instance)
        assert_equal(list(self.map.get_range(start=instance.key, finish=instance.key)), [{instance.super_column: instance}])

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
class TestDefaultValidators(unittest.TestCase):
    def setUp(self):
        credentials = {"username": "jsmith", "password": "havebadpass"}
        self.pool = ConnectionPool(pool_size=5, keyspace="Keyspace1", credentials=credentials)
        self.cf_def_valid = ColumnFamily(self.pool, "DefaultValidator")

    def tearDown(self):
        for key, cols in self.cf_def_valid.get_range():
            self.cf_def_valid.remove(key)
        self.pool.dispose()

    def test_default_validated_columns(self):

        key = "key1"

        col_cf = {"aaaaaa": 1L}
        col_cm = {"subcol": TIME1}
        col_ncf = {"aaaaaa": TIME1}
        col_ncm = {"subcol": 1L}

        # Both of these inserts work, as cf allows
        #  longs and cm for 'subcol' allows TIMEUUIDs.
        self.cf_def_valid.insert(key, col_cf)
        self.cf_def_valid.insert(key, col_cm)
        assert self.cf_def_valid.get(key) == {"aaaaaa": 1L, "subcol": TIME1}

        assert_raises(TypeError, self.cf_def_valid.insert, key, col_ncf)
        assert_raises(TypeError, self.cf_def_valid.insert, key, col_ncm)

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
class TestBigInt(unittest.TestCase):

    @classmethod
    def setup_class(cls):
        sys = SystemManager()
        sys.create_column_family(TEST_KS, 'StdInteger', comparator_type=INT_TYPE)

    @classmethod
    def teardown_class(cls):
        sys = SystemManager()
        sys.drop_column_family(TEST_KS, 'StdInteger')

    def setUp(self):
        self.key = 'TestBigInt'
        self.cf = ColumnFamily(pool, 'StdInteger')

    def tearDown(self):
        self.cf.remove(self.key)

    def test_negative_integers(self):
        self.cf.insert(self.key, {-1: '-1'})
        self.cf.insert(self.key, {-12342390: '-12342390'})
        self.cf.insert(self.key, {-255: '-255'})
        self.cf.insert(self.key, {-256: '-256'})
        self.cf.insert(self.key, {-257: '-257'})
        for key, cols in self.cf.get_range():
            self.assertEquals(str(cols.keys()[0]), cols.values()[0])

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
class TestSuperColumnFamilyMap:
    def setUp(self):
        self.client = connect_thread_local()
        self.client.login('Keyspace1', {'username': 'jsmith', 'password': 'havebadpass'})
        self.cf = ColumnFamily(self.client, 'Keyspace1', 'Super2',
                               write_consistency_level=ConsistencyLevel.ONE,
                               timestamp=self.timestamp,
                               super=True)
        self.map = ColumnFamilyMap(TestUTF8, self.cf)
        try:
            self.timestamp_n = int(self.cf.get('meta')['meta']['timestamp'])
        except NotFoundException:
            self.timestamp_n = 0
        self.clear()

    def tearDown(self):
        self.cf.insert('meta', {'meta': {'timestamp': str(self.timestamp_n)}})

    # Since the timestamp passed to Cassandra will be in the same second
    # with the default timestamp function, causing problems with removing
    # and inserting (Cassandra doesn't know which is later), we supply our own
    def timestamp(self):
        self.timestamp_n += 1
        return self.timestamp_n

    def clear(self):
        for key, columns in self.cf.get_range(include_timestamp=True):
            for subcolumns in columns.itervalues():
                for value, timestamp in subcolumns.itervalues():
                    self.timestamp_n = max(self.timestamp_n, timestamp)
            self.cf.remove(key)

    def instance(self, key, super_column):
        instance = TestUTF8()
        instance.key = key
        instance.super_column = super_column
        instance.strcol = '1'
        instance.intcol = 2
        instance.floatcol = 3.5
        instance.datetimecol = datetime.now().replace(microsecond=0)
        instance.intstrcol = 8
        instance.floatstrcol = 4.6
        instance.datetimestrcol = datetime.now().replace(microsecond=0)

        return instance

    def test_super(self):
        instance = self.instance('TestSuperColumnFamilyMap.test_super', 'super1')
        assert_raises(NotFoundException, self.map.get, instance.key)
        self.map.insert(instance)
        assert self.map.get(instance.key)[instance.super_column] == instance
        assert self.map.multiget([instance.key])[instance.key][instance.super_column] == instance
        assert list(self.map.get_range(start=instance.key, finish=instance.key)) == [{instance.super_column: instance}]

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
 def exportdata(self,destination=None):
     ''' Method to export data to files '''            
     
     logging.info("inside export data method")
     #result={}       
     count = 0
     for keyspace in self.sysmanager.list_keyspaces():            
         if (keyspace != 'system'):  #check to skip the system database
             pool = ConnectionPool(keyspace,[self.address])
             columnfamilies = self.sysmanager.get_keyspace_column_families(keyspace)               
             #result[keyspace]={}
             result = {}
             # iterate through all the column family
             for columnfamilyname in columnfamilies.keys():
                 #result[keyspace][columnfamilyname]=[]
                 colfamily = ColumnFamily(pool,columnfamilyname)
                 cols = colfamily.get_range(column_reversed=True)
                 result['keyspace']=keyspace
                 result['columnfamily']= columnfamilyname
                 result['cols']=[]                    
                 for col in cols:                        
                     result['cols'].append(col)
                     count = count + 1
                     # check count if it 10000 then reset it flush result
                     if (count == 100000):
                         filename ="%s.out" % str(uuid1()).replace("-","")
                         filepath = "output/%s" % filename
                         file = open(filepath,"w")                            
                         pickle.dump(result, file, protocol=0)
                         file.close()                
                         #reset cols
                         print sys.getsizeof(result)#in mb     
                         result['cols']=[]                               
                         count = 0
                 
     if  count > 0:
             filename ="%s.out" % str(uuid1()).replace("-","")
             filepath = "output/%s" % filename
             file = open(filepath,"w")                            
             pickle.dump(result, file, protocol=0)
             file.close()                

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
 def tearDown(self):
     pool = ConnectionPool('PycassaTestKeyspace')
     cf = ColumnFamily(pool, 'Standard1')
     for key, cols in cf.get_range():
         cf.remove(key)

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
class TestAutoPacking:

    def setUp(self):
        credentials = {'username': 'jsmith', 'password': 'havebadpass'}
        self.client = connect_thread_local('Keyspace1', credentials=credentials)

        self.cf       = ColumnFamily(self.client, 'Standard2')

        self.cf_long  = ColumnFamily(self.client, 'StdLong')
        self.cf_int   = ColumnFamily(self.client, 'StdInteger')
        self.cf_time  = ColumnFamily(self.client, 'StdTimeUUID')
        self.cf_lex   = ColumnFamily(self.client, 'StdLexicalUUID')
        self.cf_ascii = ColumnFamily(self.client, 'StdAscii')
        self.cf_utf8  = ColumnFamily(self.client, 'StdUTF8')
        self.cf_bytes = ColumnFamily(self.client, 'StdBytes')

        self.cf_suplong  = ColumnFamily(self.client, 'SuperLong', super=True)
        self.cf_supint   = ColumnFamily(self.client, 'SuperInt', super=True)
        self.cf_suptime  = ColumnFamily(self.client, 'SuperTime', super=True)
        self.cf_suplex   = ColumnFamily(self.client, 'SuperLex', super=True)
        self.cf_supascii = ColumnFamily(self.client, 'SuperAscii', super=True)
        self.cf_suputf8  = ColumnFamily(self.client, 'SuperUTF8', super=True)
        self.cf_supbytes = ColumnFamily(self.client, 'SuperBytes', super=True)

        self.cf_suplong_sublong  = ColumnFamily(self.client, 'SuperLongSubLong', super=True)
        self.cf_suplong_subint   = ColumnFamily(self.client, 'SuperLongSubInt', super=True)
        self.cf_suplong_subtime  = ColumnFamily(self.client, 'SuperLongSubTime', super=True)
        self.cf_suplong_sublex   = ColumnFamily(self.client, 'SuperLongSubLex', super=True)
        self.cf_suplong_subascii = ColumnFamily(self.client, 'SuperLongSubAscii', super=True)
        self.cf_suplong_subutf8  = ColumnFamily(self.client, 'SuperLongSubUTF8', super=True)
        self.cf_suplong_subbytes = ColumnFamily(self.client, 'SuperLongSubBytes', super=True)

        self.cf_valid_long = ColumnFamily(self.client, 'ValidatorLong')
        self.cf_valid_int = ColumnFamily(self.client, 'ValidatorInt')
        self.cf_valid_time = ColumnFamily(self.client, 'ValidatorTime')
        self.cf_valid_lex = ColumnFamily(self.client, 'ValidatorLex')
        self.cf_valid_ascii = ColumnFamily(self.client, 'ValidatorAscii')
        self.cf_valid_utf8 = ColumnFamily(self.client, 'ValidatorUTF8')
        self.cf_valid_bytes = ColumnFamily(self.client, 'ValidatorBytes')

        self.cfs = [self.cf_long, self.cf_int, self.cf_time, self.cf_lex,
                    self.cf_ascii, self.cf_utf8, self.cf_bytes,
                    self.cf_suplong, self.cf_supint, self.cf_suptime,
                    self.cf_suplex, self.cf_supascii, self.cf_suputf8,
                    self.cf_supbytes,
                    self.cf_suplong_subint, self.cf_suplong_subint,
                    self.cf_suplong_subtime, self.cf_suplong_sublex,
                    self.cf_suplong_subascii, self.cf_suplong_subutf8,
                    self.cf_suplong_subbytes,
                    self.cf_valid_long, self.cf_valid_int, self.cf_valid_time,
                    self.cf_valid_lex, self.cf_valid_ascii, self.cf_valid_utf8,
                    self.cf_valid_bytes]

        try:
            self.timestamp_n = int(self.cf.get('meta')['timestamp'])
        except NotFoundException:
            self.timestamp_n = 0
        self.clear()

    def tearDown(self):
        self.cf.insert('meta', {'timestamp': str(self.timestamp_n)})

    def timestamp(self):
        self.timestamp_n += 1
        return self.timestamp_n

    def clear(self):
        for key, columns in self.cf.get_range(include_timestamp=True):
            for value, timestamp in columns.itervalues():
                self.timestamp_n = max(self.timestamp_n, timestamp)
            self.cf.remove(key)

        for cf in self.cfs:
            for key, columns in cf.get_range():
                cf.remove(key)

    def test_basic_inserts(self):

        long_col = {1111111111111111L: VALS[0]}
        int_col = {1: VALS[0]}
        time_col = {TIME1: VALS[0]}
        lex_col = {uuid.UUID(bytes='abc abc abc abcd'): VALS[0]}
        ascii_col = {'foo': VALS[0]}
        utf8_col = {u'\u0020': VALS[0]}
        bytes_col = {'bytes': VALS[0]}

        self.cf_long.insert(KEYS[0], long_col)
        self.cf_int.insert(KEYS[0], int_col)
        self.cf_time.insert(KEYS[0], time_col)
        self.cf_lex.insert(KEYS[0], lex_col)
        self.cf_ascii.insert(KEYS[0], ascii_col)
        self.cf_utf8.insert(KEYS[0], utf8_col)
        self.cf_bytes.insert(KEYS[0], bytes_col)

        assert self.cf_long.get(KEYS[0]) == long_col
        assert self.cf_int.get(KEYS[0]) == int_col
        assert self.cf_time.get(KEYS[0]) == time_col
        assert self.cf_lex.get(KEYS[0]) == lex_col
        assert self.cf_ascii.get(KEYS[0]) == ascii_col
        assert self.cf_utf8.get(KEYS[0]) == utf8_col
#.........这里部分代码省略.........

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
class TestColumnFamily:
    def setUp(self):
        self.client = connect()
        self.client.login('Keyspace1', {'username': 'jsmith', 'password': 'havebadpass'})
        self.cf = ColumnFamily(self.client, 'Keyspace1', 'Standard2',
                               write_consistency_level=ConsistencyLevel.ONE,
                               buffer_size=2, timestamp=self.timestamp,
                               dict_class=TestDict)
        try:
            self.timestamp_n = int(self.cf.get('meta')['timestamp'])
        except NotFoundException:
            self.timestamp_n = 0
        self.clear()

    def tearDown(self):
        self.cf.insert('meta', {'timestamp': str(self.timestamp_n)})

    # Since the timestamp passed to Cassandra will be in the same second
    # with the default timestamp function, causing problems with removing
    # and inserting (Cassandra doesn't know which is later), we supply our own
    def timestamp(self):
        self.timestamp_n += 1
        return self.timestamp_n

    def clear(self):
        for key, columns in self.cf.get_range(include_timestamp=True):
            for value, timestamp in columns.itervalues():
                self.timestamp_n = max(self.timestamp_n, timestamp)
            self.cf.remove(key)

    def test_empty(self):
        key = 'TestColumnFamily.test_empty'
        assert_raises(NotFoundException, self.cf.get, key)
        assert len(self.cf.multiget([key])) == 0
        for key, columns in self.cf.get_range():
            assert len(columns) == 0

    def test_insert_get(self):
        key = 'TestColumnFamily.test_insert_get'
        columns = {'1': 'val1', '2': 'val2'}
        assert_raises(NotFoundException, self.cf.get, key)
        self.cf.insert(key, columns)
        assert self.cf.get(key) == columns

    def test_insert_multiget(self):
        key1 = 'TestColumnFamily.test_insert_multiget1'
        columns1 = {'1': 'val1', '2': 'val2'}
        key2 = 'test_insert_multiget1'
        columns2 = {'3': 'val1', '4': 'val2'}
        missing_key = 'key3'

        self.cf.insert(key1, columns1)
        self.cf.insert(key2, columns2)
        rows = self.cf.multiget([key1, key2, missing_key])
        assert len(rows) == 2
        assert rows[key1] == columns1
        assert rows[key2] == columns2
        assert missing_key not in rows

    def test_insert_get_count(self):
        key = 'TestColumnFamily.test_insert_get_count'
        columns = {'1': 'val1', '2': 'val2'}
        self.cf.insert(key, columns)
        assert self.cf.get_count(key) == 2

    def test_insert_get_range(self):
        keys = ['TestColumnFamily.test_insert_get_range%s' % i for i in xrange(5)]
        columns = {'1': 'val1', '2': 'val2'}
        for key in keys:
            self.cf.insert(key, columns)

        rows = list(self.cf.get_range(start=keys[0], finish=keys[-1]))
        assert len(rows) == len(keys)
        for i, (k, c) in enumerate(rows):
            assert k == keys[i]
            assert c == columns

    def test_remove(self):
        key = 'TestColumnFamily.test_remove'
        columns = {'1': 'val1', '2': 'val2'}
        self.cf.insert(key, columns)

        self.cf.remove(key, columns=['2'])
        del columns['2']
        assert self.cf.get(key) == {'1': 'val1'}

        self.cf.remove(key)
        assert_raises(NotFoundException, self.cf.get, key)

    def test_dict_class(self):
        key = 'TestColumnFamily.test_dict_class'
        self.cf.insert(key, {'1': 'val1'})
        assert isinstance(self.cf.get(key), TestDict)

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
class TestColumnFamily(unittest.TestCase):

    def setUp(self):
        credentials = {'username': 'jsmith', 'password': 'havebadpass'}
        self.pool = ConnectionPool(keyspace='Keyspace1', credentials=credentials)
        self.cf = ColumnFamily(self.pool, 'Standard2', dict_class=TestDict)

    def tearDown(self):
        for key, columns in self.cf.get_range():
            self.cf.remove(key)

    def test_empty(self):
        key = 'TestColumnFamily.test_empty'
        assert_raises(NotFoundException, self.cf.get, key)
        assert_equal(len(self.cf.multiget([key])), 0)
        for key, columns in self.cf.get_range():
            assert_equal(len(columns), 0)

    def test_insert_get(self):
        key = 'TestColumnFamily.test_insert_get'
        columns = {'1': 'val1', '2': 'val2'}
        assert_raises(NotFoundException, self.cf.get, key)
        self.cf.insert(key, columns)
        assert_equal(self.cf.get(key), columns)

    def test_insert_multiget(self):
        key1 = 'TestColumnFamily.test_insert_multiget1'
        columns1 = {'1': 'val1', '2': 'val2'}
        key2 = 'test_insert_multiget1'
        columns2 = {'3': 'val1', '4': 'val2'}
        missing_key = 'key3'

        self.cf.insert(key1, columns1)
        self.cf.insert(key2, columns2)
        rows = self.cf.multiget([key1, key2, missing_key])
        assert_equal(len(rows), 2)
        assert_equal(rows[key1], columns1)
        assert_equal(rows[key2], columns2)
        assert_true(missing_key not in rows)

    def test_insert_get_count(self):
        key = 'TestColumnFamily.test_insert_get_count'
        columns = {'1': 'val1', '2': 'val2'}
        self.cf.insert(key, columns)
        assert_equal(self.cf.get_count(key), 2)

        assert_equal(self.cf.get_count(key, column_start='1'), 2)
        assert_equal(self.cf.get_count(key, column_finish='2'), 2)
        assert_equal(self.cf.get_count(key, column_start='1', column_finish='2'), 2)
        assert_equal(self.cf.get_count(key, column_start='1', column_finish='1'), 1)
        assert_equal(self.cf.get_count(key, columns=['1','2']), 2)
        assert_equal(self.cf.get_count(key, columns=['1']), 1)

    def test_insert_multiget_count(self):
        keys = ['TestColumnFamily.test_insert_multiget_count1',
               'TestColumnFamily.test_insert_multiget_count2',
               'TestColumnFamily.test_insert_multiget_count3']
        columns = {'1': 'val1', '2': 'val2'}
        for key in keys:
            self.cf.insert(key, columns)
        result = self.cf.multiget_count(keys)
        assert_equal(result[keys[0]], 2)
        assert_equal(result[keys[1]], 2)
        assert_equal(result[keys[2]], 2)

        result = self.cf.multiget_count(keys, column_start='1')
        assert_equal(len(result), 3)
        assert_equal(result[keys[0]], 2)

        result = self.cf.multiget_count(keys, column_finish='2')
        assert_equal(len(result), 3)
        assert_equal(result[keys[0]], 2)

        result = self.cf.multiget_count(keys, column_start='1', column_finish='2')
        assert_equal(len(result), 3)
        assert_equal(result[keys[0]], 2)

        result = self.cf.multiget_count(keys, column_start='1', column_finish='1')
        assert_equal(len(result), 3)
        assert_equal(result[keys[0]], 1)

        result = self.cf.multiget_count(keys, columns=['1','2'])
        assert_equal(len(result), 3)
        assert_equal(result[keys[0]], 2)

        result = self.cf.multiget_count(keys, columns=['1'])
        assert_equal(len(result), 3)
        assert_equal(result[keys[0]], 1)

    def test_insert_get_range(self):
        keys = ['TestColumnFamily.test_insert_get_range%s' % i for i in xrange(5)]
        columns = {'1': 'val1', '2': 'val2'}
        for key in keys:
            self.cf.insert(key, columns)

        rows = list(self.cf.get_range(start=keys[0], finish=keys[-1]))
        assert_equal(len(rows), len(keys))
        for i, (k, c) in enumerate(rows):
            assert_equal(k, keys[i])
            assert_equal(c, columns)
#.........这里部分代码省略.........

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
class CassandraImporter:
    def __init__(self):
        parser = argparse.ArgumentParser(description="Process some integers.")
        parser.add_argument(
            "-s",
            "--source",
            help="Generally the prod cassandra path, list of machines: \
                            localhost:9162 localhost:9163",
            nargs="*",
            required=True,
        )
        parser.add_argument(
            "-d",
            "--destination",
            help="Cassandra path where you need your data: \
                            localhost:9160 localhost:9161",
            nargs="*",
            required=True,
        )
        parser.add_argument("-ks", "--keyspace", help="The keyspace: myks", required=True)
        parser.add_argument("-cf", "--column_family", help="The Column family: mycf", required=True)
        parser.add_argument("-k", "--key", help="A specific key to be imported", required=False)
        parser.add_argument("-c", "--count", help="Total count of keys to be imported", required=False)
        parser.add_argument("-a", "--all", action="store_true", help="Get all. Not recommended!", required=False)
        args = vars(parser.parse_args())

        """Connection setting with cassandra
        The script is meant to sync data. So source and destination KS
        and CF shold be the same."""

        try:
            source_pool = ConnectionPool(args["keyspace"], args["source"])
            destination_pool = ConnectionPool(args["keyspace"], args["destination"])
            self.source_cf = ColumnFamily(source_pool, args["column_family"])
            self.source_cf.autopack_names = False
            self.source_cf.autopack_values = False
            self.source_cf.autopack_keys = False
            self.source_cf.default_validation_class = pycassa.types.UTF8Type()

            self.destination_cf = ColumnFamily(destination_pool, args["column_family"])
            self.destination_cf.autopack_names = False
            self.destination_cf.autopack_values = False
            self.destination_cf.autopack_keys = False
            self.destination_cf.default_validation_class = pycassa.types.UTF8Type()

        except Exception as e:
            print "ERROR: The keyspace or the column family does not exist or request is timing out!"
            sys.exit()

        # Optional data
        self.count = args["count"]
        if self.count:
            self.count = int(self.count)
        self.key = args["key"]
        self.all = args["all"]

    def importData(self):
        data = dict()
        # Get columns for a key
        if self.key:
            column_data = self.source_cf.get(self.key)
            data[self.key] = column_data

        # Get last x keys and their columns
        elif self.count:
            counter = 0
            error_count = 0
            for value in self.source_cf.get_range(column_count=0, filter_empty=False):
                if counter < self.count:
                    try:
                        column_data = self.source_cf.get(value[0], column_count=100)
                        data[value[0]] = column_data
                        counter += 1
                    except NotFoundException:
                        # Ignore keys with empty columns
                        pass
                    except Exception:
                        error_count += 1
                        if error_count > 10:
                            # Write the read data
                            self.insertData(data)
                            print "ERROR: Remote cassandra is too slow to read, exiting after writing..."
                            sys.exit()

                        # Use this to throttle reads from cassandra
                        time.sleep(0.2)
                else:
                    break

        # Get All, Not recommended
        elif self.all:
            for value in self.source_cf.get_range(column_count=0, filter_empty=False):
                column_data = self.source_cf.get(value[0])
                key = value[0]
                data[key] = column_data
        else:
            print "Please pass -c or -k or -a arguments!"

        return data

#.........这里部分代码省略.........

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
class TestTimeUUIDs(unittest.TestCase):
    def setUp(self):
        credentials = {"username": "jsmith", "password": "havebadpass"}
        self.pool = ConnectionPool(pool_size=5, keyspace="Keyspace1", credentials=credentials)
        self.cf_time = ColumnFamily(self.pool, "StdTimeUUID")

    def tearDown(self):
        for key, cols in self.cf_time.get_range():
            self.cf_time.remove(key)
        self.pool.dispose()

    def test_datetime_to_uuid(self):

        key = "key1"

        timeline = []

        timeline.append(datetime.now())
        time1 = uuid1()
        col1 = {time1: "0"}
        self.cf_time.insert(key, col1)
        time.sleep(1)

        timeline.append(datetime.now())
        time2 = uuid1()
        col2 = {time2: "1"}
        self.cf_time.insert(key, col2)
        time.sleep(1)

        timeline.append(datetime.now())

        cols = {time1: "0", time2: "1"}

        assert_equal(self.cf_time.get(key, column_start=timeline[0]), cols)
        assert_equal(self.cf_time.get(key, column_finish=timeline[2]), cols)
        assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols)
        assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols)
        assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[1]), col1)
        assert_equal(self.cf_time.get(key, column_start=timeline[1], column_finish=timeline[2]), col2)

    def test_time_to_uuid(self):

        key = "key1"

        timeline = []

        timeline.append(time.time())
        time1 = uuid1()
        col1 = {time1: "0"}
        self.cf_time.insert(key, col1)
        time.sleep(0.1)

        timeline.append(time.time())
        time2 = uuid1()
        col2 = {time2: "1"}
        self.cf_time.insert(key, col2)
        time.sleep(0.1)

        timeline.append(time.time())

        cols = {time1: "0", time2: "1"}

        assert_equal(self.cf_time.get(key, column_start=timeline[0]), cols)
        assert_equal(self.cf_time.get(key, column_finish=timeline[2]), cols)
        assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols)
        assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[2]), cols)
        assert_equal(self.cf_time.get(key, column_start=timeline[0], column_finish=timeline[1]), col1)
        assert_equal(self.cf_time.get(key, column_start=timeline[1], column_finish=timeline[2]), col2)

    def test_auto_time_to_uuid1(self):

        key = "key"

        t = time.time()
        col = {t: "foo"}
        self.cf_time.insert(key, col)
        uuid_res = self.cf_time.get(key).keys()[0]
        timestamp = convert_uuid_to_time(uuid_res)
        assert_almost_equal(timestamp, t, places=3)

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
 count = 0
 try:
   os.mkdir('output')
 except Exception, e:
   pass 
 for keyspace in self.sysmanager.list_keyspaces():            
   if (keyspace != 'system'):  #check to skip the system database
     pool = ConnectionPool(keyspace,[self.address])
     columnfamilies = self.sysmanager.get_keyspace_column_families(keyspace)
     result = {}
   relt['cols']=[]
     # iterate through all the column family
     for columnfamilyname in columnfamilies.keys():
       #result[keyspace][columnfamilyname]=[]
       colfamily = ColumnFamily(pool,columnfamilyname)
       cols = colfamily.get_range(column_reversed=True)
       result['keyspace']=keyspace
       result['columnfamily']= columnfamilyname
       for col in cols:                        
         result['cols'].append(col)
         count = count + 1
         # check count if it 10000 then reset it flush result
         if (count == 100000):
           filename ="%s.out" % str(uuid1()).replace("-","")
           filepath = "output/%s" % filename
           file = open(filepath,"w")                            
           pickle.dump(result, file, protocol=-1)
           file.close()                
           #reset cols
           print sys.getsizeof(result)#in mb     
           result['cols']=[]                               

# 需要导入模块: from pycassa import ColumnFamily [as 别名]
# 或者: from pycassa.ColumnFamily import get_range [as 别名]
class TestMutator(unittest.TestCase):
    def setUp(self):
        credentials = {"username": "jsmith", "password": "havebadpass"}
        self.pool = ConnectionPool(keyspace="Keyspace1", credentials=credentials)
        self.cf = ColumnFamily(self.pool, "Standard2")
        self.scf = ColumnFamily(self.pool, "Super1")

    def tearDown(self):
        for key, cols in self.cf.get_range():
            self.cf.remove(key)
        for key, cols in self.scf.get_range():
            self.scf.remove(key)

    def test_insert(self):
        batch = self.cf.batch()
        for key, cols in ROWS.iteritems():
            batch.insert(key, cols)
        batch.send()
        for key, cols in ROWS.items():
            assert self.cf.get(key) == cols

    def test_insert_supercolumns(self):
        batch = self.scf.batch()
        batch.insert("one", ROWS)
        batch.insert("two", ROWS)
        batch.insert("three", ROWS)
        batch.send()
        assert self.scf.get("one") == ROWS
        assert self.scf.get("two") == ROWS
        assert self.scf.get("three") == ROWS

    def test_queue_size(self):
        batch = self.cf.batch(queue_size=2)
        batch.insert("1", ROWS["1"])
        batch.insert("2", ROWS["2"])
        batch.insert("3", ROWS["3"])
        assert self.cf.get("1") == ROWS["1"]
        assert_raises(NotFoundException, self.cf.get, "3")
        batch.send()
        for key, cols in ROWS.items():
            assert self.cf.get(key) == cols

    def test_remove_key(self):
        batch = self.cf.batch()
        batch.insert("1", ROWS["1"])
        batch.remove("1")
        batch.send()
        assert_raises(NotFoundException, self.cf.get, "1")

    def test_remove_columns(self):
        batch = self.cf.batch()
        batch.insert("1", {"a": "123", "b": "123"})
        batch.remove("1", ["a"])
        batch.send()
        assert self.cf.get("1") == {"b": "123"}

    def test_remove_supercolumns(self):
        batch = self.scf.batch()
        batch.insert("one", ROWS)
        batch.insert("two", ROWS)
        batch.insert("three", ROWS)
        batch.remove("two", ["b"], "2")
        batch.send()
        assert self.scf.get("one") == ROWS
        assert self.scf.get("two")["2"] == {"a": "234"}
        assert self.scf.get("three") == ROWS

    def test_chained(self):
        batch = self.cf.batch()
        batch.insert("1", ROWS["1"]).insert("2", ROWS["2"]).insert("3", ROWS["3"]).send()
        assert self.cf.get("1") == ROWS["1"]
        assert self.cf.get("2") == ROWS["2"]
        assert self.cf.get("3") == ROWS["3"]

    def test_contextmgr(self):
        if sys.version_info < (2, 5):
            raise SkipTest("No context managers in Python < 2.5")
        exec """with self.cf.batch(queue_size=2) as b:
    b.insert('1', ROWS['1'])
    b.insert('2', ROWS['2'])
    b.insert('3', ROWS['3'])
assert self.cf.get('3') == ROWS['3']"""

    def test_multi_column_family(self):
        batch = batch_mod.Mutator(self.pool)
        cf2 = self.cf
        batch.insert(self.cf, "1", ROWS["1"])
        batch.insert(self.cf, "2", ROWS["2"])
        batch.remove(cf2, "1", ROWS["1"])
        batch.send()
        assert self.cf.get("2") == ROWS["2"]
        assert_raises(NotFoundException, self.cf.get, "1")