Python ColumnFamily.batch方法代码示例

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import batch [as 别名]
    def load_builders(self, o):
        cf = ColumnFamily(self._pool, 'builders')
        indices = ColumnFamily(self._pool, 'indices')

        batch = cf.batch()
        i_batch = indices.batch()

        for builder_id, params in o.items():
            cat = params['category']
            columns = {
                'category': cat,
                'master': unicode(params['master_id']),
                'name': params['name'],
            }

            batch.insert(builder_id, columns)
            i_batch.insert('builder_category_to_builder_ids',
                {cat: {builder_id: ''}})
            i_batch.insert('master_id_to_slave_ids', {columns['master']: {
                builder_id: ''}})

            if len(params['slaves']):
                i_batch.insert('builder_id_to_slave_ids', {builder_id: {
                    unicode(slave_id): '' for slave_id in params['slaves']}})

        batch.send()
        i_batch.send()

        return len(o)

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import batch [as 别名]
def main(filename):
    data =  file(filename)
    # Set up the connection pool
    pool = ConnectionPool('tuitterdb',['localhost:9160'])
    
    # CF connections
    user_family = ColumnFamily(pool, 'user')
    tweet_family = ColumnFamily(pool, 'tweet')
    user_tweets_family = ColumnFamily(pool, 'userTweets')
    followers = ColumnFamily(pool, 'followers')
    followerTweets = ColumnFamily(pool, 'followsTweets')

    # Batch Definitions
    user_batch = user_family.batch(queue_size=1000)
    followers_batch = followers.batch(queue_size=500)
    user_tweets_batch = user_tweets_family.batch(queue_size = 500)
    followerTweets_batch = followerTweets.batch(queue_size = 500)
    
                           
    while True: # loop
        line = data.readline()
        if line == "": break # This isn't handled properly
        else:
            tweet = tweet_get(line)
            try:
                tweet_data = get_tweet_data(tweet)
                if check_user(tweet[u"from_user_id_str"]) == False: # check in script if user is there.
                    sender = get_sender(tweet)
                    user_batch.insert(sender.user_id,{'user_name':sender.user_name, 'screen_name':sender.from_user}) # create user entry for sender
                    user_tweets_batch.insert(sender.user_id,{line:''}) # insert the whole tweet into a userTweets column header

                if tweet[u"to_user"] is not None and check_user(tweet[u"to_user_id"]) == False:
                    to_user = get_to_user(tweet)
                    user_batch.insert(to_user.user_id,{'user_name':to_user.user_name, 'screen_name':to_user.from_user})
                    followers_batch.insert(to_user.user_id,{sender.user_id:'follower_id'}) 
                    followerTweets_batch.insert(to_user.user_id,{line:''}) # insert the whole tweet into a followeTweets column header for the to user.


                if u"entities" in tweet: # iterate over the users in mentions and add them to users and follows if necessary
                    if tweet[u"entities"][u"user_mentions"] != []:
                        user_mentions = get_mentions(tweet)
                        for obj in user_mentions:
                            if check_user(obj.user_id) == False:
                                user_batch.insert(obj.user_id,{'user_name':obj.user_name, 'screen_name':obj.from_user})
                            followers_batch.insert(obj.user_id,{'user_id':sender.user_id})
                            followerTweets_batch.insert(obj.user_id,{line:''}) # insert the whole tweet to a followerTweet entry for the mentioned user
                else:
                    continue

                tweet_family.insert(tweet_data.tweet_id,{'text':tweet_data.textbody,'user_id':sender.user_id,'timeanddate':tweet_data.timestamp})

            except Exception:
                err = sys.exc_info()
                print "Broken cos %s %s %s" % (err[0],err[1], traceback.print_tb(err[2])) #print the exception data with traceback and continue.
                continue

    # Pools Closed.
    pool.dispose()

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import batch [as 别名]
def main(filename):
    data =  file(filename)
    # Set up the connection pool
    pool = ConnectionPool('tuitterdb',['localhost:9160'])
    # CF connections
    user_family = ColumnFamily(pool, 'user')
    tweet_family = ColumnFamily(pool, 'tweet')
    user_tweets_family = ColumnFamily(pool, 'userTweets')
    #follows_tweets_family = ColumnFamily(pool, 'follows.tweets')
    followers = ColumnFamily(pool, 'followers')

    # Batch Definitions
    user_batch = user_family.batch(queue_size=1000)
    followers_batch = followers.batch(queue_size=500)
    user_tweets_batch = user_tweets_family.batch(queue_size = 500)
    
                           
    while True:
        line = data.readline()
        if line is None: break
        else:
            tweet = tweet_get(line)
            try:
                tweet_data = get_tweet_data(tweet)
                if check_user(tweet[u"from_user_id_str"]) == False:
                    sender = get_sender(tweet)
                    user_batch.insert(sender.user_id,{'user_name':sender.user_name, 'screen_name':sender.from_user})
                    user_tweets_batch.insert(sender.user_id,{tweet_data.tweet_id:tweet_data.timestamp})

                if tweet[u"to_user"] is not None and check_user(tweet[u"to_user_id"]) == False:
                    to_user = get_to_user(tweet)
                    user_batch.insert(to_user.user_id,{'user_name':to_user.user_name, 'screen_name':to_user.from_user})
                    followers_batch.insert(to_user.user_id,{'user_id':sender.user_id})


                if u"entities" in tweet:
                    if tweet[u"entities"][u"user_mentions"] != []:
                        user_mentions = get_mentions(tweet)
                        for obj in user_mentions:
                            user_batch.insert(obj.user_id,{'user_name':obj.user_name, 'screen_name':obj.from_user})
                            followers_batch.insert(obj.user_id,{'user_id':sender.user_id})
                else:
                    continue

                tweet_family.insert(tweet_data.tweet_id,{'text':tweet_data.textbody,'user_id':sender.user_id,'timeanddate':tweet_data.timestamp})

            except Exception:
                err = sys.exc_info()
                print "Broken cos %s %s %s" % (err[0],err[1], traceback.print_tb(err[2]))
                continue

    # Pools Closed.
    pool.dispose()

#if __name__ == "__main__":
    #unittest.main()

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import batch [as 别名]
    def load_builds(self, o, builders):
        cf = ColumnFamily(self._pool, 'builds')
        batch = cf.batch()

        indices = ColumnFamily(self._pool, 'indices')
        i_batch = indices.batch()

        simple_indices = ColumnFamily(self._pool, 'simple_indices')
        si_batch = simple_indices.batch()

        # We defer counter inserts until then end because Python
        # increments are cheaper than Cassandra increments (hopefully).
        counters = {
            'builder_number': Counter(),
            'builder_duration': Counter(),
            'builder_number_by_day': {},
            'builder_duration_by_day': {},
            'builder_number_by_category': {},
            'builder_duration_by_category': {},
            'builder_number_by_day_and_category': {},
            'builder_duration_by_day_and_category': {},
        }

        existing_filenames = set(self._connection.filenames())

        for build in o:
            self._load_build(batch, i_batch, si_batch, counters, build,
                builders, existing_filenames)

        batch.send()
        i_batch.send()
        si_batch.send()

        cf = ColumnFamily(self._pool, 'counters')
        for builder, count in counters['builder_number'].items():
            cf.add('builder_number', builder, count)

        for builder, count in counters['builder_duration'].items():
            cf.add('builder_duration', builder, count)

        cf = ColumnFamily(self._pool, 'super_counters')
        del counters['builder_number']
        del counters['builder_duration']

        for key, super_columns in counters.items():
            for super_column, counts in super_columns.items():
                for column, count in counts.items():
                    cf.add(key, column, count, super_column)

        return len(o)

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import batch [as 别名]
    def truncate_log_metadata(self):
        for cf in ['build_timelines']:
            cf = ColumnFamily(self.pool, cf)
            cf.truncate()

        cf = ColumnFamily(self.pool, 'indices')
        for key in LOG_METADATA_INDICES:
            cf.remove(key)

        cf = ColumnFamily(self.pool, 'counters')
        for key in LOG_METADATA_COUNTERS:
            cf.remove(key)

        cf = ColumnFamily(self.pool, 'super_counters')
        for key in LOG_METADATA_SUPER_COUNTERS:
            cf.remove(key)

        cf = ColumnFamily(self.pool, 'builds')
        batch = cf.batch()
        # Remove log parsing state from builds.
        for key, cols in cf.get_range(columns=['log_parsing_version']):
            if 'log_parsing_version' not in cols:
                continue

            batch.remove(key, ['log_parsing_version'])

        batch.send()

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import batch [as 别名]
    def load_slaves(self, o):
        cf = ColumnFamily(self._pool, 'slaves')
        with cf.batch() as batch:
            for slave_id, name in o.items():
                batch.insert(slave_id, {'name': name})

        return len(o)

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import batch [as 别名]
    def load_masters(self, o):
        cf = ColumnFamily(self._pool, 'masters')

        with cf.batch() as batch:
            for master_id, info in o.items():
                batch.insert(master_id, info)

        return len(o)

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import batch [as 别名]
    def update(self, values):
        """
        Changes an entity that already exists in the database.

        :param values: A list of (field, new-value) pairs.
        """

        pool = self.connection
        column_family_name = get_column_family()
        col_fam = CF(pool, column_family_name)

        pk_column = get_pk_column()
    
        pk_index = -1
        fields = self.get_fields()

        for index in range(len(fields)):
            if fields[index].column == pk_column:
                pk_index = index
                break
        
        if pk_index == -1:
            raise DatabaseError('Invalid primary key column.')

        b = col_fam.batch(write_consistency_level=self.connection.write_consistency_level)
        row_count = 0
        for result in self.results_iter():
            row_count += 1
            key = result[pk_index]

            for k, v in values.items():
                b.insert(k, v)

        b.send()

        return row_count

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import batch [as 别名]
def add_data(opt):
    pool = ConnectionPool('CrashData',
                          ['localhost:9160'])

    col_fam = ColumnFamily(pool, 'CrashInfo2')
    col_fam.insert('7d625afa-ca2b-41e7-bcf3-e180d2140202',
                    {"useragent_locale": "en-US",
                    "AdapterVendorID": "0x10de",
                    "TotalVirtualMemory": "4294836224",
                    "BreakpadReserveAddress": "44826624",
                    "Theme": "classic/1.0",
                    "Version": "29.0a1",
                    "id": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}",
                    "BIOS_Manufacturer": "stuff",
                    "Vendor": "Mozilla",
                    "uuid": "7d625afa-ca2b-41e7-bcf3-e180d2140202",
                    "EMCheckCompatibility": "true",
                    "Throttleable": "1",
                    "throttle_rate": "100",
                    "AvailablePageFile": "14036480000",
                    "version": "29.0a1",
                    "AdapterDeviceID": "0x1080",
                    "ReleaseChannel": "nightly",
                    "submitted_timestamp": "2014-02-02T23:32:59.584636+00:00",
                    "buildid": "20140202030204",
                    "Notes": "AdapterVendorID: 0x10de, AdapterDeviceID: 0x1080, AdapterSubsysID: 15803842, AdapterDriverVersion: 9.18.13.3158\nD2D? D2D+ DWrite? DWrite+ D3D10 Layers? D3D10 Layers+ D3D10 Layers- D3D9 Layers? D3D9 Layers- ",
                    "CrashTime": "1391383937",
                    "Winsock_LSP": "MSAFD Tcpip [TCP/IP] : 2 : 1 : %SystemRoot%\\system32\\mswsock.dll \n MSAFD Tcpip [UDP/IP] : 2 : 2 :  \n MSAFD Tcpip [RAW/IP] : 2 : 3 : %SystemRoot%\\system32\\mswsock.dll \n MSAFD Tcpip [TCP/IPv6] : 2 : 1 :  \n MSAFD Tcpip [UDP/IPv6] : 2 : 2 : %SystemRoot%\\system32\\mswsock.dll \n MSAFD Tcpip [RAW/IPv6] : 2 : 3 :  \n RSVP TCPv6 Service Provider : 2 : 1 : %SystemRoot%\\system32\\mswsock.dll \n RSVP TCP Service Provider : 2 : 1 :  \n RSVP UDPv6 Service Provider : 2 : 2 : %SystemRoot%\\system32\\mswsock.dll \n RSVP UDP Service Provider : 2 : 2 : ",
                    "FramePoisonBase": "00000000f0de0000",
                    "AvailablePhysicalMemory": "5240811520",
                    "FramePoisonSize": "65536",
                    "BreakpadReserveSize": "37748736",
                    "StartupTime": "1391382356",
                    "Add-ons": "%7B972ce4c6-7e08-4474-a285-3208198ce6fd%7D:29.0a1",
                    "BuildID": "20140202030204",
                    "SecondsSinceLastCrash": "930758",
                    "ProductName": "Firefox",
                    "legacy_processing": "0",
                    "BlockedDllList": "",
                    "AvailableVirtualMemory": "3497549824",
                    "SystemMemoryUsePercentage": "38",
                    "ProductID": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}"})

    crash_column_family = ColumnFamily(pool, opt.column_family_counter)
    crash_batch = crash_column_family.batch(queue_size=100)

    # Insert buckets of data hourly for crashes

    crash_seed_signatures = [
        "FakeSignature1",
        "FakeSignature2",
        "FakeSignature3",
        "FakeSignature4",
        "FakeSignature5",
        "FakeSignature6",
        "FakeSignature7",
        "FakeSignature8",
        "FakeSignature9"
    ]

    now = datetime.datetime.now()
    for i in xrange(100):
        current_time = now + datetime.timedelta(seconds=i)

        row_bucket = "h-%d" % int(current_time.strftime("%H"))
        print "Adding row_bucket %s" % row_bucket
        column_bucket = int(current_time.strftime("%M"))
        print "Adding to column_bucket %s" % column_bucket

        crash_batch.insert(row_bucket, {column_bucket: 1})

        next_sig = random.randint(0, len(crash_seed_signatures)-1)
        #new_row_bucket = "%s-%s" % (row_bucket, crash_seed_signatures[next_sig])
        new_row_bucket = '{ "hour": "%d", "signature": "%s" }' % (
            int(current_time.strftime("%H")), crash_seed_signatures[next_sig]
        )
        print "Adding row_bucket %s, column_bucket %s" % (new_row_bucket, column_bucket)
        crash_batch.insert(new_row_bucket, {column_bucket: 1})
        print "Just put %s into cassandra." % crash_seed_signatures[next_sig]

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import batch [as 别名]
from pycassa.pool import ConnectionPool
from pycassa.columnfamily import ColumnFamily

pool = ConnectionPool("pykeyspace", ["localhost:9160"])
col_family = ColumnFamily(pool, "UserInfo")
col_family.insert("dosht2", {"email": "[email protected]", "name": "mostafa"})
# print col_family.get("dosht2", columns=["email"])['email']
print col_family.get("dosht2")
b = col_family.batch()
b.insert("dodo", {"email": "[email protected]"})
b.remove("dosht2", ["name"])
b.send()
print col_family.get("dosht2")
print col_family.multiget(["dosht", "dodo"])["dodo"]

from pycassa.types import *


class User(object):
    key = UTF8Type()  # name key is mandatory
    email = AsciiType()
    age = IntegerType()

    def __repr__(self):
        return "User(key: %s, email: %s, age: %s)" % (self.key, self.email, self.age)


from pycassa.columnfamilymap import ColumnFamilyMap

cfmap = ColumnFamilyMap(User, pool, "UserInfo")
user = User()

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import batch [as 别名]
import pycassa
from pycassa.pool import ConnectionPool
from pycassa.columnfamily import ColumnFamily

pool = ConnectionPool('ApplicationData',
                      ['localhost:9160'])
col_fam = ColumnFamily(pool, 'UserInfo')
col_fam.insert('Diego', {'email': '[email protected]'})

readData = col_fam.get('Diego', columns=['email'])

col_fam.remove('Diego', columns=['email'])

#batch

b = col_fam.batch(queue_size=10)

b.insert('John',
         {'email': '[email protected]',
          'state': 'IL',
          'gender': 'M'})

b.insert('Jane',
         {'email': '[email protected]',
          'state': 'CA'})

b.remove('John', ['gender'])
b.remove('Jane')
b.send()