Python ColumnFamily.insert方法代码示例

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
def loadData():
  con = util.getConnection()
  cf = ColumnFamily(con, 'videos')
  tagCF = ColumnFamily(con, 'tag_videos_composite')
  movies = util.readCSV('data/movies')
  for movie in movies:
    title = movie[0]
    uploader = movie[1]
    runtime = int(movie[2]) #convert to match column validator
    tags = movie[3]
    rowKey = title+":"+uploader


    print "Inserting in videos: {}.".format(str(movie))
    row = \
      {
        'title':title,
        'user_name':uploader,
        'runtime_in_sec':runtime,
        'tags_csv': tags
      }

    cf.insert(rowKey, row)

    print 'inserting tags: {}'.format(tags)
    for tag in tags.split(','): 
      tagCF.insert( 
          tag.strip().lower(),        #row-key = tag name 
          { 
            (uploader, rowKey): title #(uploader,videoId)=title 
          } 
      );

  print 'finishished insertion.'    
  con.dispose()

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
class TestCustomTypes(unittest.TestCase):

    class IntString(types.CassandraType):

        @staticmethod
        def pack(intval):
            return str(intval)

        @staticmethod
        def unpack(strval):
            return int(strval)

    class IntString2(types.CassandraType):

        def __init__(self, *args, **kwargs):
            self.pack = lambda val: str(val)
            self.unpack = lambda val: int(val)

    def test_staticmethod_funcs(self):
        self.cf = ColumnFamily(pool, 'Standard1')
        self.cf.key_validation_class = TestCustomTypes.IntString()
        self.cf.insert(1234, {'col': 'val'})
        assert_equal(self.cf.get(1234), {'col': 'val'})

    def test_constructor_lambdas(self):
        self.cf = ColumnFamily(pool, 'Standard1')
        self.cf.key_validation_class = TestCustomTypes.IntString2()
        self.cf.insert(1234, {'col': 'val'})
        assert_equal(self.cf.get(1234), {'col': 'val'})

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
 def _update_analytics_start_time(self, start_times):
     try:
         col_family = ColumnFamily(self._pool, SYSTEM_OBJECT_TABLE)
         col_family.insert(SYSTEM_OBJECT_ANALYTICS, start_times)
     except Exception as e:
         self._logger.error("Exception: update_analytics_start_time "
             "Connection Failure %s" % e)

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
    def test_static_composite_slicing(self):
        cf = ColumnFamily(pool, 'StaticComposite')
        u1 = uuid.uuid1()
        u4 = uuid.uuid4()
        col0 = (0, 1, u1, u4, '', '', '')
        col1 = (1, 1, u1, u4, '', '', '')
        col2 = (1, 2, u1, u4, '', '', '')
        col3 = (1, 3, u1, u4, '', '', '')
        col4 = (2, 1, u1, u4, '', '', '')
        cf.insert('key2', {col0: '', col1: '', col2: '', col3: '', col4: ''})

        result = cf.get('key2', column_start=((1, True),), column_finish=((1, True),))
        assert_equal(result, {col1: '', col2: '', col3: ''})

        result = cf.get('key2', column_start=(1,), column_finish=((2, False), ))
        assert_equal(result, {col1: '', col2: '', col3: ''})

        result = cf.get('key2', column_start=((1, True),), column_finish=((2, False), ))
        assert_equal(result, {col1: '', col2: '', col3: ''})

        result = cf.get('key2', column_start=(1, ), column_finish=((2, False), ))
        assert_equal(result, {col1: '', col2: '', col3: ''})

        result = cf.get('key2', column_start=((0, False), ), column_finish=((2, False), ))
        assert_equal(result, {col1: '', col2: '', col3: ''})

        result = cf.get('key2', column_start=(1, 1), column_finish=(1, 3))
        assert_equal(result, {col1: '', col2: ''})

        result = cf.get('key2', column_start=(1, 1), column_finish=(1, (3, True)))
        assert_equal(result, {col1: '', col2: '', col3: ''})

        result = cf.get('key2', column_start=(1, (1, True)), column_finish=((2, False), ))
        assert_equal(result, {col1: '', col2: '', col3: ''})

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
def loadData():
  con = util.getConnection()
  cf = ColumnFamily(con, 'videos')
  tagCF = ColumnFamily(con, 'tag_videos_sup')
  movies = util.readCSV('data/movies')
  for movie in movies:
    title = movie[0]
    uploader = movie[1]
    runtime = int(movie[2]) #convert to match column validator
    tags = movie[3]
    rowKey = title+":"+uploader


    print "Inserting in videos: {}.".format(str(movie))
    cf.insert(
      rowKey, 
      {
        'title':title,
        'user_name':uploader,
        'runtime_in_sec':runtime,
        'tags_csv': tags
      })
    for tag in tags.split(','):
      print 'adding tag: {0} for movie: {1}'.format(tag, title)
      tagCF.insert(
          tag.strip().lower(),   # row-key = tag name
          {
            uploader: {          # level 1 nesting = uploader name
              rowKey: title      # level 2 nesting = videoId, value = title
             }
          }
      );

  print 'finishished insertion.'    
  con.dispose()

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
    def insert(self, data, return_id = False):
        """
        Creates a new entity to represent a model.

        :param data: Model object represented by a list of (field, value) pairs.
                     Each value is prepared for the insert operation.
        :param return_id: Value whether to return the id or key of newly created
                          entity.
        """

        pool = self.connection
        column_family_name = get_column_family()
        col_fam = CF(pool, column_family_name)

        col_fam_data = {}
        for field, value in data.iteritems():
            col_fam_data[field] = value

        key = data.get(pk_column)
        
        if not key:
            key = str(uuid4())
            
        try:
            col_fam.insert(key=key,
                           columns=col_fam_data,
                           write_consistency_level=self.connection.write_consistency_level)
        except Exception, e:
            print str(e)

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
def get_row_key_id(domain):
    counter_column, counter_lock = domain_counter_map[domain]
    
    ## acquire lock before getting value of 
    counter_lock.acquire()
    try:
        client = db_connection.get_client()
        cf = ColumnFamily(client, CONFIG_DOMAIN)
        
        ## get new key id
        id_key = cf.get(CONFIG_ROW, counter_column)[counter_column]
        
        ## increment value if not None
        if id_key:
            new_id_key = id_key + 1
            cf.insert(CONFIG_ROW, {counter_column: new_id_key}, write_consistency_level=ConsistencyLevel.ALL)
        
        return id_key
        
        """
        if id_key:
            str_id_key = str(id_key)
            str_id_key.zfill(MAX_PADDING_RANGE)
            return str_id_key
        else:
            return None
        """
        
    finally:
        ## release lock before returning from this function
        counter_lock.release()

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
class ClientCassandra():
    
    def __init__(self,keySpace):
        self.pool = ConnectionPool(keySpace, ['localhost:9160'])
        self.col_fam_page        = ColumnFamily(self.pool, 'Page')
        self.col_fam_publication = ColumnFamily(self.pool, 'Publication')
        self.col_fam_company     = ColumnFamily(self.pool, 'Company')
        self.col_fam_location    = ColumnFamily(self.pool, 'Location')
        self.col_fam_category    = ColumnFamily(self.pool, 'Category')
        
    #pycassaShell
    #SYSTEM_MANAGER.create_keyspace('BlwData', strategy_options={"replication_factor": "1"});
    #SYSTEM_MANAGER.create_column_family('BlwData', 'Page');
    #SYSTEM_MANAGER.create_column_family('BlwData', 'Publication');
    #SYSTEM_MANAGER.create_column_family('BlwData', 'Company');
    #SYSTEM_MANAGER.create_column_family('BlwData', 'Location');
    #SYSTEM_MANAGER.create_column_family('BlwData', 'Category');
        
    def insertPage(self,page):
        timestamp= self.col_fam_page.insert(page.getUrl(), page.toJson())
        print "sizeof category " + page.category.name + " is " + str(self.col_fam_category.get_count(page.category.name))
        self.col_fam_category.insert(page.category.name,{'url': page.getUrl()})
        print "sizeof category " + page.category.name + " is " + str(self.col_fam_category.get_count(page.category.name))
        return timestamp
        # should raise an exception for an immutable sequence
        #self.assertRaises(TypeError, random.shuffle, (1,2,3))
    def getPages(self,url,column):
        readData = self.col_fam_page .get(url,columns=[column])
        return readData
    
    def getCountCategory(self,category):
            return self.col_fam_category.get_count(category)

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
def loadData():
  con = util.getConnection()
  cf = ColumnFamily(con, 'videos_denorm')
  tagCF = ColumnFamily(con, 'tag_videos_composite')
  movies = util.readCSV('data/movies')
  for movie in movies:
    title = movie[0]
    uploader = movie[1]
    runtime = int(movie[2]) #convert to match column validator
    tags = movie[3]
    rowKey = title+":"+uploader


    print "Inserting in videos: {}.".format(str(movie))
    row = \
      {
        'title':title,
        'user_name':uploader,
        'runtime_in_sec':runtime,
      }

    for tag in tags.split(','):
      print 'adding tag: {0} for movie: {1}'.format(tag, title)
      row['tag:{}'.format(tag.strip().lower())] = tag.strip()

    print 'inserting denorm: {}'.format(row)
    cf.insert(rowKey, row)

  print 'finishished insertion.'    
  con.dispose()

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
    def test_static_composite_slicing(self):
        cf = ColumnFamily(pool, "StaticComposite")
        u1 = uuid.uuid1()
        u4 = uuid.uuid4()
        col0 = (0, 1, u1, u4, "", "", "")
        col1 = (1, 1, u1, u4, "", "", "")
        col2 = (1, 2, u1, u4, "", "", "")
        col3 = (1, 3, u1, u4, "", "", "")
        col4 = (2, 1, u1, u4, "", "", "")
        cf.insert("key2", {col0: "", col1: "", col2: "", col3: "", col4: ""})

        result = cf.get("key2", column_start=((1, True),), column_finish=((1, True),))
        assert_equal(result, {col1: "", col2: "", col3: ""})

        result = cf.get("key2", column_start=(1,), column_finish=((2, False),))
        assert_equal(result, {col1: "", col2: "", col3: ""})

        result = cf.get("key2", column_start=((1, True),), column_finish=((2, False),))
        assert_equal(result, {col1: "", col2: "", col3: ""})

        result = cf.get("key2", column_start=(1,), column_finish=((2, False),))
        assert_equal(result, {col1: "", col2: "", col3: ""})

        result = cf.get("key2", column_start=((0, False),), column_finish=((2, False),))
        assert_equal(result, {col1: "", col2: "", col3: ""})

        result = cf.get("key2", column_start=(1, 1), column_finish=(1, 3))
        assert_equal(result, {col1: "", col2: "", col3: ""})

        result = cf.get("key2", column_start=(1, 1), column_finish=(1, (3, True)))
        assert_equal(result, {col1: "", col2: "", col3: ""})

        result = cf.get("key2", column_start=(1, (1, True)), column_finish=((2, False),))
        assert_equal(result, {col1: "", col2: "", col3: ""})

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
    def store_file(self, filename, content, mtime=-1, compression_state=None,
        compressed_size=None):

        cf = ColumnFamily(self.pool, 'files')

        compression_state = compression_state or 'unknown'
        assert compression_state in ('none', 'unknown', 'gzip')

        sha1 = self.store_blob(filename, content)

        cols = {
            'version': 1,
            'sha1': sha1,
            'size': len(content),
            'mtime': mtime,
            'compression_state': compression_state,
        }

        if compressed_size is not None:
            cols['compressed_size'] = compressed_size

        cf.insert(filename, cols)

        indices = ColumnFamily(self.pool, 'simple_indices')
        indices.insert('files', {filename: ''})

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
    def test_single_component_composite(self):
        sys = SystemManager()
        sys.create_column_family(TEST_KS, 'SingleComposite',
                comparator_type=CompositeType(IntegerType()))

        cf = ColumnFamily(pool, 'SingleComposite')
        cf.insert('key', {(123456,): 'val'})
        assert_equal(cf.get('key'), {(123456,): 'val'})

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
def main(filename):
    data =  file(filename)
    # Set up the connection pool
    pool = ConnectionPool('tuitterdb',['localhost:9160'])
    
    # CF connections
    user_family = ColumnFamily(pool, 'user')
    tweet_family = ColumnFamily(pool, 'tweet')
    user_tweets_family = ColumnFamily(pool, 'userTweets')
    followers = ColumnFamily(pool, 'followers')
    followerTweets = ColumnFamily(pool, 'followsTweets')

    # Batch Definitions
    user_batch = user_family.batch(queue_size=1000)
    followers_batch = followers.batch(queue_size=500)
    user_tweets_batch = user_tweets_family.batch(queue_size = 500)
    followerTweets_batch = followerTweets.batch(queue_size = 500)
    
                           
    while True: # loop
        line = data.readline()
        if line == "": break # This isn't handled properly
        else:
            tweet = tweet_get(line)
            try:
                tweet_data = get_tweet_data(tweet)
                if check_user(tweet[u"from_user_id_str"]) == False: # check in script if user is there.
                    sender = get_sender(tweet)
                    user_batch.insert(sender.user_id,{'user_name':sender.user_name, 'screen_name':sender.from_user}) # create user entry for sender
                    user_tweets_batch.insert(sender.user_id,{line:''}) # insert the whole tweet into a userTweets column header

                if tweet[u"to_user"] is not None and check_user(tweet[u"to_user_id"]) == False:
                    to_user = get_to_user(tweet)
                    user_batch.insert(to_user.user_id,{'user_name':to_user.user_name, 'screen_name':to_user.from_user})
                    followers_batch.insert(to_user.user_id,{sender.user_id:'follower_id'}) 
                    followerTweets_batch.insert(to_user.user_id,{line:''}) # insert the whole tweet into a followeTweets column header for the to user.


                if u"entities" in tweet: # iterate over the users in mentions and add them to users and follows if necessary
                    if tweet[u"entities"][u"user_mentions"] != []:
                        user_mentions = get_mentions(tweet)
                        for obj in user_mentions:
                            if check_user(obj.user_id) == False:
                                user_batch.insert(obj.user_id,{'user_name':obj.user_name, 'screen_name':obj.from_user})
                            followers_batch.insert(obj.user_id,{'user_id':sender.user_id})
                            followerTweets_batch.insert(obj.user_id,{line:''}) # insert the whole tweet to a followerTweet entry for the mentioned user
                else:
                    continue

                tweet_family.insert(tweet_data.tweet_id,{'text':tweet_data.textbody,'user_id':sender.user_id,'timeanddate':tweet_data.timestamp})

            except Exception:
                err = sys.exc_info()
                print "Broken cos %s %s %s" % (err[0],err[1], traceback.print_tb(err[2])) #print the exception data with traceback and continue.
                continue

    # Pools Closed.
    pool.dispose()

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
    def test_retrieve_with_custom_composite(self):
        cf_std = ColumnFamily(pool, "CustomComposite1")
        cf_cust = ColumnFamily(pool, "CustomComposite1")
        cf_cust.column_name_class = CompositeType(TestCustomComposite.IntDateType(), TestCustomComposite.IntString())

        std_col = (20120312, "321")
        cust_col = (date(2012, 3, 12), 321)
        cf_std.insert("cust_insert_key_2", {std_col: "cust_insert_val_2"})
        assert_equal(cf_cust.get("cust_insert_key_2"), {cust_col: "cust_insert_val_2"})

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import insert [as 别名]
def main(filename):
    data =  file(filename)
    # Set up the connection pool
    pool = ConnectionPool('tuitterdb',['localhost:9160'])
    # CF connections
    user_family = ColumnFamily(pool, 'user')
    tweet_family = ColumnFamily(pool, 'tweet')
    user_tweets_family = ColumnFamily(pool, 'userTweets')
    #follows_tweets_family = ColumnFamily(pool, 'follows.tweets')
    followers = ColumnFamily(pool, 'followers')

    # Batch Definitions
    user_batch = user_family.batch(queue_size=1000)
    followers_batch = followers.batch(queue_size=500)
    user_tweets_batch = user_tweets_family.batch(queue_size = 500)
    
                           
    while True:
        line = data.readline()
        if line is None: break
        else:
            tweet = tweet_get(line)
            try:
                tweet_data = get_tweet_data(tweet)
                if check_user(tweet[u"from_user_id_str"]) == False:
                    sender = get_sender(tweet)
                    user_batch.insert(sender.user_id,{'user_name':sender.user_name, 'screen_name':sender.from_user})
                    user_tweets_batch.insert(sender.user_id,{tweet_data.tweet_id:tweet_data.timestamp})

                if tweet[u"to_user"] is not None and check_user(tweet[u"to_user_id"]) == False:
                    to_user = get_to_user(tweet)
                    user_batch.insert(to_user.user_id,{'user_name':to_user.user_name, 'screen_name':to_user.from_user})
                    followers_batch.insert(to_user.user_id,{'user_id':sender.user_id})


                if u"entities" in tweet:
                    if tweet[u"entities"][u"user_mentions"] != []:
                        user_mentions = get_mentions(tweet)
                        for obj in user_mentions:
                            user_batch.insert(obj.user_id,{'user_name':obj.user_name, 'screen_name':obj.from_user})
                            followers_batch.insert(obj.user_id,{'user_id':sender.user_id})
                else:
                    continue

                tweet_family.insert(tweet_data.tweet_id,{'text':tweet_data.textbody,'user_id':sender.user_id,'timeanddate':tweet_data.timestamp})

            except Exception:
                err = sys.exc_info()
                print "Broken cos %s %s %s" % (err[0],err[1], traceback.print_tb(err[2]))
                continue

    # Pools Closed.
    pool.dispose()

#if __name__ == "__main__":
    #unittest.main()