Python ColumnFamily.get_range方法代码示例

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
def columnfamily_dump(host, port, keyspace, columnfamily, columns, limit, outfile, header, delimiter):
    pool = ConnectionPool(keyspace, ['{}:{}'.format(host, port)], timeout=None)
    col_fam = ColumnFamily(pool, columnfamily)

    if columns:
        keys = set(columns.split(u','))
    else:
        rows = col_fam.get_range(row_count=limit)
        keys = set(key for key in itertools.chain.from_iterable(row[1].iterkeys() for row in rows))

    keys.add(u'{}_id'.format(columnfamily))

    writer = csv.DictWriter(outfile, keys, extrasaction=u'ignore', delimiter=delimiter)

    if header:
        writer.writeheader()

    rows = col_fam.get_range(columns=keys, row_count=limit)
    row_buffer_count = 0
    csv_rows = []
    for (id, data) in rows:
        d = {u'{}_id'.format(columnfamily): id}
        d.update(data)
        csv_rows.append(d)
        row_buffer_count += 1

        if row_buffer_count >= col_fam.buffer_size:
            writer.writerows(csv_rows)
            csv_rows = []
            row_buffer_count = 0
    else:
        writer.writerows(csv_rows)

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
	def colum_family_content(self,machine_id,keyspace_name,column_family_name):
		"""Returns content of column family of given keyspace """
		print "->>>in column family content function"
		pool = ConnectionPool(keyspace_name, [machine_id])
		col_fam = ColumnFamily(pool, column_family_name)
		result = col_fam.get_range(start='', finish='')
		return result

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
class CassandraRepository(object):
    def __init__(self, keyspace, column_family_name):
        self.pool = ConnectionPool(keyspace,
                                   cassandra_settings.NODE_POOL)

        self.cf = ColumnFamily(self.pool, column_family_name)

        self.batch = {}

    def add_batch(self, batch, start_time=None):
        """
        :param batch:
        """

        self.cf.batch_insert(batch)
        if start_time is not None:
            print 'time to  insert batch: %s ms' % (int(time.time() * 1000) - start_time)


    def get(self, timestamp):
        return self.cf.get(str(timestamp))

    def get_range(self, start, end):
        return list(self.cf.get_range(start=str(start), finish=str(end)))

    def close(self):
        self.sys.close()

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
    def truncate_log_metadata(self):
        for cf in ['build_timelines']:
            cf = ColumnFamily(self.pool, cf)
            cf.truncate()

        cf = ColumnFamily(self.pool, 'indices')
        for key in LOG_METADATA_INDICES:
            cf.remove(key)

        cf = ColumnFamily(self.pool, 'counters')
        for key in LOG_METADATA_COUNTERS:
            cf.remove(key)

        cf = ColumnFamily(self.pool, 'super_counters')
        for key in LOG_METADATA_SUPER_COUNTERS:
            cf.remove(key)

        cf = ColumnFamily(self.pool, 'builds')
        batch = cf.batch()
        # Remove log parsing state from builds.
        for key, cols in cf.get_range(columns=['log_parsing_version']):
            if 'log_parsing_version' not in cols:
                continue

            batch.remove(key, ['log_parsing_version'])

        batch.send()

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
    def slaves(self):
        """Obtain basic metadata about all slaves."""

        cf = ColumnFamily(self.pool, 'slaves')

        for key, cols in cf.get_range(columns=['name']):
            yield key, cols['name']

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
class TestBigInt(unittest.TestCase):

    @classmethod
    def setup_class(cls):
        sys = SystemManager()
        sys.create_column_family(TEST_KS, 'StdInteger', comparator_type=IntegerType())

    @classmethod
    def teardown_class(cls):
        sys = SystemManager()
        sys.drop_column_family(TEST_KS, 'StdInteger')

    def setUp(self):
        self.key = 'TestBigInt'
        self.cf = ColumnFamily(pool, 'StdInteger')

    def tearDown(self):
        self.cf.remove(self.key)

    def test_negative_integers(self):
        self.cf.insert(self.key, {-1: '-1'})
        self.cf.insert(self.key, {-12342390: '-12342390'})
        self.cf.insert(self.key, {-255: '-255'})
        self.cf.insert(self.key, {-256: '-256'})
        self.cf.insert(self.key, {-257: '-257'})
        for key, cols in self.cf.get_range():
            self.assertEquals(str(cols.keys()[0]), cols.values()[0])

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
 def colum_family_content(self, keyspace_name, column_family_name):
     """Returns content of column family of given keyspace """
     keyspace.cf_result = []
     keyspace.error = "Unknown error : May be one of node in your cluster is down please check?"
     if not self.keyspace_contains(keyspace.local_system, keyspace_name, column_family_name):
         keyspace.error = "Desired Keyspace,Column Family pair could not be found."
         return False
     try:
         pool = ConnectionPool(keyspace=keyspace_name, server_list=keyspace.server_ips, prefill=False)
     except Exception as e:
         print e
         return False
     try:
         col_fam = ColumnFamily(pool, column_family_name)
     except Exception as e:
         print e
         return False
     result = []
     try:
         tmp_result = col_fam.get_range(start='', finish='', buffer_size=10)
         for i in tmp_result:
             result.append(i)
     except Exception as e:
         print e
         return False
     keyspace.cf_result = result
     keyspace.tempks = keyspace_name
     keyspace.tempcf = column_family_name
     print result
     return result

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
def query(pool):
    cf_logs = ColumnFamily(pool, CF_LOGS)
    row_key = ymd_from_epoch()
    try:
        cf_logs.get(row_key, column_count=0)
    except NotFoundException:
        # FIXME: this is extremely inefficient!
        row_key = cf_logs.get_range().next()[0]

    logging.info("-" * 120)  # ------------------------------
    logging.info("Querying for key %s", row_key)
    logging.info("-" * 120)  # ------------------------------
    count = 20
    for k, v in cf_logs.get(row_key, column_reversed=True).iteritems():  # @UnusedVariable
        logging.info(v)
        count -= 1
        if count == 0:
            break
    del cf_logs

    logging.info("-" * 120)  # ------------------------------
    cf_logs_by_app = ColumnFamily(pool, CF_LOGS_BY_APP)
    row_key = EXAMPLE_APPS[0]
    logging.info("Querying for key %s", row_key)
    logging.info("-" * 120)  # ------------------------------
    count = 20
    for k, v in cf_logs_by_app.get(row_key, column_reversed=True).iteritems():  # @UnusedVariable
        logging.info(v)
        count -= 1
        if count == 0:
            break
    del cf_logs_by_app

    logging.info("-" * 120)  # ------------------------------
    cf_logs_by_host = ColumnFamily(pool, CF_LOGS_BY_HOST)
    row_key = EXAMPLE_HOSTS[0]
    logging.info("Querying for key %s", row_key)
    logging.info("-" * 120)  # ------------------------------
    count = 20
    for k, v in cf_logs_by_host.get(row_key, column_reversed=True).iteritems():  # @UnusedVariable
        logging.info(v)
        count -= 1
        if count == 0:
            break
    del cf_logs_by_host

    logging.info("-" * 120)  # ------------------------------
    cf_logs_by_severity = ColumnFamily(pool, CF_LOGS_BY_SEVERITY)
    row_key = "WARN"
    logging.info("Querying for key %s", row_key)
    logging.info("-" * 120)  # ------------------------------
    count = 20
    for k, v in cf_logs_by_severity.get(row_key, column_reversed=True).iteritems():  # @UnusedVariable
        logging.info(v)
        count -= 1
        if count == 0:
            break
    del cf_logs_by_severity

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
def query(pool):
    logging.info("-" * 120) # ------------------------------
    logging.info("-" * 120) # ------------------------------
    cf_logs = ColumnFamily(pool, CF_LOGS)
    for obj in cf_logs.get_range(): #@UnusedVariable
        print "Key: {0}".format(obj[0])
        # print dir(obj[1])
        for k, v in obj[1].iteritems():
            print "    {0} -> {1}".format(k, v)

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
    def simple_select(self, columnfamily, *args):
        slice = ['', '', self.max_rows]
        key = None
    
        if args and args[1]:
            if ':' not in args[1]:
                key = args[1]
            for i, part in enumerate(args[1].split(':', 2)):
                slice[i] = part

        try:
            cf = ColumnFamily(self.pool, columnfamily)
        except NotFoundException:
            return super(CCli, self).default(' '.join([columnfamily] + list(args)))

        if key:
            pt = PrettyTable()
            pt.field_names = ['Key', key]
            pt.align["Key"] = "l"
            pt.align[key] = 'r'

            for k, v in cf.get(key).items():
                pt.add_row([k, (v[:self.max_data_size - 3] + '...' if self.max_data_size and len(v) > self.max_data_size else v)])

            print pt.get_string(sortby='Key')
            return

        data = dict(cf.get_range(start=slice[0], finish=slice[1], row_count=int(slice[2])))

        columns = []
        for key, row in data.items():
            columns.extend(row.keys())
        columns = list(set(columns))
        columns.sort()

        pt = PrettyTable()
        pt.field_names = ['Key / Column'] + columns
        pt.align["Key / Column"] = "l"
        for column in columns:
            pt.align[column] = "r"

        for key, row in data.items():
            prow = [key]
            for column in columns:
                value = row.get(column, '---')
                if len(value) > self.max_data_size:
                    value = value[:self.max_data_size - 3] + '...'
                    
                prow.append(value)
            pt.add_row(prow)

        print pt.get_string(sortby='Key / Column')

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
	def colum_family_content(self,machine_id,keyspace_name,column_family_name):
		"""Returns content of column family of given keyspace """
		if (self.keyspace_contains(keyspace_name,column_family_name) == False):
			print "Error : Keyspace:column family could not be found."
			return False
		pool = ConnectionPool(keyspace = keyspace_name, server_list = keyspace.server_ips, prefill = False)
		col_fam = ColumnFamily(pool, column_family_name)
		tmp_result = col_fam.get_range(start='', finish='',row_count=5,buffer_size=10)
		result = []			
		for i in tmp_result:
			result.append(i)
		#print sum(1 for _ in result)			# for count
		return result

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
class Processor(Llama):
    def __init__(self, client, qname):
        super(Processor, self).__init__(client, qname)
        self.pool = ConnectionPool('processing_llama_Processor')
        self.trends = ColumnFamily(self.pool, 'Trend')

	def get_sleep_time():
		return 60

    def do_message(self, message):
        if not isinstance(message, tuple) or len(message) != 4:
			return
        woeid, as_of, trend_name, query = message
        try:
          trend = self.trends.get(trend_name, super_column=woeid)
          trend['lastseen'] = as_of
          trend['number_seen'] += 1
          trend = { woeid: trend }
          self.trends.insert(trend_name, trend)
        except ttypes.NotFoundException:
		  self.trends.insert(trend_name, { woeid: { 'firstseen': as_of, 'lastseen': as_of, 'number_seen': 1}})
		  self.trends.insert(trend_name, {'data': { 'query': query, 'tracking': "False" }})

    def has_many_woeids(self, x):
        key, values = x
        return len(values) > 2 # one woeid plus general data

    def do_action(self):
      try:
        for trend_name, country_specifics in filter(self.has_many_woeids, self.trends.get_range()):
          if country_specifics['data']['tracking'] == "False":
            self.track_trend(trend_name, country_specifics['data']['query'], filter(lambda x: x != "data", country_specifics.keys()))
      except:
		exc, value, exctraceback = sys.exc_info()
		logging.error("Error in processing_llama.processor.Processor.do_action:\n" + traceback.format_exc())
		traceback.print_tb(exctraceback)
        
    def track_trend(self, trend_name, query, woeids):
      logging.info("Tracking %s from %s" % (trend_name, woeids))
      self.mark_tracking(trend_name)
      self.publish((trend_name, query, woeids), "trend_to_track")

    def mark_tracking(self, trend_name):
      try:
          trend = self.trends.get(trend_name, super_column='data')
          trend['tracking'] = "True"
          self.trends.insert(trend_name, { 'data': trend })
      except ttypes.NotFoundException:
          pass

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
    def add_new_post(self, thread_id, name, content):
        post = ColumnFamily(self.conn, thread_id)

        ret = list(post.get_range())
        if len(ret) > 1000:
            return False

        key = len(ret) + 1
        key = '%s' % key

        dt = datetime.datetime.today()
        str_dt = dt.strftime('%Y-%m-%d %H:%M:%S')
        post.insert(key, {'name': name, 'content': content, 'post_time': str_dt})

        return True

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
def do_query(opt, start, finish):
    pool = ConnectionPool('CrashData',
                          ['localhost:9160'])

    cassandra = ColumnFamily(pool, opt.column_family_counter)
    graph = Pyasciigraph()
    for key, contents in cassandra.get_range(column_count=60,filter_empty=False):
        # get_range returns:
        # key: OrderedDict() [made of: (minute, count of instances) tuples]
        things = ''
        try:
            jsonkey = json.loads(key)
            things =  jsonkey.items()
        except:
            continue
        for line in graph.graph(str(things), sorted(contents.items())):
            print(line)

# 需要导入模块: from pycassa.columnfamily import ColumnFamily [as 别名]
# 或者: from pycassa.columnfamily.ColumnFamily import get_range [as 别名]
    def create_new_thread(self, thread_name):
        threads = ColumnFamily(self.conn, 'threads')

        ret = list(threads.get_range())
        if len(ret) > 99:
            oldest_thread = self._get_oldest_thread()
            oldest_thread_id = str(oldest_thread['thread_id'])
            threads.remove(oldest_thread_id)
            self._drop_cf(oldest_thread_id)

        thread_id = '%s' % random.randint(1,sys.maxint)

        dt = datetime.datetime.today()
        str_dt = dt.strftime('%Y-%m-%d %H:%M:%S')
        threads.insert(thread_id, {'thread_name': thread_name, 'post_count': '1', 'create_time': str_dt, 'update_time': str_dt})

        self._create_cf(thread_id)
        return thread_id