Python conf.BROWSE_PARTITIONED_TABLE_LIMIT类代码示例

  def get_partitions(self, db_name, table, max_parts=None):
    if max_parts is None or max_parts > BROWSE_PARTITIONED_TABLE_LIMIT.get():
      max_parts = BROWSE_PARTITIONED_TABLE_LIMIT.get()

    # DB name not supported in SHOW PARTITIONS
    self.use(db_name)

    return self.client.get_partitions(db_name, table.name, max_parts)

 def _get_browse_limit_clause(self, table):
   """Get the limit clause when browsing a partitioned table"""
   if table.partition_keys:
     limit = BROWSE_PARTITIONED_TABLE_LIMIT.get()
     if limit > 0:
       return "LIMIT %d" % (limit,)
   return ""

  def get_sample(self, database, table, column=None, nested=None):
    result = None
    hql = None

    if not table.is_view:
      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())

      if column or nested: # Could do column for any type, then nested with partitions 
        if self.server_name == 'impala':
          select_clause, from_clause = ImpalaDbms.get_nested_select(database, table.name, column, nested)
          hql = 'SELECT %s FROM %s LIMIT %s' % (select_clause, from_clause, limit)
      else:
        if table.partition_keys:  # Filter on max # of partitions for partitioned tables
          hql = self._get_sample_partition_query(database, table, limit)
        else:
          hql = "SELECT * FROM `%s`.`%s` LIMIT %s" % (database, table.name, limit)

      if hql:
        query = hql_query(hql)
        handle = self.execute_and_wait(query, timeout_sec=5.0)

        if handle:
          result = self.fetch(handle, rows=100)
          self.close(handle)

    return result

  def get_sample(self, database, table, column=None, nested=None):
    result = None
    hql = None

    if not table.is_view:

      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())

      if column or nested: # Could do column for any type, then nested with partitions 
        if self.server_name == 'impala':
          select_clause, from_clause = ImpalaDbms.get_nested_select(database, table.name, column, nested)
          hql = 'SELECT %s FROM %s LIMIT %s' % (select_clause, from_clause, limit)
      else:
        partition_query = ""
        if table.partition_keys:
          partitions = self.get_partitions(database, table, partition_spec=None, max_parts=1)
          partition_query = 'WHERE ' + ' AND '.join(["%s='%s'" % (table.partition_keys[idx].name, key) for idx, key in enumerate(partitions[0].values)])
        hql = "SELECT * FROM `%s`.`%s` %s LIMIT %s" % (database, table.name, partition_query, limit)

      if hql:
        query = hql_query(hql)
        handle = self.execute_and_wait(query, timeout_sec=5.0)

        if handle:
          result = self.fetch(handle, rows=100)
          self.close(handle)

    return result

 def select_star_from(self, database, table):
   if table.partition_keys:  # Filter on max # of partitions for partitioned tables
     limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
     hql = self._get_sample_partition_query(database, table, limit)
   else:
     hql = "SELECT * FROM `%s`.`%s`" % (database, table.name)
   return self.execute_statement(hql)

  def test_describe_partitioned_table_with_limit(self):
    # We have 2 partitions in the test table
    finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("1")
    try:
      response = self.client.get("/metastore/table/%s/test_partitions/partitions" % self.db_name)
      partition_values_json = json.loads(response.context['partition_values_json'])
      assert_equal(1, len(partition_values_json))
    finally:
      finish()

    finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("3")
    try:
      response = self.client.get("/metastore/table/%s/test_partitions/partitions" % self.db_name)
      partition_values_json = json.loads(response.context['partition_values_json'])
      assert_equal(2, len(partition_values_json))
    finally:
      finish()

 def test_browse_partitioned_table_with_limit(self):
   # Limit to 90
   finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("90")
   try:
     response = self.client.get("/metastore/table/default/test_partitions")
     assert_true("0x%x" % 89 in response.content, response.content)
     assert_false("0x%x" % 90 in response.content, response.content)
   finally:
     finish()

  def get_sample(self, database, table):
    """No samples if it's a view (HUE-526)"""
    if not table.is_view:
      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
      hql = "SELECT * FROM `%s.%s` LIMIT %s" % (database, table.name, limit)
      query = hql_query(hql)
      handle = self.execute_and_wait(query, timeout_sec=5.0)

      if handle:
        return self.fetch(handle)

  def test_describe_partitioned_table_with_limit(self):
    if is_live_cluster():
      raise SkipTest('HUE-2902: Test is not re-entrant')

    # Limit to 90
    finish = BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("90")
    try:
      response = self.client.get("/metastore/table/%s/test_partitions" % self.db_name)
      assert_true("0x%x" % 89 in response.content, response.content)
      assert_false("0x%x" % 90 in response.content, response.content)
    finally:
      finish()

  def test_browse_limit_overrides_partition_limit(self):
    # Limit to 45
    finish = [
        BROWSE_TABLE_LIMIT.set_for_testing("45"),
        BROWSE_PARTITIONED_TABLE_LIMIT.set_for_testing("90"),
    ]

    try:
      response = self.client.get("/metastore/table/default/test_partitions")
      assert_true("0x%x" % 44 in response.content, response.content)
      assert_false("0x%x" % 45 in response.content, response.content)
    finally:
      for f in finish:
        f()

  def get_browse_partition_clause(self, table, partitions):
    """Get the where clause to limit reading data in the first available partitions"""
    if partitions and BROWSE_PARTITIONED_TABLE_LIMIT.get() > 0:
      partition_values = partitions[0].values
      partition_keys = table.partition_keys
      partition_dict = zip(partition_keys, partition_values)

      fields = []
      for key, value in partition_dict:
          if key.type == "string":
            fields.append("`%s` = '%s'" % (key.name, value))
          else:
            fields.append("`%s` = %s" % (key.name, value))

      return "WHERE " + " AND ".join(fields)

  def get_sample(self, database, table):
    """No samples if it's a view (HUE-526)"""
    if not table.is_view:
      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())
      partition_query = ""
      if table.partition_keys:
        partitions = self.get_partitions(database, table, partition_spec=None, max_parts=1)
        partition_query = 'WHERE ' + ' AND '.join(["%s='%s'" % (table.partition_keys[idx].name, key) for idx, key in enumerate(partitions[0].values)])
      hql = "SELECT * FROM `%s`.`%s` %s LIMIT %s" % (database, table.name, partition_query, limit)
      query = hql_query(hql)
      handle = self.execute_and_wait(query, timeout_sec=5.0)

      if handle:
        result = self.fetch(handle, rows=100)
        self.close(handle)
        return result

  def get_sample(self, database, table):
    """No samples if it's a view (HUE-526)"""
    if not table.is_view:
      limit = BROWSE_TABLE_LIMIT.get()
      if not limit:
        limit = 100

      limit = min(limit, BROWSE_PARTITIONED_TABLE_LIMIT.get())
      hql = "SELECT * FROM %s.%s LIMIT %s" % (database, table.name, limit)
      query = hql_query(hql)
      handle = self.execute_and_wait(query, timeout_sec=5.0)

      if handle:
        result = self.fetch(handle, rows=limit)
        self.close(handle)
        return result

  def get_sample(self, database, table, column=None, nested=None):
    result = None
    hql = None

    if not table.is_view:
      limit = min(100, BROWSE_PARTITIONED_TABLE_LIMIT.get())

      if table.partition_keys:  # Filter on max # of partitions for partitioned tables
        hql = self._get_sample_partition_query(database, table, limit)
      else:
        hql = "SELECT * FROM `%s`.`%s` LIMIT %s" % (database, table.name, limit)

      if hql:
        query = hql_query(hql)
        handle = self.execute_and_wait(query, timeout_sec=5.0)

        if handle:
          result = self.fetch(handle, rows=100)
          self.close(handle)

    return result

  def get_partitions(self, db_name, table, max_parts=None):
    if max_parts is None or max_parts > BROWSE_PARTITIONED_TABLE_LIMIT.get():
      max_parts = BROWSE_PARTITIONED_TABLE_LIMIT.get()

    return self.client.get_partitions(db_name, table.name, max_parts)