Python models.QueryHistory类代码示例

    def execute_query(self, query, design):
        from beeswax.models import QueryHistory

        sql_query = query.sql_query
        query_history = QueryHistory.build(
            owner=self.client.user,
            query=sql_query,
            server_host="%(server_host)s" % self.client.query_server,
            server_port="%(server_port)d" % self.client.query_server,
            server_name="%(server_name)s" % self.client.query_server,
            server_type=self.server_type,
            last_state=QueryHistory.STATE.available.index,
            design=design,
            notify=False,
            query_type=query.query["type"],
            statement_number=0,
        )
        query_history.save()

        LOG.debug(
            "Updated QueryHistory id %s user %s statement_number: %s"
            % (query_history.id, self.client.user, query_history.statement_number)
        )

        return query_history

  def execute_and_watch(self, query, design=None, notify=False):
    """
    Run query and return a QueryHistory object in order to see its progress on a Web page.
    """
    query_statement = query.query['query']
    query_history = QueryHistory.build(
                                owner=self.client.user,
                                query=query_statement,
                                server_host='%(server_host)s' % self.client.query_server,
                                server_port='%(server_port)d' % self.client.query_server,
                                server_name='%(server_name)s' % self.client.query_server,
                                server_type=self.server_type,
                                last_state=QueryHistory.STATE.submitted.index,
                                design=design,
                                notify=notify)
    query_history.save()

    LOG.debug("Made new QueryHistory id %s user %s query: %s..." % (query_history.id, self.client.user, query_history.query[:25]))

    try:
      handle = self.client.query(query)
      if not handle.is_valid():
        msg = _("Server returning invalid handle for query id %(id)d [%(query)s]...") % \
              {'id': query_history.id, 'query': query_statement[:40]}
        raise BeeswaxException(msg)
    except BeeswaxException, ex: # TODO HS2
      LOG.exception(ex)
      # Kind of expected (hql compile/syntax error, etc.)
      if hasattr(ex, 'handle') and ex.handle:
        query_history.server_id = ex.handle.id
        query_history.log_context = ex.handle.log_context
      query_history.save_state(QueryHistory.STATE.failed)
      raise ex

def execute_directly(user, query_msg, design=None, notify=False):
  """
  execute_directly(user, query_msg [,design]) -> QueryHistory object

  This method also creates a QueryHistory object and saves it.
  user - Django user.
  query_msg - The thrift Query object.
  design - The SavedQuery object (i.e. design) associated with this query.
  notify - Whether to notify the user upon completion.
  """
  query_history = QueryHistory(
                            owner=user,
                            query=query_msg.query,
                            last_state=QueryHistory.STATE.submitted.index,
                            design=design,
                            notify=notify)
  query_history.save()
  LOG.debug("Made new QueryHistory id %s user %s query: %s..." %
            (query_history.id, user, query_history.query[:25]))

  # Now submit it
  try:
    handle = db_client().query(query_msg)
    if not handle or not handle.id or not handle.log_context:
      # It really shouldn't happen
      msg = "BeeswaxServer returning invalid handle for query id %d [%s]..." % \
            (query_history.id, query_msg.query[:40])
      raise Exception(msg)
  except BeeswaxException, bex:
    # Kind of expected (hql compile/syntax error, etc.)
    if bex.handle:
      query_history.server_id = bex.handle.id
      query_history.log_context = bex.handle.log_context
    query_history.save_state(QueryHistory.STATE.failed)
    raise bex

def authorized_get_history(request, query_history_id, owner_only=False, must_exist=False):
  if query_history_id is None and not must_exist:
    return None
  try:
    query_history = QueryHistory.get(id=query_history_id)
  except QueryHistory.DoesNotExist:
    if must_exist:
      raise PopupException(_('QueryHistory %(id)s does not exist.') % {'id': query_history_id})
    else:
      return None

  query_history.design.doc.get().can_read_or_exception(request.user)

  return query_history

  def handle(self, *args, **options):
    days = int(args[0]) if len(args) >= 1 else 7
    close_all = args[1] == 'all' if len(args) >= 2 else False

    self.stdout.write('Closing (all=%s) HiveServer2 queries older than %s days...\n' % (close_all, days))

    queries = QueryHistory.objects.filter(last_state__in=[QueryHistory.STATE.expired.value, QueryHistory.STATE.failed.value, QueryHistory.STATE.available.value])

    if close_all:
      queries = QueryHistory.objects.all()

    queries = queries.filter(submission_date__lte=datetime.today() - timedelta(days=days))

    import os
    import beeswax
    from beeswax import conf
    from beeswax import hive_site
    try:
      beeswax.conf.HIVE_CONF_DIR.set_for_testing(os.environ['HIVE_CONF_DIR'])
    except:
      LOG.exception('failed to lookup HIVE_CONF_DIR in environment')
      self.stdout.write('Did you export HIVE_CONF_DIR=/etc/hive/conf?\n')
      raise

    hive_site.reset()
    hive_site.get_conf()

    closed_queries = 0
    already_closed_queries = 0

    for query in queries:
      try:
        query_history = QueryHistory.get(id=query.id)
        if query_history.server_id is not None:
          handle = query_history.get_handle()
          dbms.get(user=query_history.owner).close_operation(handle)
          closed_queries += 1
        else:
          already_closed_queries += 1

        query.last_state = QueryHistory.STATE.expired.value
        query.save()
      except Exception, e:
        if 'None' in str(e) or 'Invalid OperationHandle' in str(e):
          already_closed_queries += 1
          query.last_state = QueryHistory.STATE.expired.value
          query.save()
        else:
          self.stdout.write('Info: %s\n' % e)

  def test_get_runtime_profile(self):
    query = """
      SELECT COUNT(1) FROM tweets;
    """

    response = _make_query(self.client, query, database=self.DATABASE, local=False, server_name='impala')
    content = json.loads(response.content)
    query_history = QueryHistory.get(content['id'])

    wait_for_query_to_finish(self.client, response, max=180.0)

    resp = self.client.post(reverse('impala:get_runtime_profile', kwargs={'query_history_id': query_history.id}))
    data = json.loads(resp.content)
    assert_equal(0, data['status'], data)
    assert_true('Execution Profile' in data['profile'], data)

def authorized_get_query_history(request, query_history_id, owner_only=False, must_exist=False):
  if query_history_id is None and not must_exist:
    return None
  try:
    query_history = QueryHistory.get(id=query_history_id)
  except QueryHistory.DoesNotExist:
    if must_exist:
      raise PopupException(_('QueryHistory %(id)s does not exist.') % {'id': query_history_id})
    else:
      return None

  # Some queries don't have a design so are not linked to Document Model permission
  if query_history.design is None or not query_history.design.doc.exists():
    if not request.user.is_superuser and request.user != query_history.owner:
      raise PopupException(_('Permission denied to read QueryHistory %(id)s') % {'id': query_history_id})
  else:
    query_history.design.doc.get().can_read_or_exception(request.user)

  return query_history

    def execute_and_watch(self, query, design=None, query_history=None):
        """
    Run query and return a QueryHistory object in order to see its progress on a Web page.
    """
        hql_query = query.hql_query
        if query_history is None:
            query_history = QueryHistory.build(
                owner=self.client.user,
                query=hql_query,
                server_host="%(server_host)s" % self.client.query_server,
                server_port="%(server_port)d" % self.client.query_server,
                server_name="%(server_name)s" % self.client.query_server,
                server_type=self.server_type,
                last_state=QueryHistory.STATE.submitted.index,
                design=design,
                notify=query.query.get("email_notify", False),
                query_type=query.query["type"],
                statement_number=0,
            )
            query_history.save()

            LOG.debug(
                "Made new QueryHistory id %s user %s query: %s..."
                % (query_history.id, self.client.user, query_history.query[:25])
            )

        try:
            handle = self.client.query(query, query_history.statement_number)
            if not handle.is_valid():
                msg = _("Server returning invalid handle for query id %(id)d [%(query)s]...") % {
                    "id": query_history.id,
                    "query": query[:40],
                }
                raise QueryServerException(msg)
        except QueryServerException, ex:
            LOG.exception(ex)
            # Kind of expected (hql compile/syntax error, etc.)
            if hasattr(ex, "handle") and ex.handle:
                query_history.server_id, query_history.server_guid = ex.handle.id, ex.handle.id
                query_history.log_context = ex.handle.log_context
            query_history.save_state(QueryHistory.STATE.failed)
            raise ex

    def test_basic_flow(self):
        dbs = self.db.get_databases()
        assert_true("_impala_builtins" in dbs, dbs)
        assert_true(self.DATABASE in dbs, dbs)

        tables = self.db.get_tables(database=self.DATABASE)
        assert_true("tweets" in tables, tables)

        QUERY = """
      SELECT * FROM tweets ORDER BY row_num;
    """
        response = _make_query(self.client, QUERY, database=self.DATABASE, local=False, server_name="impala")
        content = json.loads(response.content)
        query_history = QueryHistory.get(content["id"])

        response = wait_for_query_to_finish(self.client, response, max=180.0)

        results = []

        # Check that we multiple fetches get all the result set
        while len(results) < 5:
            content = fetch_query_result_data(
                self.client, response, n=len(results), server_name="impala"
            )  # We get less than 5 results most of the time, so increase offset
            results += content["results"]

        assert_equal([1, 2, 3, 4, 5], [col[0] for col in results])

        # Check start over
        results_start_over = []

        while len(results_start_over) < 5:
            content = fetch_query_result_data(self.client, response, n=len(results_start_over), server_name="impala")
            results_start_over += content["results"]

        assert_equal(results_start_over, results)

        # Check cancel query
        resp = self.client.post(reverse("impala:api_cancel_query", kwargs={"query_history_id": query_history.id}))
        content = json.loads(resp.content)
        assert_equal(0, content["status"])

def execute_directly(user, query_msg, query_server=None, design=None, notify=False):
  """
  execute_directly(user, query_msg [,design]) -> QueryHistory object

  This method also creates a QueryHistory object and saves it.
  user - Django user.
  query_msg - The thrift Query object.
  design - The SavedQuery object (i.e. design) associated with this query.
  notify - Whether to notify the user upon completion.
  """
  if query_server is None:
    query_server = get_query_server(support_ddl=True) # For convenience with DDL queries only

  query_history = QueryHistory(
                            owner=user,
                            query=query_msg.query,
                            server_host='%(server_host)s' % query_server,
                            server_port='%(server_port)d' % query_server,
                            server_name='%(server_name)s' % query_server,
                            last_state=QueryHistory.STATE.submitted.index,
                            design=design,
                            notify=notify)
  query_history.save()
  LOG.debug("Made new QueryHistory id %s user %s query: %s..." %
            (query_history.id, user, query_history.query[:25]))

  # Now submit it
  try:
    handle = db_client(query_server).query(query_msg)
    if not handle or not handle.id:
      # It really shouldn't happen
      msg = _("BeeswaxServer returning invalid handle for query id %(id)d [%(query)s]...") % \
            {'id': query_history.id, 'query': query_msg.query[:40]}
      raise Exception(msg)
  except BeeswaxException, bex:
    # Kind of expected (hql compile/syntax error, etc.)
    if bex.handle:
      query_history.server_id = bex.handle.id
      query_history.log_context = bex.handle.log_context
    query_history.save_state(QueryHistory.STATE.failed)
    raise bex

  def test_get_exec_summary(self):
    query = """
      SELECT COUNT(1) FROM tweets;
    """

    response = _make_query(self.client, query, database=self.DATABASE, local=False, server_name='impala')
    content = json.loads(response.content)
    query_history = QueryHistory.get(content['id'])

    wait_for_query_to_finish(self.client, response, max=180.0)

    resp = self.client.post(reverse('impala:get_exec_summary', kwargs={'query_history_id': query_history.id}))
    data = json.loads(resp.content)
    assert_equal(0, data['status'], data)
    assert_true('nodes' in data['summary'], data)
    assert_true(len(data['summary']['nodes']) > 0, data['summary']['nodes'])

    # Attempt to call get_exec_summary on a closed query
    resp = self.client.post(reverse('impala:get_exec_summary', kwargs={'query_history_id': query_history.id}))
    data = json.loads(resp.content)
    assert_equal(0, data['status'], data)
    assert_true('nodes' in data['summary'], data)
    assert_true(len(data['summary']['nodes']) > 0, data['summary']['nodes'])