Python base.Notebook类代码示例

def editor(request):
  editor_id = request.GET.get('editor')

  if editor_id:
    editor = Notebook(document=Document2.objects.get(id=editor_id))
  else:
    editor = Notebook()
    data = editor.get_data()
    data['name'] = 'Hive SQL Editor'
    data['snippets'] = json.loads('[{"id":"c111cbb4-f475-4050-c5a1-02df6c31e3d8","name":"","type":"hive","editorMode":"text/x-hiveql","statement_raw":"Example: SELECT * FROM tablename, or press CTRL + space","codemirrorSize":100,"status":"ready","properties":{"settings":[],"files":[]},"variables":[],"variableNames":[],"statement":"Example: SELECT * FROM tablename, or press CTRL + space","result":{"id":"149347d9-3ae7-8d06-4cc8-d4bce5e72dc8","type":"table","hasResultset":true,"handle":{},"meta":[],"cleanedMeta":[],"fetchedOnce":false,"startTime":"2015-07-17T20:38:21.970Z","endTime":"2015-07-17T20:38:21.970Z","executionTime":0,"cleanedNumericMeta":[],"cleanedStringMeta":[],"cleanedDateTimeMeta":[],"data":[],"logs":"","logLines":0,"errors":"","hasSomeResults":false},"showGrid":true,"showChart":false,"showLogs":false,"progress":0,"size":12,"offset":0,"isLoading":false,"klass":"snippet card card-widget","editorKlass":"editor span12","resultsKlass":"results hive","errorsKlass":"results hive alert alert-error","chartType":"bars","chartSorting":"none","chartYMulti":[],"chartData":[],"tempChartOptions":{},"isLeftPanelVisible":false,"codeVisible":true,"settingsVisible":false,"checkStatusTimeout":null}]')
    editor.data = json.dumps(data)

  autocomplete_base_url = ''
  try:
    autocomplete_base_url = reverse('beeswax:api_autocomplete_databases', kwargs={})
  except:
    LOG.exception('failed to get autocomplete base url')

  return render('editor.mako', request, {
      'notebooks_json': json.dumps([editor.get_data()]),
      'options_json': json.dumps({
          'languages': [{"name": "Hive SQL", "type": "hive"}],
          'snippet_placeholders' : {
              'scala': _('Example: 1 + 1, or press CTRL + space'),
              'python': _('Example: 1 + 1, or press CTRL + space'),
              'impala': _('Example: SELECT * FROM tablename, or press CTRL + space'),
              'hive': _('Example: SELECT * FROM tablename, or press CTRL + space'),
              'text': _('<h2>This is a text snippet</h2>Type your text here')
          }
      }),
      'autocomplete_base_url': autocomplete_base_url,
  })

def get_history(request):
  response = {'status': -1}

  doc_type = request.GET.get('doc_type')
  limit = max(request.GET.get('len', 50), 100)

  response['status'] = 0
  history = []
  for doc in Document2.objects.get_history(doc_type='query-%s' % doc_type, user=request.user).order_by('-last_modified')[:limit]:
    notebook = Notebook(document=doc).get_data()
    if 'snippets' in notebook:
      history.append({
        'name': doc.name,
        'id': doc.id,
        'uuid': doc.uuid,
        'type': doc.type,
        'data': {
            'statement_raw': notebook['snippets'][0]['statement_raw'][:1001],
            'lastExecuted':  notebook['snippets'][0]['lastExecuted'],
            'status':  notebook['snippets'][0]['status'],
            'parentUuid': notebook.get('parentUuid', '')
        } if notebook['snippets'] else {},
        'absoluteUrl': doc.get_absolute_url(),
      })
    else:
      LOG.error('Incomplete History Notebook: %s' % notebook)
  response['history'] = history
  response['message'] = _('History fetched')

  return JsonResponse(response)

def notebook(request):
  notebook_id = request.GET.get('notebook')

  if notebook_id:
    notebook = Notebook(document=Document2.objects.get(id=notebook_id))
  else:
    notebook = Notebook()

  autocomplete_base_url = ''
  try:
    autocomplete_base_url = reverse('beeswax:api_autocomplete_databases', kwargs={})
  except:
    LOG.exception('failed to get autocomplete base url')

  return render('notebook.mako', request, {
      'notebooks_json': json.dumps([notebook.get_data()]),
      'options_json': json.dumps({
          'languages': get_interpreters(),
          'snippet_placeholders' : {
              'sql': _('Example: 1 + 1, or press CTRL + space'),
              'spark': _('Example: 1 + 1, or press CTRL + space'),
              'pyspark': _('Example: 1 + 1, or press CTRL + space'),
              'impala': _('Example: SELECT * FROM tablename, or press CTRL + space'),
              'hive': _('Example: SELECT * FROM tablename, or press CTRL + space'),
              'r': _('Example: 1 + 1, or press CTRL + space')
          },
          'session_properties': SparkApi.PROPERTIES
      }),
      'autocomplete_base_url': autocomplete_base_url,
      'is_yarn_mode': LIVY_SERVER_SESSION_KIND.get()
  })

def notebook(request):
  notebook_id = request.GET.get('notebook')

  if notebook_id:
    notebook = Notebook(document=Document2.objects.get(id=notebook_id))
  else:
    notebook = Notebook()

  autocomplete_base_url = ''
  try:
    autocomplete_base_url = reverse('beeswax:api_autocomplete_databases', kwargs={})
  except:
    LOG.exception('failed to get autocomplete base url')

  is_yarn_mode = False
  try:
    from spark.conf import LIVY_SERVER_SESSION_KIND
    is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
  except:
    LOG.exception('Spark is not enabled')

  return render('notebook.mako', request, {
      'notebooks_json': json.dumps([notebook.get_data()]),
      'options_json': json.dumps({
          'languages': get_interpreters(request.user),
          'session_properties': SparkApi.PROPERTIES,
      }),
      'autocomplete_base_url': autocomplete_base_url,
      'is_yarn_mode': is_yarn_mode
  })

def editor(request):
  editor_id = request.GET.get('editor')
  editor_type = request.GET.get('type', 'hive')

  if editor_id:
    editor = Notebook(document=Document2.objects.get(id=editor_id))
  else:
    editor = Notebook()
    data = editor.get_data()
    data['name'] = 'Untitled %s Query' % editor_type.title()
    data['type'] = 'query-%s' % editor_type
    editor.data = json.dumps(data)

  autocomplete_base_url = ''
  try:
    autocomplete_base_url = reverse('beeswax:api_autocomplete_databases', kwargs={})
  except:
    LOG.exception('failed to get autocomplete base url')

  return render('editor.mako', request, {
      'notebooks_json': json.dumps([editor.get_data()]),
      'options_json': json.dumps({
          'languages': [{"name": "%s SQL" % editor_type.title(), "type": editor_type}],
          'mode': 'editor',
      }),
      'editor_type': editor_type,
      'autocomplete_base_url': autocomplete_base_url,
  })

def action_parameters(request):
  response = {'status': -1}
  parameters = set()

  try:
    node_data = json.loads(request.POST.get('node', '{}'))

    parameters = parameters.union(set(Node(node_data).find_parameters()))

    script_path = node_data.get('properties', {}).get('script_path', {})
    if script_path:
      script_path = script_path.replace('hdfs://', '')

      if request.fs.do_as_user(request.user, request.fs.exists, script_path):
        data = request.fs.do_as_user(request.user, request.fs.read, script_path, 0, 16 * 1024 ** 2)

        if node_data['type'] in ('hive', 'hive2'):
          parameters = parameters.union(set(find_dollar_braced_variables(data)))
        elif node_data['type'] == 'pig':
          parameters = parameters.union(set(find_dollar_variables(data)))
    elif node_data['type'] == 'hive-document':
      notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid']))
      parameters = parameters.union(set(find_dollar_braced_variables(notebook.get_str())))

    response['status'] = 0
    response['parameters'] = list(parameters)
  except Exception, e:
    response['message'] = str(e)

def execute_and_watch(request):
  notebook_id = request.GET.get('editor', request.GET.get('notebook'))
  snippet_id = int(request.GET['snippet'])
  action = request.GET['action']
  destination = request.GET['destination']

  notebook = Notebook(document=Document2.objects.get(id=notebook_id))
  snippet = notebook.get_data()['snippets'][snippet_id]
  editor_type = snippet['type']

  api = get_api(request, snippet)

  if action == 'save_as_table':
    sql, success_url = api.export_data_as_table(snippet, destination)
    editor = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready-execute')
  elif action == 'insert_as_query':
    sql, success_url = api.export_large_data_to_hdfs(snippet, destination)
    editor = make_notebook(name='Execute and watch', editor_type=editor_type, statement=sql, status='ready-execute')
  else:
    raise PopupException(_('Action %s is unknown') % action)

  return render('editor.mako', request, {
      'notebooks_json': json.dumps([editor.get_data()]),
      'options_json': json.dumps({
          'languages': [{"name": "%s SQL" % editor_type.title(), "type": editor_type}],
          'mode': 'editor',
          'success_url': success_url
      }),
      'editor_type': editor_type,
  })

def notebook(request):
    notebook_id = request.GET.get("notebook")

    if notebook_id:
        notebook = Notebook(document=Document2.objects.get(id=notebook_id))
    else:
        notebook = Notebook()

    autocomplete_base_url = ""
    try:
        autocomplete_base_url = reverse("beeswax:api_autocomplete_databases", kwargs={})
    except:
        LOG.exception("failed to get autocomplete base url")

    is_yarn_mode = False
    try:
        from spark.conf import LIVY_SERVER_SESSION_KIND

        is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
    except:
        LOG.exception("Spark is not enabled")

    return render(
        "notebook.mako",
        request,
        {
            "notebooks_json": json.dumps([notebook.get_data()]),
            "options_json": json.dumps(
                {"languages": get_interpreters(request.user), "session_properties": SparkApi.PROPERTIES}
            ),
            "autocomplete_base_url": autocomplete_base_url,
            "is_yarn_mode": is_yarn_mode,
        },
    )

def make_notebook2(name='Browse', description='', is_saved=False, snippets=None):

  from notebook.connectors.hiveserver2 import HS2Api

  editor = Notebook()

  _snippets = []

  for snippet in snippets:
    default_properties = {
        'files': [],
        'functions': [],
        'settings': []
    }

    if snippet['type'] == 'hive':
      pass
    elif snippet['type'] == 'impala':
      pass
    elif snippet['type'] == 'java':
      pass

    _snippets.append(snippet)

  print _snippets

  data = {
    'name': name,
    'uuid': str(uuid.uuid4()),
    'description': description,
    'sessions': [
      {
         'type': _snippet['type'],
         'properties': HS2Api.get_properties(snippet['type']),
         'id': None
      } for _snippet in _snippets # Non unique types currently
    ],
    'selectedSnippet': _snippets[0]['type'],
    'type': 'notebook',
    'showHistory': False,
    'isSaved': is_saved,
    'snippets': [
      {
         'status': _snippet.get('status', 'ready'),
         'id': str(uuid.uuid4()),
         'statement_raw': _snippet.get('statement', ''),
         'statement': _snippet.get('statement', ''),
         'type': _snippet.get('type'),
         'properties': _snippet.properties,
         'name': name,
         'database': _snippet.get('database'),
         'result': {}
      } for _snippet in _snippets
    ]
  }

  editor.data = json.dumps(data)

  return editor

def open_notebook(request):
    response = {"status": -1}

    notebook_id = request.GET.get("notebook")
    notebook = Notebook(document=Document2.objects.get(id=notebook_id))

    response["status"] = 0
    response["notebook"] = notebook.get_json()
    response["message"] = _("Notebook loaded successfully")

def open_notebook(request):
  response = {'status': -1}

  notebook_id = request.GET.get('notebook')
  notebook = Notebook(document=Document2.objects.get(id=notebook_id))

  response['status'] = 0
  response['notebook'] = notebook.get_json()
  response['message'] = _('Notebook loaded successfully')

def make_notebook(name='Browse', description='', editor_type='hive', statement='', status='ready',
                  files=None, functions=None, settings=None):

  from notebook.connectors.hiveserver2 import HS2Api

  editor = Notebook()

  properties = HS2Api.get_properties(editor_type)

  if editor_type == 'hive':
    if files is not None:
      _update_property_value(properties, 'files', files)

    if functions is not None:
      _update_property_value(properties, 'functions', functions)

    if settings is not None:
      _update_property_value(properties, 'settings', settings)
  elif editor_type == 'impala':
    if settings is not None:
      _update_property_value(properties, 'files', files)

  editor.data = json.dumps({
    'name': name,
    'description': description,
    'sessions': [
      {
         'type': editor_type,
         'properties': properties,
         'id': None
      }
    ],
    'selectedSnippet': editor_type,
    'type': 'query-%s' % editor_type,
    'showHistory': True,

    'snippets': [
      {
         'status': status,
         'id': str(uuid.uuid4()),
         'statement_raw': statement,
         'statement': statement,
         'type': editor_type,
         'properties': {
            'files': [] if files is None else files,
            'functions': [] if functions is None else functions,
            'settings': [] if settings is None else settings
         },
         'name': name,
         'database': 'default',
         'result': {}
      }
    ]
  })
  
  return editor

def open_notebook(request):
  response = {'status': -1}

  notebook_id = request.GET.get('notebook')
  notebook = Notebook(document=Document2.objects.get(id=notebook_id))

  response['status'] = 0
  response['notebook'] = notebook.get_json()
  response['message'] = _('Notebook saved !')

  return JsonResponse(response)

def browse(request, database, table):
  editor_type = request.GET.get('type', 'hive')

  snippet = {'type': editor_type}
  sql_select = get_api(request.user, snippet, request.fs, request.jt).get_select_star_query(snippet, database, table)

  editor = Notebook()
  editor.data = json.dumps({
    'description':'',
    'sessions':[
      {
         'type':'hive',
         'properties':[

         ],
         'id':None
      }
    ],
    'selectedSnippet':'hive',
    'type': 'query-%s' % editor_type,

    'snippets':[
      {
         'status':'ready-execute',
         'id':'e8b323b3-88ef-3a84-6264-af11fa5fbefb',
         'statement_raw': sql_select,
         'statement': sql_select,
         'type': editor_type,
         'properties':{
            'files':[
            ],
            'settings':[
            ]
         },
         'name': 'Browse',
         'database':'default',
         'result':{  }
      }
    ],
    'name':'Browse'
  })

  return render('editor.mako', request, {
      'notebooks_json': json.dumps([editor.get_data()]),
      'options_json': json.dumps({
          'languages': [{"name": "%s SQL" % editor_type.title(), "type": editor_type}],
          'mode': 'editor',
      }),
      'editor_type': editor_type,
  })

def get_history(request):
    response = {"status": -1}

    doc_type = request.GET.get("doc_type")
    doc_text = request.GET.get("doc_text")
    limit = min(request.GET.get("len", 50), 100)

    docs = Document2.objects.get_history(doc_type="query-%s" % doc_type, user=request.user)

    if doc_text:
        docs = docs.filter(
            Q(name__icontains=doc_text) | Q(description__icontains=doc_text) | Q(search__icontains=doc_text)
        )

    history = []
    for doc in docs.order_by("-last_modified")[:limit]:
        notebook = Notebook(document=doc).get_data()
        if "snippets" in notebook:
            statement = _get_statement(notebook)
            history.append(
                {
                    "name": doc.name,
                    "id": doc.id,
                    "uuid": doc.uuid,
                    "type": doc.type,
                    "data": {
                        "statement": statement[:1001] if statement else "",
                        "lastExecuted": notebook["snippets"][0]["lastExecuted"],
                        "status": notebook["snippets"][0]["status"],
                        "parentSavedQueryUuid": notebook.get("parentSavedQueryUuid", ""),
                    }
                    if notebook["snippets"]
                    else {},
                    "absoluteUrl": doc.get_absolute_url(),
                }
            )
        else:
            LOG.error("Incomplete History Notebook: %s" % notebook)
    response["history"] = sorted(history, key=lambda row: row["data"]["lastExecuted"], reverse=True)
    response["message"] = _("History fetched")
    response["status"] = 0

    return JsonResponse(response)