Python Submission.run方法代码示例

# 需要导入模块: from liboozie.submission2 import Submission [as 别名]
# 或者: from liboozie.submission2.Submission import run [as 别名]
def submit_external_job(request, application_path):
  ParametersFormSet = formset_factory(ParameterForm, extra=0)

  if request.method == 'POST':
    params_form = ParametersFormSet(request.POST)

    if params_form.is_valid():
      mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
      mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
      application_name = os.path.basename(application_path)
      application_class = Bundle if application_name == 'bundle.xml' else Coordinator if application_name == 'coordinator.xml' else get_workflow()
      mapping[application_class.get_application_path_key()] = application_path

      try:
        submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping)
        job_id = submission.run(application_path)
      except RestException, ex:
        detail = ex._headers.get('oozie-error-message', ex)
        if 'Max retries exceeded with url' in str(detail):
          detail = '%s: %s' % (_('The Oozie server is not running'), detail)

        LOG.exception(smart_str(detail))

        raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail)

      request.info(_('Oozie job submitted'))
      view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow'
      return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id}))
    else:
      request.error(_('Invalid submission form: %s' % params_form.errors))

# 需要导入模块: from liboozie.submission2 import Submission [as 别名]
# 或者: from liboozie.submission2.Submission import run [as 别名]
def _submit_bundle(request, bundle, properties):
  try:
    deployment_mapping = {}
    coords = dict([(c.uuid, c) for c in Document2.objects.filter(type='oozie-coordinator2', uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])])

    for i, bundled in enumerate(bundle.data['coordinators']):
      coord = coords[bundled['coordinator']]
      workflow = Workflow(document=coord.dependencies.filter(type='oozie-workflow2')[0])
      wf_dir = Submission(request.user, workflow, request.fs, request.jt, properties).deploy()
      deployment_mapping['wf_%s_dir' % i] = request.fs.get_hdfs_path(wf_dir)

      coordinator = Coordinator(document=coord)
      coord_dir = Submission(request.user, coordinator, request.fs, request.jt, properties).deploy()
      deployment_mapping['coord_%s_dir' % i] = request.fs.get_hdfs_path(coord_dir)
      deployment_mapping['coord_%s' % i] = coord

      # Convert start/end dates of coordinator to server timezone
      for prop in bundled['properties']:
        if prop['name'] in ('end_date', 'start_date'):
          prop['value'] = convert_to_server_timezone(prop['value'], local_tz=coordinator.data['properties']['timezone'])

    properties.update(deployment_mapping)

    submission = Submission(request.user, bundle, request.fs, request.jt, properties=properties)
    job_id = submission.run()

    return job_id
  except RestException, ex:
    LOG.exception('Error submitting bundle')
    raise PopupException(_("Error submitting bundle %s") % (bundle,), detail=ex._headers.get('oozie-error-message', ex), error_code=200)

# 需要导入模块: from liboozie.submission2 import Submission [as 别名]
# 或者: from liboozie.submission2.Submission import run [as 别名]
def _submit_bundle(request, bundle, properties):
  try:
    deployment_mapping = {}
    coords = dict([(c.uuid, c) for c in Document2.objects.filter(type='oozie-coordinator2', uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])])

    for i, bundled in enumerate(bundle.data['coordinators']):
      coord = coords[bundled['coordinator']]
      workflow = Workflow(document=coord.dependencies.all()[0])
      wf_dir = Submission(request.user, workflow, request.fs, request.jt, properties).deploy()
      deployment_mapping['wf_%s_dir' % i] = request.fs.get_hdfs_path(wf_dir)

      coordinator = Coordinator(document=coord)
      coord_dir = Submission(request.user, coordinator, request.fs, request.jt, properties).deploy()
      deployment_mapping['coord_%s_dir' % i] = coord_dir
      deployment_mapping['coord_%s' % i] = coord

    properties.update(deployment_mapping)

    submission = Submission(request.user, bundle, request.fs, request.jt, properties=properties)
    job_id = submission.run()

    return job_id
  except RestException, ex:
    LOG.exception('Error submitting bundle')
    raise PopupException(_("Error submitting bundle %s") % (bundle,), detail=ex._headers.get('oozie-error-message', ex))

# 需要导入模块: from liboozie.submission2 import Submission [as 别名]
# 或者: from liboozie.submission2.Submission import run [as 别名]
def _submit_workflow(user, fs, jt, workflow, mapping):
  try:
    submission = Submission(user, workflow, fs, jt, mapping)
    job_id = submission.run()
    return job_id
  except RestException, ex:
    detail = ex._headers.get('oozie-error-message', ex)
    if 'Max retries exceeded with url' in str(detail):
      detail = '%s: %s' % (_('The Oozie server is not running'), detail)
    LOG.error(smart_str(detail))
    raise PopupException(_("Error submitting workflow %s") % (workflow,), detail=detail)

# 需要导入模块: from liboozie.submission2 import Submission [as 别名]
# 或者: from liboozie.submission2.Submission import run [as 别名]
def _submit_workflow(user, fs, jt, workflow, mapping):
  try:
    submission = Submission(user, workflow, fs, jt, mapping)
    job_id = submission.run()

    workflow.document.add_to_history(submission.user, {'properties': submission.properties, 'oozie_id': submission.oozie_id})

    return job_id
  except RestException, ex:
    detail = ex._headers.get('oozie-error-message', ex)
    if 'Max retries exceeded with url' in str(detail):
      detail = '%s: %s' % (_('The Oozie server is not running'), detail)
    LOG.exception('Error submitting workflow: %s' % smart_str(detail))
    raise PopupException(_("Error submitting workflow %s: %s") % (workflow, detail))

# 需要导入模块: from liboozie.submission2 import Submission [as 别名]
# 或者: from liboozie.submission2.Submission import run [as 别名]
def _submit_coordinator(request, coordinator, mapping):
  try:
    wf_doc = Document2.objects.get_by_uuid(user=request.user, uuid=coordinator.data['properties']['workflow'])
    wf_dir = Submission(request.user, Workflow(document=wf_doc), request.fs, request.jt, mapping, local_tz=coordinator.data['properties']['timezone']).deploy()

    properties = {'wf_application_path': request.fs.get_hdfs_path(wf_dir)}
    properties.update(mapping)

    submission = Submission(request.user, coordinator, request.fs, request.jt, properties=properties)
    job_id = submission.run()

    return job_id
  except RestException, ex:
    LOG.exception('Error submitting coordinator')
    raise PopupException(_("Error submitting coordinator %s") % (coordinator,), detail=ex._headers.get('oozie-error-message', ex))

# 需要导入模块: from liboozie.submission2 import Submission [as 别名]
# 或者: from liboozie.submission2.Submission import run [as 别名]
def submit_external_job(request, application_path):
    ParametersFormSet = formset_factory(ParameterForm, extra=0)

    if request.method == "POST":
        params_form = ParametersFormSet(request.POST)

        if params_form.is_valid():
            mapping = dict([(param["name"], param["value"]) for param in params_form.cleaned_data])
            mapping["dryrun"] = request.POST.get("dryrun_checkbox") == "on"
            application_name = os.path.basename(application_path)
            application_class = (
                Bundle
                if application_name == "bundle.xml"
                else Coordinator
                if application_name == "coordinator.xml"
                else get_workflow()
            )
            mapping[application_class.get_application_path_key()] = application_path

            try:
                submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping)
                job_id = submission.run(application_path)
            except RestException, ex:
                detail = ex._headers.get("oozie-error-message", ex)
                if "Max retries exceeded with url" in str(detail):
                    detail = "%s: %s" % (_("The Oozie server is not running"), detail)

                LOG.exception(smart_str(detail))

                raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail)

            request.info(_("Oozie job submitted"))
            view = (
                "list_oozie_bundle"
                if application_name == "bundle.xml"
                else "list_oozie_coordinator"
                if application_name == "coordinator.xml"
                else "list_oozie_workflow"
            )
            return redirect(reverse("oozie:%s" % view, kwargs={"job_id": job_id}))
        else:
            request.error(_("Invalid submission form: %s" % params_form.errors))

# 需要导入模块: from liboozie.submission2 import Submission [as 别名]
# 或者: from liboozie.submission2.Submission import run [as 别名]
  def _schedule_oozie_job(self, workspace_path, collection_name, input_path):
    oozie = get_oozie(self.username)

    properties = {
      "dryrun": "False",
      "zkHost":  zkensemble(),
      # these libs can be installed from here:
      # https://drive.google.com/a/cloudera.com/folderview?id=0B1gZoK8Ae1xXc0sxSkpENWJ3WUU&usp=sharing
      "oozie.libpath": CONFIG_INDEXER_LIBS_PATH.get(),
      "security_enabled": "False",
      "collectionName": collection_name,
      "filePath": input_path,
      "outputDir": "/user/%s/indexer" % self.username,
      "workspacePath": workspace_path,
      'oozie.wf.application.path': "${nameNode}%s" % workspace_path,
      'user.name': self.username
    }

    submission = Submission(self.username, fs=self.fs, properties=properties)
    job_id = submission.run(workspace_path)

    return job_id

# 需要导入模块: from liboozie.submission2 import Submission [as 别名]
# 或者: from liboozie.submission2.Submission import run [as 别名]
def _submit_coordinator(request, coordinator, mapping):
  try:
    wf = coordinator.workflow
    if IS_MULTICLUSTER_ONLY.get() and has_multi_cluster():
      mapping['auto-cluster'] = {
        u'additionalClusterResourceTags': [],
        u'automaticTerminationCondition': u'EMPTY_JOB_QUEUE', #'u'NONE',
        u'cdhVersion': u'CDH514',
        u'clouderaManagerPassword': u'guest',
        u'clouderaManagerUsername': u'guest',
        u'clusterName': u'analytics4', # Add time variable
        u'computeWorkersConfiguration': {
          u'bidUSDPerHr': 0,
          u'groupSize': 0,
          u'useSpot': False
        },
        u'environmentName': u'crn:altus:environments:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:environment:analytics/236ebdda-18bd-428a-9d2b-cd6973d42946',
        u'instanceBootstrapScript': u'',
        u'instanceType': u'm4.xlarge',
        u'jobSubmissionGroupName': u'',
        u'jobs': [{
            u'failureAction': u'INTERRUPT_JOB_QUEUE',
            u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
            u'sparkJob': {
              u'applicationArguments': ['5'],
              u'jars': [u's3a://datawarehouse-customer360/ETL/spark-examples.jar'],
              u'mainClass': u'org.apache.spark.examples.SparkPi'
            }
          },
  #         {
  #           u'failureAction': u'INTERRUPT_JOB_QUEUE',
  #           u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
  #           u'sparkJob': {
  #             u'applicationArguments': ['10'],
  #             u'jars': [u's3a://datawarehouse-customer360/ETL/spark-examples.jar'],
  #             u'mainClass': u'org.apache.spark.examples.SparkPi'
  #           }
  #         },
  #         {
  #           u'failureAction': u'INTERRUPT_JOB_QUEUE',
  #           u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
  #           u'sparkJob': {
  #             u'applicationArguments': [u'filesystems3.conf'],
  #             u'jars': [u's3a://datawarehouse-customer360/ETL/envelope-0.6.0-SNAPSHOT-c6.jar'],
  #             u'mainClass': u'com.cloudera.labs.envelope.EnvelopeMain',
  #             u'sparkArguments': u'--archives=s3a://datawarehouse-customer360/ETL/filesystems3.conf'
  #           }
  #         }
        ],
        u'namespaceName': u'crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:analytics/7ea35fe5-dbc9-4b17-92b1-97a1ab32e410',
        u'publicKey': DEFAULT_PUBLIC_KEY.get(),
        u'serviceType': u'SPARK',
        u'workersConfiguration': {},
        u'workersGroupSize': u'3'
      }
    wf_dir = Submission(request.user, wf, request.fs, request.jt, mapping, local_tz=coordinator.data['properties']['timezone']).deploy()

    properties = {'wf_application_path': request.fs.get_hdfs_path(wf_dir)}
    properties.update(mapping)

    submission = Submission(request.user, coordinator, request.fs, request.jt, properties=properties)
    job_id = submission.run()

    return job_id
  except RestException, ex:
    LOG.exception('Error submitting coordinator')
    raise PopupException(_("Error submitting coordinator %s") % (coordinator,), detail=ex._headers.get('oozie-error-message', ex), error_code=200)