Python processing_job.ProcessingJob类代码示例

def execute(job_id):
    """Executes a job through the plugin system

    Parameters
    ----------
    job_id : str
        The id of the job to execute
    """
    # Create the new job
    job = ProcessingJob(job_id)
    job_dir = join(get_work_base_dir(), job.id)
    software = job.command.software
    plugin_start_script = software.start_script
    plugin_env_script = software.environment_script

    # Get the command to start the plugin
    cmd = '%s "%s" "%s" "%s" "%s" "%s"' % (
        qiita_config.plugin_launcher, plugin_env_script, plugin_start_script,
        qiita_config.base_url, job.id, job_dir)

    # Start the plugin
    std_out, std_err, return_value = system_call(cmd)
    if return_value != 0:
        # Something wrong happened during the plugin start procedure
        job.status = 'error'
        log = LogEntry.create(
            'Runtime',
            "Error starting plugin '%s':\nStd output:%s\nStd error:%s"
            % (software.name, std_out, std_err))
        job.log = log

    def post(self, preprocessed_data_id):
        user = self.current_user
        # make sure user is admin and can therefore actually submit to EBI
        if user.level != 'admin':
            raise HTTPError(403, reason="User %s cannot submit to EBI!" %
                            user.id)
        submission_type = self.get_argument('submission_type')

        if submission_type not in ['ADD', 'MODIFY']:
            raise HTTPError(403, reason="User: %s, %s is not a recognized "
                            "submission type" % (user.id, submission_type))

        study = Artifact(preprocessed_data_id).study
        state = study.ebi_submission_status
        if state == 'submitting':
            message = "Cannot resubmit! Current state is: %s" % state
            self.display_template(preprocessed_data_id, message, 'danger')
        else:
            qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
            cmd = qiita_plugin.get_command('submit_to_EBI')
            params = Parameters.load(
                cmd, values_dict={'artifact': preprocessed_data_id,
                                  'submission_type': submission_type})
            job = ProcessingJob.create(user, params, True)

            r_client.set('ebi_submission_%s' % preprocessed_data_id,
                         dumps({'job_id': job.id, 'is_qiita_job': True}))
            job.submit()

            level = 'success'
            message = 'EBI submission started. Job id: %s' % job.id

            self.redirect("%s/study/description/%d?level=%s&message=%s" % (
                qiita_config.portal_dir, study.id, level, url_escape(message)))

    def post(self, preprocessed_data_id):
        user = self.current_user
        # make sure user is admin and can therefore actually submit to VAMPS
        if user.level != 'admin':
            raise HTTPError(403, "User %s cannot submit to VAMPS!" %
                            user.id)
        msg = ''
        msg_level = 'success'

        plugin = Software.from_name_and_version('Qiita', 'alpha')
        cmd = plugin.get_command('submit_to_VAMPS')
        artifact = Artifact(preprocessed_data_id)

        # Check if the artifact is already being submitted to VAMPS
        is_being_submitted = any(
            [j.status in ('queued', 'running')
             for j in artifact.jobs(cmd=cmd)])

        if is_being_submitted == 'submitting':
            msg = "Cannot resubmit! Data is already being submitted"
            msg_level = 'danger'
            self.display_template(preprocessed_data_id, msg, msg_level)
        else:
            params = Parameters.load(
                cmd, values_dict={'artifact': preprocessed_data_id})
            job = ProcessingJob.create(user, params, True)
            job.submit()
            self.redirect('/study/description/%s' % artifact.study.study_id)

def study_delete_req(study_id, user_id):
    """Delete a given study

    Parameters
    ----------
    study_id : int
        Study id to delete
    user_id : str
        User requesting the deletion

    Returns
    -------
    dict
        Status of deletion, in the format
        {status: status,
         message: message}
    """
    access_error = check_access(study_id, user_id)
    if access_error:
        return access_error

    qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
    cmd = qiita_plugin.get_command('delete_study')
    params = Parameters.load(cmd, values_dict={'study': study_id})
    job = ProcessingJob.create(User(user_id), params, True)
    # Store the job id attaching it to the sample template id
    r_client.set(STUDY_KEY_FORMAT % study_id,
                 dumps({'job_id': job.id}))

    job.submit()

    return {'status': 'success', 'message': ''}

def artifact_post_req(user, artifact_id):
    """Deletes the artifact

    Parameters
    ----------
    user : qiita_db.user.User
        The user requesting the action
    artifact_id : int
        Id of the artifact being deleted
    """
    artifact_id = int(artifact_id)
    artifact = Artifact(artifact_id)
    check_artifact_access(user, artifact)

    analysis = artifact.analysis

    if analysis:
        # Do something when deleting in the analysis part to keep track of it
        redis_key = "analysis_%s" % analysis.id
    else:
        pt_id = artifact.prep_templates[0].id
        redis_key = PREP_TEMPLATE_KEY_FORMAT % pt_id

    qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
    cmd = qiita_plugin.get_command('delete_artifact')
    params = Parameters.load(cmd, values_dict={'artifact': artifact_id})
    job = ProcessingJob.create(user, params, True)

    r_client.set(redis_key, dumps({'job_id': job.id, 'is_qiita_job': True}))

    job.submit()

    return {'job': job.id}

 def _create_job(self, cmd_name, values_dict):
     self.user = User('[email protected]')
     qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
     cmd = qiita_plugin.get_command(cmd_name)
     params = Parameters.load(cmd, values_dict=values_dict)
     job = ProcessingJob.create(self.user, params, True)
     job._set_status('queued')
     return job

def sample_template_handler_post_request(study_id, user, filepath,
                                         data_type=None, direct_upload=False):
    """Creates a new sample template

    Parameters
    ----------
    study_id: int
        The study to add the sample information
    user: qiita_db.user import User
        The user performing the request
    filepath: str
        The path to the sample template file
    data_type: str, optional
        If filepath is a QIIME mapping file, the data type of the prep
        information file
    direct_upload: boolean, optional
        If filepath is a direct upload; if False we need to process the
        filepath as part of the study upload folder

    Returns
    -------
    dict of {'job': str}
        job: the id of the job adding the sample information to the study

    Raises
    ------
    HTTPError
        404 if the filepath doesn't exist
    """
    # Check if the current user has access to the study
    sample_template_checks(study_id, user)

    # Check if the file exists
    if not direct_upload:
        fp_rsp = check_fp(study_id, filepath)
        if fp_rsp['status'] != 'success':
            raise HTTPError(404, reason='Filepath not found')
        filepath = fp_rsp['file']

    is_mapping_file = looks_like_qiime_mapping_file(filepath)
    if is_mapping_file and not data_type:
        raise HTTPError(400, reason='Please, choose a data type if uploading '
                        'a QIIME mapping file')

    qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
    cmd = qiita_plugin.get_command('create_sample_template')
    params = Parameters.load(
        cmd, values_dict={'fp': filepath, 'study_id': study_id,
                          'is_mapping_file': is_mapping_file,
                          'data_type': data_type})
    job = ProcessingJob.create(user, params, True)
    r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
                 dumps({'job_id': job.id}))
    job.submit()
    return {'job': job.id}

    def test_artifact_summary_post_request(self):
        # No access
        with self.assertRaises(QiitaHTTPError):
            artifact_summary_post_request(User('[email protected]'), 1)

        # Returns already existing job
        job = ProcessingJob.create(
            User('[email protected]'),
            Parameters.load(Command(7), values_dict={'input_data': 2})
        )
        job._set_status('queued')
        obs = artifact_summary_post_request(User('[email protected]'), 2)
        exp = {'job': [job.id, 'queued', None]}
        self.assertEqual(obs, exp)

    def post(self):
        analysis_id = int(self.get_argument('analysis_id'))

        user = self.current_user
        check_analysis_access(user, Analysis(analysis_id))

        qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
        cmd = qiita_plugin.get_command('delete_analysis')
        params = Parameters.load(cmd, values_dict={'analysis_id': analysis_id})
        job = ProcessingJob.create(user, params, True)
        # Store the job id attaching it to the sample template id
        r_client.set('analysis_delete_%d' % analysis_id,
                     dumps({'job_id': job.id}))
        job.submit()

        self.redirect("%s/analysis/list/" % (qiita_config.portal_dir))

def artifact_summary_post_request(user_id, artifact_id):
    """Launches the HTML summary generation and returns the job information

    Parameters
    ----------
    user_id : str
        The user making the request
    artifact_id : int or str
        The artifact id

    Returns
    -------
    dict of objects
        A dictionary containing the artifact summary information
        {'status': str,
         'message': str,
         'job': list of [str, str, str]}
    """
    artifact_id = int(artifact_id)
    artifact = Artifact(artifact_id)

    access_error = check_access(artifact.study.id, user_id)
    if access_error:
        return access_error

    # Check if the summary is being generated or has been already generated
    command = Command.get_html_generator(artifact.artifact_type)
    jobs = artifact.jobs(cmd=command)
    jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
    if jobs:
        # The HTML summary is either being generated or already generated.
        # Return the information of that job so we only generate the HTML
        # once
        job = jobs[0]
    else:
        # Create a new job to generate the HTML summary and return the newly
        # created job information
        job = ProcessingJob.create(
            User(user_id),
            Parameters.load(command, values_dict={'input_data': artifact_id}))
        job.submit()

    return {'status': 'success',
            'message': '',
            'job': [job.id, job.status, job.step]}

    def post(self, study_id):
        method = self.get_argument('remote-request-type')
        url = self.get_argument('inputURL')
        ssh_key = self.request.files['ssh-key'][0]['body']
        status = 'success'
        message = ''

        try:
            study = Study(int(study_id))
        except QiitaDBUnknownIDError:
            raise HTTPError(404, reason="Study %s does not exist" % study_id)
        check_access(
            self.current_user, study, no_public=True, raise_error=True)

        _, upload_folder = get_mountpoint("uploads")[0]
        upload_folder = join(upload_folder, study_id)
        ssh_key_fp = join(upload_folder, '.key.txt')

        create_nested_path(upload_folder)

        with open(ssh_key_fp, 'w') as f:
            f.write(ssh_key)

        qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
        if method == 'list':
            cmd = qiita_plugin.get_command('list_remote_files')
            params = Parameters.load(cmd, values_dict={
                'url': url, 'private_key': ssh_key_fp, 'study_id': study_id})
        elif method == 'transfer':
            cmd = qiita_plugin.get_command('download_remote_files')
            params = Parameters.load(cmd, values_dict={
                'url': url, 'private_key': ssh_key_fp,
                'destination': upload_folder})
        else:
            status = 'error'
            message = 'Not a valid method'

        if status == 'success':
            job = ProcessingJob.create(self.current_user, params, True)
            job.submit()
            r_client.set(
                UPLOAD_STUDY_FORMAT % study_id, dumps({'job_id': job.id}))

        self.write({'status': status, 'message': message})

def artifact_summary_post_request(user, artifact_id):
    """Launches the HTML summary generation and returns the job information

    Parameters
    ----------
    user : qiita_db.user.User
        The user making the request
    artifact_id : int or str
        The artifact id

    Returns
    -------
    dict of objects
        A dictionary containing the job summary information
        {'job': [str, str, str]}
    """
    artifact_id = int(artifact_id)
    artifact = Artifact(artifact_id)

    check_artifact_access(user, artifact)

    # Check if the summary is being generated or has been already generated
    command = Command.get_html_generator(artifact.artifact_type)
    jobs = artifact.jobs(cmd=command)
    jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']]
    if jobs:
        # The HTML summary is either being generated or already generated.
        # Return the information of that job so we only generate the HTML
        # once - Magic number 0 -> we are ensuring that there is only one
        # job generating the summary, so we can use the index 0 to access to
        # that job
        job = jobs[0]
    else:
        # Create a new job to generate the HTML summary and return the newly
        # created job information
        job = ProcessingJob.create(user, Parameters.load(
            command, values_dict={'input_data': artifact_id}), True)
        job.submit()

    return {'job': [job.id, job.status, job.step]}

def _submit(ctx, user, parameters):
    """Submit a plugin job to a cluster

    Parameters
    ----------
    ctx : qiita_db.ware.Dispatch
        A Dispatch object to submit through
    user : qiita_db.user.User
        The user doing the submission
    parameters : qiita_db.software.Parameters
        The parameters of the job

    Returns
    -------
    str
        The job id
    """
    job = ProcessingJob.create(user, parameters)
    redis_deets = {'job_id': job.id, 'pubsub': user.id,
                   'messages': user.id + ':messages'}
    ctx.submit_async(_redis_wrap, execute, redis_deets, job.id)
    return job.id

def sample_template_handler_delete_request(study_id, user):
    """Deletes the sample template

    Parameters
    ----------
    study_id: int
        The study to delete the sample information
    user: qiita_db.user
        The user performing the request

    Returns
    -------
    dict of {'job': str}
        job: the id of the job deleting the sample information to the study

    Raises
    ------
    HTTPError
        404 If the sample template doesn't exist
    """
    # Check if the current user has access to the study and if the sample
    # template exists
    sample_template_checks(study_id, user, check_exists=True)

    qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
    cmd = qiita_plugin.get_command('delete_sample_template')
    params = Parameters.load(cmd, values_dict={'study': int(study_id)})
    job = ProcessingJob.create(user, params, True)

    # Store the job if deleteing the sample template
    r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
                 dumps({'job_id': job.id}))

    job.submit()

    return {'job': job.id}

def sample_template_handler_patch_request(user, req_op, req_path,
                                          req_value=None, req_from=None):
    """Patches the sample template

    Parameters
    ----------
    user: qiita_db.user.User
        The user performing the request
    req_op : str
        The operation to perform on the sample template
    req_path : str
        The path to the attribute to patch
    req_value : str, optional
        The new value
    req_from : str, optional
        The original path of the element

    Returns
    -------

    Raises
    ------
    HTTPError
        400 If the path parameter doens't follow the expected format
        400 If the given operation is not supported
    """
    req_path = [v for v in req_path.split('/') if v]
    # At this point we know the path should be at least length 2
    if len(req_path) < 2:
        raise HTTPError(400, reason='Incorrect path parameter')

    study_id = int(req_path[0])
    # Check if the current user has access to the study and if the sample
    # template exists
    sample_template_checks(study_id, user, check_exists=True)

    if req_op == 'remove':
        # Path format
        # column: study_id/columns/column_name
        # sample: study_id/samples/sample_id
        if len(req_path) != 3:
            raise HTTPError(400, reason='Incorrect path parameter')

        attribute = req_path[1]
        attr_id = req_path[2]

        qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
        cmd = qiita_plugin.get_command('delete_sample_or_column')
        params = Parameters.load(
            cmd, values_dict={'obj_class': 'SampleTemplate',
                              'obj_id': study_id,
                              'sample_or_col': attribute,
                              'name': attr_id})
        job = ProcessingJob.create(user, params, True)
        # Store the job id attaching it to the sample template id
        r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
                     dumps({'job_id': job.id}))
        job.submit()
        return {'job': job.id}
    elif req_op == 'replace':
        # WARNING: Although the patch operation is a replace, is not a full
        # true replace. A replace is in theory equivalent to a remove + add.
        # In this case, the replace operation doesn't necessarily removes
        # anything (e.g. when only new columns/samples are being added to the)
        # sample information.
        # Path format: study_id/data
        # Forcing to specify data for extensibility. In the future we may want
        # to use this function to replace other elements of the sample
        # information
        if len(req_path) != 2:
            raise HTTPError(400, reason='Incorrect path parameter')

        attribute = req_path[1]

        if attribute == 'data':
            # Update the sample information
            if req_value is None:
                raise HTTPError(400, reason="Value is required when updating "
                                "sample information")

            # Check if the file exists
            fp_rsp = check_fp(study_id, req_value)
            if fp_rsp['status'] != 'success':
                raise HTTPError(404, reason='Filepath not found')
            filepath = fp_rsp['file']

            qiita_plugin = Software.from_name_and_version('Qiita', 'alpha')
            cmd = qiita_plugin.get_command('update_sample_template')
            params = Parameters.load(
                cmd, values_dict={'study': study_id,
                                  'template_fp': filepath})
            job = ProcessingJob.create(user, params, True)

            # Store the job id attaching it to the sample template id
            r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id,
                         dumps({'job_id': job.id}))

            job.submit()
            return {'job': job.id}
        else:
#.........这里部分代码省略.........