Python Popen.wait方法代碼示例

# 需要導入模塊: from subprocess32 import Popen [as 別名]
# 或者: from subprocess32.Popen import wait [as 別名]
    def execute_command_locally(self, command_obj, directory, job_log):
        # create a temporary log file
        temp_log_file = None
        if not command_obj.dont_log_output:
            temp_log_filename = get_absolute_path("tmp.pipelinelog." + command_obj.command_id + "." + str(random.randint(10*10, 10**11 - 1)), directory, allow_compressed_version=False)
            temp_log_file = open(temp_log_filename, "w+")
            #temp_log_file = tempfile.NamedTemporaryFile(bufsize=0) #

        spawned_process = Popen(command_obj.command, bufsize = 0, shell=True, cwd=directory, stdout=temp_log_file, stderr=temp_log_file, executable=self.shell)

        if temp_log_file:
            # while the job is running, continually read from the temp_log_file and copy this to the job_log
            self.copy_command_output_to_log(command_obj, spawned_process, temp_log_file, job_log)

            temp_log_file.close()
            os.remove(temp_log_filename)
            #os.remove(temp_file_path)
        else:
            spawned_process.wait()

        if spawned_process.returncode is not None and spawned_process.returncode != 0:
            raise Exception("Non-zero return code: " + str(spawned_process.returncode))

# 需要導入模塊: from subprocess32 import Popen [as 別名]
# 或者: from subprocess32.Popen import wait [as 別名]
    def wait(self, timeout=None):
        """Wait for the started process to complete.

        "timeout" is a floating point number of seconds after
            which to timeout.  Default is None, which is to never timeout.

        If the wait time's out it will raise a ProcessError. Otherwise it
        will return the child's exit value. Note that in the case of a timeout,
        the process is still running. Use kill() to forcibly stop the process.
        """
        if timeout is None or timeout < 0:
            # Use the parent call.
            try:
                return Popen.wait(self)
            except OSError as ex:
                # If the process has already ended, that is fine. This is
                # possible when wait is called from a different thread.
                if ex.errno != 10:  # No child process
                    raise
                return self.returncode

        # We poll for the retval, as we cannot rely on self.__hasTerminated
        # to be called, as there are some code paths that do not trigger it.
        # The accuracy of this wait call is between 0.1 and 1 second.
        time_now = time.time()
        time_end = time_now + timeout
        # These values will be used to incrementally increase the wait period
        # of the polling check, starting from the end of the list and working
        # towards the front. This is to avoid waiting for a long period on
        # processes that finish quickly, see bug 80794.
        time_wait_values = [1.0, 0.5, 0.2, 0.1]
        while time_now < time_end:
            result = self.poll()
            if result is not None:
                return result
            # We use hasTerminated here to get a faster notification.
            self.__hasTerminated.acquire()
            if time_wait_values:
                wait_period = time_wait_values.pop()
            self.__hasTerminated.wait(wait_period)
            self.__hasTerminated.release()
            time_now = time.time()
        # last chance
        result = self.poll()
        if result is not None:
            return result

        raise ProcessError("Process timeout: waited %d seconds, "
                           "process not yet finished." % (timeout,),
                           WAIT_TIMEOUT)

# 需要導入模塊: from subprocess32 import Popen [as 別名]
# 或者: from subprocess32.Popen import wait [as 別名]
def run2(command, check=True, timeout=None, *args, **kwargs):
    ''' Run a command.

        If check=True (the default),
        then if return code is not zero or there is stderr output,
        raise CalledProcessError. Return any output in the exception.

        If timeout (in seconds) is set and command times out, raise TimeoutError. '''

    ''' Parts from subprocess32.check_output(). '''

    raise Exception('Deprecated. Use the sh module.')

    # use subprocess32 for timeout
    from subprocess32 import Popen, CalledProcessError, TimeoutExpired

    process = Popen(command, stdout=stdout, stderr=stderr, *args, **kwargs)
    try:
        process.wait(timeout=timeout)
    except TimeoutExpired:
        print('TimeoutExpired') #DEBUG
        #print('stdout: %s, (%d)' % (str(stdout), len(str(stdout)))) #DEBUG
        #print('stderr: %s, (%d)' % (str(stderr), len(str(stderr)))) #DEBUG
        try:
            process.kill()
            process.wait()
        finally:
            print('after kill/wait') #DEBUG
            #print('stdout: %s, (%d)' % (str(stdout), len(str(stdout)))) #DEBUG
            #print('stderr: %s, (%d)' % (str(stderr), len(str(stderr)))) #DEBUG
            raise TimeoutExpired(process.args, timeout)

    if check:
        retcode = process.poll()
        if retcode:
            raise CalledProcessError(retcode, process.args)

# 需要導入模塊: from subprocess32 import Popen [as 別名]
# 或者: from subprocess32.Popen import wait [as 別名]
    def wait(self, timeout=None):
        """Wait for the started process to complete.

        "timeout" is a floating point number of seconds after
            which to timeout.  Default is None, which is to never timeout.

        If the wait time's out it will raise a ProcessError. Otherwise it
        will return the child's exit value. Note that in the case of a timeout,
        the process is still running. Use kill() to forcibly stop the process.
        """
        if timeout is None or timeout < 0:
            # Use the parent call.
            try:
                return Popen.wait(self)
            except OSError, ex:
                # If the process has already ended, that is fine. This is
                # possible when wait is called from a different thread.
                if ex.errno != 10:  # No child process
                    raise
                return self.returncode

# 需要導入模塊: from subprocess32 import Popen [as 別名]
# 或者: from subprocess32.Popen import wait [as 別名]
    def execute_command_using_SGE(self, command_obj, directory, job_log):
        # TODO put this in config file
        qsub_executable = "/opt/gridengine/bin/lx-amd64/qsub"
        #qsub_executable = "qsub"
        tempdir = tempfile.gettempdir() # "/db/ngs-bioinfo/tmp"

        #if not os.path.isfile(qsub_executable):
        #	raise Exception("Cannot submit job to SGE. %(qsub_executable)s command not found." % locals())


        num_cores = min(command_obj.num_threads_used, self.SGE_max_cores_to_reserve)

        # compute SGE job name
        sge_job_name = ""
        if job_log.name:
            sge_job_name += job_log.name
        if command_obj.name:
            if sge_job_name:
                sge_job_name += " " # add spacer
            sge_job_name += command_obj.name

        if sge_job_name:
            sge_job_name = sge_job_name.replace(" ", "_")
        else:
            sge_job_name = os.path.basename(command_obj.command.split(" ")[0])

        # create a temporary log file
        temp_log_filename = None
        if not command_obj.dont_log_output:
            temp_log_filename = get_absolute_path("tmp.pipelinelog." + command_obj.command_id + "." + str(random.randint(10*10, 10**11 - 1)), directory, allow_compressed_version=False)
            temp_log_file = open(temp_log_filename, "a+") # create log file
            temp_log_file.close()
            temp_log_file = open(temp_log_filename, "r") # open this file for reading



        qsub_script_name = os.path.join(tempdir, sge_job_name + ".%d.sh" % random.randint(10**9, 10**10 - 1))
        f = open( qsub_script_name, "w+")
        #f.write("source ~/.bashrc;" + "\n")
        f.write("/db/ngs-bioinfo/prog/bash-4.2/bin/bash \n") # make sure the shell is a bash shell
        f.write("echo Running on $HOSTNAME. `uptime | cut -c 40-`, cpus: `cat /proc/cpuinfo | grep processor | wc -l`, `cat /proc/meminfo | grep MemTotal`, `cat /proc/meminfo | grep MemFree`;" + "\n")
        f.write("cd " + directory + "\n")
        f.write(command_obj.command + "\n")
        f.close()

        os.system("chmod 777 " + qsub_script_name)
        qsub_command = qsub_executable + " "
        qsub_command += " -V "  # import all environment
        if num_cores:
            qsub_command += " -pe orte " + str(num_cores) + "  "

        qsub_command += " -sync y "  # write err to out
        qsub_command += " -j y "  # write err to out
        qsub_command += " -o " + temp_log_filename + " "
        #qsub_command += " -e " + stderr + " "
        qsub_command += qsub_script_name # + " >& /dev/null"


        spawned_qsub_process = Popen(qsub_command, bufsize = 0, shell=True, cwd=directory)

        # while the job is running, continually read from the temp_log_file and copy this to the job_log
        if temp_log_filename:
            self.copy_command_output_to_log(command_obj, spawned_qsub_process, temp_log_file, job_log)

            temp_log_file.close()
            os.remove(temp_log_filename)
        else:
            spawned_qsub_process.wait()

        if spawned_qsub_process.returncode is not None and spawned_qsub_process.returncode != 0:
            raise Exception("Non-zero return code: " + str(spawned_qsub_process.returncode))


        os.remove(qsub_script_name)

# 需要導入模塊: from subprocess32 import Popen [as 別名]
# 或者: from subprocess32.Popen import wait [as 別名]
    def execute_command_using_LSF(self, command_obj, directory, job_log):
        # TODO put this in config file
        tempdir = "/broad/hptmp/weisburd/tmp/" #tempfile.gettempdir()

        #if not os.path.isfile(qsub_executable):
        #	raise Exception("Cannot submit job to SGE. %(qsub_executable)s command not found." % locals())


        num_cores = min(command_obj.num_threads_used, self.SGE_max_cores_to_reserve)

        # compute SGE job name
        lsf_job_name = ""
        if job_log.name:
            lsf_job_name += job_log.name
        if command_obj.name:
            if lsf_job_name:
                lsf_job_name += " " # add spacer
            lsf_job_name += command_obj.name

        if lsf_job_name:
            lsf_job_name = lsf_job_name.replace(" ", "_")
        else:
            lsf_job_name = os.path.basename(command_obj.command.split(" ")[0])

        # create a temporary log file
        temp_log_filename = None
        if not command_obj.dont_log_output:
            temp_log_filename = get_absolute_path("tmp.pipelinelog." + command_obj.command_id + "." + str(random.randint(10*10, 10**11 - 1)), directory, allow_compressed_version=False)
            temp_log_file = open(temp_log_filename, "a+") # create log file
            temp_log_file.close()
            temp_log_file = open(temp_log_filename, "r") # open this file for reading



        qsub_script_name = os.path.join(tempdir, lsf_job_name + ".%d.sh" % random.randint(10**9, 10**10 - 1))

        f = open( qsub_script_name, "w+")
        #f.write("source ~/.bashrc;" + "\n")
        #f.write("/db/ngs-bioinfo/prog/bash-4.2/bin/bash \n") # make sure the shell is a bash shell
        f.write("echo Running on $HOSTNAME. `uptime | cut -c 40-`, cpus: `cat /proc/cpuinfo | grep processor | wc -l`, `cat /proc/meminfo | grep MemTotal`, `cat /proc/meminfo | grep MemFree`;" + "\n")
        f.write("cd " + directory + "\n")
        f.write(command_obj.command + "\n")
        f.close()

        os.system("chmod 777 " + qsub_script_name)
        qsub_command = "bsub "
        qsub_command += " -K " # make hte command run interactively
        qsub_command += " -q week "  # hour or week
        qsub_command += " -J %s " % lsf_job_name
        qsub_command += " -P project "  # email when done
        if num_cores:
            qsub_command += " -n %s " % num_cores
        qsub_command  += " -R rusage[mem=%s] " % command_obj.memory_in_gb # memory usage
        qsub_command += " -o " + temp_log_filename + " "  # this also capture stderr (since stdout is not specified)
        qsub_command += qsub_script_name
        qsub_command += ' > /dev/null'

        spawned_qsub_process = Popen(qsub_command, bufsize = 0, shell=True, cwd=directory)

        # while the job is running, continually read from the temp_log_file and copy this to the job_log
        if temp_log_filename:
            self.copy_command_output_to_log(command_obj, spawned_qsub_process, temp_log_file, job_log)

            temp_log_file.close()
            os.remove(temp_log_filename)
        else:
            spawned_qsub_process.wait()

        if spawned_qsub_process.returncode is not None and spawned_qsub_process.returncode != 0:
            raise Exception("Non-zero return code: " + str(spawned_qsub_process.returncode))


        os.remove(qsub_script_name)