Python EDUtilsParallel.EDUtilsParallel类代码示例

    def start(self, _strXmlInput):
        """
        Launch EDNA with the given XML stream
        @param _strXmlInput:  XML to be passed to the plugin
        @type  _strXmlInput: python string representing the XML data structure
        """
        if (_strXmlInput is None) or (_strXmlInput == "") :
            return
        #This is a trick to work-around bug #463:
        #  Run the fist thread alone (delay the second, third, ...)
        #    semaphore._Semaphore__value is the current value of the value, unfortunatly it is a protected value without getter 
        #    I need the value of the semaphore to guess if the current call is the first or not.
        #    Nota semaphore are decreased in value from self.__iNbThreads to 0. When Zero, the semaphore is blocking.
        # Them all other, limited by the semaphore.

        if self.__bIsFirstExecute:
            sys.stdout.write("Waiting for first thread to initialize ....")
            while EDUtilsParallel.getNbRunning() > 0:
                time.sleep(0.5)
                sys.stdout.write(".")
        EDUtilsParallel.semaphoreNbThreadsAcquire()

        edPlugin = EDParallelExecute.__edFactoryPlugin.loadPlugin(self.__strPluginName)

        if (edPlugin is not None):

            edPlugin.setDataInput(_strXmlInput)
            edPlugin.connectSUCCESS(self.successPluginExecution)
            edPlugin.connectFAILURE(self.failurePluginExecution)
            edPlugin.execute()
            #edPlugin.executeSynchronous()
        else:
            EDUtilsParallel.semaphoreNbThreadsRelease()
            self.__bIsFirstExecute = False
            EDVerbose.screen("ERROR! Plugin not found : " + self.__strPluginName)

    def preProcess(self):
        """
        PreProcess of the execution test: download a set of images  from http://www.edna-site.org
        and remove any existing output file 
        """
        EDTestCasePluginExecute.preProcess(self)
        self.loadTestImage([ "bioSaxsMask.edf", "bioSaxsNormalized.edf", "bioSaxsIntegratedv1_3.dat", "bioSaxsCorrected.edf"])
        strExpectedOutput = self.readAndParseFile (self.getReferenceDataOutputFile())
        EDVerbose.DEBUG("strExpectedOutput:" + strExpectedOutput)
        xsDataResultReference = XSDataResultBioSaxsAzimutIntv1_0.parseString(strExpectedOutput)
        self.integratedCurve = xsDataResultReference.getIntegratedCurve().getPath().value
        EDVerbose.DEBUG("Output file is %s" % self.integratedCurve)
        if not os.path.isdir(os.path.dirname(self.integratedCurve)):
            os.makedirs(os.path.dirname(self.integratedCurve))
        if os.path.isfile(self.integratedCurve):
            EDVerbose.DEBUG(" Output Integrated Curve file exists %s, I will remove it" % self.integratedCurve)
            os.remove(self.integratedCurve)

        EDUtilsParallel.initializeNbThread()
#        self.correctedImage = xsDataResultReference.getCorrectedImage().getPath().value
#        EDVerbose.DEBUG("Output Corrected Image file is %s" % self.correctedImage)
#        if not os.path.isdir(os.path.dirname(self.correctedImage)):
#            os.makedirs(os.path.dirname(self.correctedImage))
#        if os.path.isfile(self.correctedImage):
#            EDVerbose.DEBUG(" Output Corrected Image file exists %s, I will remove it" % self.correctedImage)
#            os.remove(self.correctedImage)
        if not pyFAI.version.startswith("0.7"):
            EDVerbose.ERROR('pyFAI is not the right version, tested only with 0.7.2, here running version %s' % pyFAI.version)

    def __init__(self, _strPluginName, _functXMLin, \
                  _functXMLout=None, _functXMLerr=None, \
                  _iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
        """
        This is the constructor of the edna plugin launcher.
        
        @param _strPluginName: the name of the ENDA plugin
        @type  _strPluginName: python string
        
        @param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin. 
        @type  _functXMLin: python function
        
        @param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
        @type  _functXMLOut: python function
         
        @param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
        @type  _functXMLErr: python function 
        
        @param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores  will be auto-detected. 
        @type  _iNbThreads: python integer
        
        @param _fDelay: The delay in seconds between two directories analysis 
        @type  _fDelay: python float
        
        @param _bVerbose:  Do you want the EDNA plugin execution to be verbose ?
        @type  _bVerbose: boolean

        @param _bDebug:  Do you want EDNA plugin execution debug output (OBS! very verbose) ?
        @type  _bDebug: boolean
        """
        EDLogging.__init__(self)
        self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
        EDUtilsParallel.initializeNbThread(self.__iNbThreads)
################################################################################
# #We are not using the one from EDUtilsParallel to leave it to control the number of  execPlugins.
################################################################################
        self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
        self.__strPluginName = _strPluginName
        self.__functXMLin = _functXMLin
        self.__functXMLout = _functXMLout
        self.__functXMLerr = _functXMLerr
        self.__strCurrWorkDir = os.getcwd()
        self.__strTempDir = None
        self.__listInputPaths = []
        self.__dictCurrentlyRunning = {}
        if _bVerbose is not None:
            if _bVerbose:
                self.setVerboseDebugOn()
            else:
                self.setVerboseOff()
        if _bDebug is not None:
            if _bDebug:
                self.setVerboseDebugOn()
            else:
                self.setVerboseDebugOff()
        self.__fDelay = _fDelay #default delay between two directory checks.
        self.__bQuit = False    # To check if we should quit the application
        self.__bIsFirstExecute = True
        signal.signal(signal.SIGTERM, self.handleKill)
        signal.signal(signal.SIGINT, self.handleKill)

 def finallyProcess(self, _edObject=None):
     """
     after processing of the plugin:
     Release a CPU resource by releasing the semaphore
     """
     self.DEBUG("Release semaphore nbCPU by plugin %s" % self.getPluginName())
     EDUtilsParallel.semaphoreNbThreadsRelease()
     EDPlugin.finallyProcess(self, _edObject)

    def __init__(self, _strPluginName, _strPluginDir=None, _strTestName=None):
        """
        Initialize the Plugin Unit test
        """
        EDTestCasePlugin.__init__(self, _strPluginName, _strPluginDir, _strTestName)
        self.__edPlugin = None
#        To track dead locks
        EDUtilsParallel.initializeNbThread(1)

 def finallyProcess(self, _edObject=None):
     """
     after processing of the plugin:
     Release a CPU resource by releasing the semaphore
     """
     EDVerbose.DEBUG("Release semaphore nbCPU by plugin %s, currently value: %s" % (self.getPluginName(), EDUtilsParallel.getSemaphoreValue()))
     EDUtilsParallel.semaphoreNbThreadsRelease()
     EDPlugin.finallyProcess(self, _edObject)

 def preProcess(self, _edObject=None):
     """
     preProcess of the plugin:
     Ensure a CPU resource is available for the processing by acquiring a semaphore
     """
     EDVerbose.DEBUG("Acquire semaphore nbCPU by plugin %s, currently value: %s" % (self.getPluginName(), EDUtilsParallel.getSemaphoreValue()))
     EDUtilsParallel.semaphoreNbThreadsAcquire()
     EDPlugin.preProcess(self, _edObject)

 def preProcess(self, _edObject=None):
     """
     preProcess of the plugin:
     Ensure a CPU resource is available for the processing by acquiring a semaphore
     """
     self.DEBUG("Acquire semaphore nbCPU by plugin %s" % self.getPluginName())
     EDUtilsParallel.semaphoreNbThreadsAcquire()
     EDPlugin.preProcess(self, _edObject)

 def unitTestDetectNumberOfCPUs(self):
     """
     test the execution of detectNumberOfCPUs
     """
     iNbCPU = EDUtilsParallel.detectNumberOfCPUs()
     EDVerbose.unitTest("Detection of the number of Cores: %s" % iNbCPU)
     EDAssert.equal(types.IntType, type(iNbCPU), "Number of CPU is an integer")
     iNbCPU = EDUtilsParallel.detectNumberOfCPUs(1) #limited to 1
     EDAssert.equal(1, iNbCPU, "Limit number of CPU")
     iNbCPU = EDUtilsParallel.detectNumberOfCPUs(100, True) #forced to 100
     EDAssert.equal(100, iNbCPU, "Force number of CPU")

 def failurePluginExecution(self, _edObject=None):
     """
     Method called when the execution of the plugin failed 
     """
     EDUtilsParallel.semaphoreNbThreadsRelease()
     self.__bIsFirstExecute = False
     if self.__functXMLerr is None:
         EDVerbose.screen("Plugin %s execution ended with failure" % self.__strPluginName)
     else:
         self.__semaphoreErr.acquire()
         self.__functXMLerr(_edObject.dataInput.marshal())
         self.__semaphoreErr.release()

 def successPluginExecution(self, _edObject=None):
     """
     Method called when the execution of the plugin succeeds 
     """
     EDUtilsParallel.semaphoreNbThreadsRelease()
     self.__bIsFirstExecute = False
     if self.__functXMLout is None:
         EDVerbose.screen("Plugin %s execution ended with success" % self.__strPluginName)
     else:
         self.__semaphoreOut.acquire()
         self.__functXMLout(_edObject.dataInput.marshal(), _edObject.getDataOutput().marshal())
         self.__semaphoreOut.release()

 def cleanUp(self, _edPlugin):
     """
     Cleans up some empty directories 
     (mainly working directories that were created automatically, but not filled as Unit tests do not execute the plugin)
     """
     workingDirectory = _edPlugin.getWorkingDirectory()
     if(workingDirectory is not None):
         fileList = glob.glob(os.path.join(workingDirectory, "*"))
         if(len(fileList) == 0):
             EDVerbose.DEBUG("Deleting " + workingDirectory + " ...")
             shutil.rmtree(workingDirectory)
     EDUtilsParallel.semaphoreNbThreadsRelease()

 def executeControlledPlugins(self):
     """
     Execute all plugins under control: 
     """
     bAllFinished = False
     while not bAllFinished:
         #acquire all semaphores to be sure no plugins are under configuration !
         for sem in self.lstSem:
             with sem:
                 pass
         if self.queue.empty():
             self.synchronizePlugins()
             bAllFinished = self.queue.empty()
         else:
             while not self.queue.empty():
                 try:
                     plugin = self.queue.get()
                 except Exception:
                     self.WARNING("In EDPluginControlAlignStackv1_0, exception in self.queue.get()")
                     break
                 else:
                     #this is a hack to prevent thousands of threads to be launched at once.
                     with EDUtilsParallel.getSemaphoreNbThreads():
                         pass
                     plugin.execute()
             self.synchronizePlugins()

 def waitForAllProcessToFinish(self):
     """
     as it names says, this method waits for all plug-ins which are currently running to finish before returning.
     """
     sys.stderr.write("Waiting for launched jobs to finish .")
     while (EDUtilsParallel.getNbRunning() > 0):
         time.sleep(1)
         sys.stderr.write(".")
     sys.stderr.write("Done.\n")

    def __init__(self, strPluginName, iNbCpu=None):
        EDLogging.__init__(self)
        self.pluginName = strPluginName
        self.startTime = time.time()
        try:
            self.iNbCpu = int(iNbCpu)
        except:
            self.iNbCpu = EDUtilsParallel.detectNumberOfCPUs()

        self.screen("Initializing Reprocess with max %i jobs in parallel." % self.iNbCpu)
        self.__semaphoreNbThreads = Semaphore(self.iNbCpu)
        EDUtilsParallel.initializeNbThread(self.iNbCpu)
        self.jobQueue = Queue()
        self.processingSem = Semaphore()
        self.statLock = Semaphore()
        self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
        self.lastFailure = "No job Failed (yet)"
        self.lastSuccess = "No job succeeded (yet)"