Python EDUtilsParallel.detectNumberOfCPUs方法代码示例

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import detectNumberOfCPUs [as 别名]
 def unitTestDetectNumberOfCPUs(self):
     """
     test the execution of detectNumberOfCPUs
     """
     iNbCPU = EDUtilsParallel.detectNumberOfCPUs()
     EDVerbose.unitTest("Detection of the number of Cores: %s" % iNbCPU)
     EDAssert.equal(types.IntType, type(iNbCPU), "Number of CPU is an integer")
     iNbCPU = EDUtilsParallel.detectNumberOfCPUs(1) #limited to 1
     EDAssert.equal(1, iNbCPU, "Limit number of CPU")
     iNbCPU = EDUtilsParallel.detectNumberOfCPUs(100, True) #forced to 100
     EDAssert.equal(100, iNbCPU, "Force number of CPU")

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import detectNumberOfCPUs [as 别名]
    def __init__(self, _strPluginName, _functXMLin, \
                  _functXMLout=None, _functXMLerr=None, \
                  _iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
        """
        This is the constructor of the edna plugin launcher.
        
        @param _strPluginName: the name of the ENDA plugin
        @type  _strPluginName: python string
        
        @param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin. 
        @type  _functXMLin: python function
        
        @param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
        @type  _functXMLOut: python function
         
        @param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
        @type  _functXMLErr: python function 
        
        @param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores  will be auto-detected. 
        @type  _iNbThreads: python integer
        
        @param _fDelay: The delay in seconds between two directories analysis 
        @type  _fDelay: python float
        
        @param _bVerbose:  Do you want the EDNA plugin execution to be verbose ?
        @type  _bVerbose: boolean

        @param _bDebug:  Do you want EDNA plugin execution debug output (OBS! very verbose) ?
        @type  _bDebug: boolean
        """
        EDLogging.__init__(self)
        self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
        EDUtilsParallel.initializeNbThread(self.__iNbThreads)
################################################################################
# #We are not using the one from EDUtilsParallel to leave it to control the number of  execPlugins.
################################################################################
        self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
        self.__strPluginName = _strPluginName
        self.__functXMLin = _functXMLin
        self.__functXMLout = _functXMLout
        self.__functXMLerr = _functXMLerr
        self.__strCurrWorkDir = os.getcwd()
        self.__strTempDir = None
        self.__listInputPaths = []
        self.__dictCurrentlyRunning = {}
        if _bVerbose is not None:
            if _bVerbose:
                self.setVerboseDebugOn()
            else:
                self.setVerboseOff()
        if _bDebug is not None:
            if _bDebug:
                self.setVerboseDebugOn()
            else:
                self.setVerboseDebugOff()
        self.__fDelay = _fDelay #default delay between two directory checks.
        self.__bQuit = False    # To check if we should quit the application
        self.__bIsFirstExecute = True
        signal.signal(signal.SIGTERM, self.handleKill)
        signal.signal(signal.SIGINT, self.handleKill)

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import detectNumberOfCPUs [as 别名]
    def chooseWorker(self):
        """
        selects the worker and sets self.worker
        """
#        self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s" % self.getId())
        if EDPluginSPDCorrectv10.__iNumberOfWorker == 0:
            EDPluginSPDCorrectv10.__iNumberOfWorker = EDUtilsParallel.detectNumberOfCPUs(self._iConfigNumberOfWorker)
        self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s nb of workers = %s lockWorker=%s" % (self.getId(), EDPluginSPDCorrectv10.__iNumberOfWorker, EDPluginSPDCorrectv10.__lockWorker._Semaphore__value))
#        self.DEBUG("chooseWorker: \t EDPluginSPDCorrectv10.__lock.acquire(), currently: %i" % EDPluginSPDCorrectv10.__lock._Semaphore__value)
        EDPluginSPDCorrectv10.__lockWorker.acquire()
        for oneWorker in EDPluginSPDCorrectv10.__listOfWorker :
            self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s Status of worker %i: %s" % (self.getId(), oneWorker.pid, oneWorker.getStatus()))
            if (oneWorker.getConfig() == self._SPDconfig)  and (oneWorker.getStatus() in ["free", "uninitialized"]):
                self.worker = oneWorker
        if self.worker is None:
            if len(EDPluginSPDCorrectv10.__listOfWorker) < EDPluginSPDCorrectv10.__iNumberOfWorker :
                self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Initializing %i th worker" % (self.getId(), len(EDPluginSPDCorrectv10.__listOfWorker)))
                self.worker = SPDworker()
                EDPluginSPDCorrectv10.__listOfWorker.append(self.worker)
                self.worker.setExecutable(self.getExecutable())
                self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(self.worker)
                self.worker.setLogFilename(os.path.join(self.getSPDCommonDirectory(), "worker-%02i.log" % self.workerID))
                self.worker.initialize(self._SPDconfig)
                self.worker.setTimeOut(self.getTimeOut())
        while self.worker is None:
            self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: No workers still to initialize" % self.getId())
            bConfigOK = False
            for idx, oneWorker in enumerate(EDPluginSPDCorrectv10.__listOfWorker) :
#                self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), oneWorker.getConfig()))
#                self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: %s " % (self.getId(), self._SPDconfig))
                #bug1 those string must specifically be the same ... change line 248 in spdCake1.5
                if (oneWorker.getConfig() == self._SPDconfig):
                    bConfigOK = True
                    if (oneWorker.getStatus() in ["free", "uninitialized"]):
                        self.worker = oneWorker
                        self.workerID = idx
            if bConfigOK == False:
                for idx, oneWorker in enumerate(EDPluginSPDCorrectv10.__listOfWorker[:]) :
#                    self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: worker %i status %s " % (self.getId(), idx, oneWorker.status))
                    if (oneWorker.getStatus() in ["free", "uninitialized"]):
                        #Bug2 why doest this work ???
#                        self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Resetting worker %i status %s " % (self.getId(), idx, oneWorker.status))
                        oneWorker.initialize(self._SPDconfig)
                        self.worker = oneWorker
                        self.workerID = idx
                        EDPluginSPDCorrectv10.__listOfWorker[idx] = (self.worker)
            self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s: Left loop worker %s status %s " % (self.getId(), self.workerID, oneWorker))
            time.sleep(0.1)

        if self.workerID is None:
            self.workerID = EDPluginSPDCorrectv10.__listOfWorker.index(self.worker)
        self.DEBUG("EDPluginSPDCorrectv10.chooseWorker %s Release lockWorker=%s" % (self.getId(), EDPluginSPDCorrectv10.__lockWorker._Semaphore__value))
        EDPluginSPDCorrectv10.__lockWorker.release()

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import detectNumberOfCPUs [as 别名]
 def __init__(self, cl, name):
     EDLogging.__init__(self)
     PyTango.Device_4Impl.__init__(self, cl, name)
     self.init_device()
     if isinstance(iNbCpu, int):
         self.screen("Initializing tangoDS with max %i jobs in parallel." % iNbCpu)
         self.__semaphoreNbThreads = threading.Semaphore(iNbCpu)
     else:
         self.__semaphoreNbThreads = threading.Semaphore(EDUtilsParallel.detectNumberOfCPUs())
     self.jobQueue = Queue()
     self.processingSem = threading.Semaphore()
     self.statLock = threading.Lock()
     self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
     self.lastFailure = "No job Failed (yet)"
     self.lastSuccess = "No job succeeded (yet)"

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import detectNumberOfCPUs [as 别名]
    def __init__(self, strPluginName, iNbCpu=None):
        EDLogging.__init__(self)
        self.pluginName = strPluginName
        self.startTime = time.time()
        try:
            self.iNbCpu = int(iNbCpu)
        except:
            self.iNbCpu = EDUtilsParallel.detectNumberOfCPUs()

        self.screen("Initializing Reprocess with max %i jobs in parallel." % self.iNbCpu)
        self.__semaphoreNbThreads = Semaphore(self.iNbCpu)
        EDUtilsParallel.initializeNbThread(self.iNbCpu)
        self.jobQueue = Queue()
        self.processingSem = Semaphore()
        self.statLock = Semaphore()
        self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
        self.lastFailure = "No job Failed (yet)"
        self.lastSuccess = "No job succeeded (yet)"

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import detectNumberOfCPUs [as 别名]
 def process(self, _edPlugin=None):
     """
     Executes the action cluster. This method will return only when
     all actions have finished.
     """
     self.DEBUG("EDActionCluster.process")
     if (not self.__iNoThreads):
         self.__iNoThreads = EDUtilsParallel.detectNumberOfCPUs()
     self.__semaphoreActionCluster = threading.Semaphore(self.__iNoThreads)
     for edAction in self.__lActions:
         edAction.connectSUCCESS(self.__semaphoreRelease)
         edAction.connectFAILURE(self.__setActionClusterFailure)
         edAction.connectFAILURE(self.__semaphoreRelease)
         self.DEBUG("EDActionCluster.process : Starting action %s" % edAction.getClassName())
         self.__semaphoreActionCluster.acquire()
         edAction.execute()
     # Wait for all threads to finish
     for edAction in self.__lActions:
         edAction.join()

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import detectNumberOfCPUs [as 别名]
 def __init__(self, cl, name):
     EDLogging.__init__(self)
     PyTango.Device_4Impl.__init__(self, cl, name)
     self.init_device()
     if isinstance(iNbCpu, int):
         self.screen("Initializing tangoDS with max %i jobs in parallel." % iNbCpu)
         self.__semaphoreNbThreads = threading.Semaphore(iNbCpu)
     else:
         self.__semaphoreNbThreads = threading.Semaphore(EDUtilsParallel.detectNumberOfCPUs())
     self.quit = False
     self.jobQueue = Queue()  # queue containing jobs to process
     self.eventQueue = Queue()  # queue containing finished jobs
     self.statLock = threading.Lock()
     self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
     self.lastFailure = "No job Failed (yet)"
     self.lastSuccess = "No job succeeded (yet)"
     self.processingThread = threading.Thread(target=self.startProcessing)
     self.processingThread.start()
     self.finishingThread = threading.Thread(target=self.process_event)
     self.finishingThread.start()

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import detectNumberOfCPUs [as 别名]
        installHDF5.downloadLibrary()
        installHDF5.unZipArchive()
        pthreadPath = installHDF5.searchCLib("libpthread.so")
        EDVerbose.DEBUG("Libpthread found in %s" % pthreadPath)
        if pthreadPath is None:
            try:
                installHDF5.configure("--prefix=%s" % (installHDF5.getDestinationDirectory()))
            except:
                EDVerbose.ERROR("Error in the configure step, no pthread")
        else:
            try:
                installHDF5.configure("--prefix=%s --enable-threadsafe --with-pthread=%s" % (installHDF5.getDestinationDirectory(), pthreadPath))
            except:
                EDVerbose.ERROR("Error in the configure step, with pthread")
        try:
            installHDF5.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs())
        except:
            EDVerbose.ERROR("Error in the 'make' step")

        try:
            installHDF5.make("install")
        except:
            EDVerbose.ERROR("Error in the 'make install' step")
        hdfPath = installHDF5.getDestinationDirectory()
        installHDF5.cleanSources()
    else:
        hdfPath = os.path.dirname(hdfPath)
    EDVerbose.DEBUG("Building H5Py with HDF5 library from %s " % (hdfPath))
    install = EDUtilsLibraryInstaller(installDir, h5pyLibrary)
    install.checkPythonVersion()
    install.dependency("numpy", "20090405-Numpy-1.3")

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import detectNumberOfCPUs [as 别名]
            elif oneline.startswith("LOADER"):
                makeFile.write("LOADER = %s  %s" % (fortranCompiler, os.linesep))
#            elif oneline.startswith("ARCHFLAGS"):
#                makeFile.write("ARCHFLAGS = -shared -o %s" % (os.linesep))
#            elif oneline.startswith("ARCH"):
#                makeFile.write("ARCH = %s %s" % (fortranCompiler, os.linesep))
#            elif  oneline.startswith("RANLIB"):
#                makeFile.write("RANLIB = ls %s" % (os.linesep))
            elif  oneline.startswith("BLASLIB"):
                makeFile.write("BLASLIB = %s %s" % (libblas, os.linesep))
            else:
                makeFile.write(oneline)

        makeFile.close()
        try:
            installBlas.make("-j %i" % EDUtilsParallel.detectNumberOfCPUs())
        except Exception:
            EDVerbose.ERROR("Error for BLAS in the 'make' step")
        if not os.path.isdir(os.path.join(installBlas.getDestinationDirectory(), "lib")):
            os.makedirs(os.path.join(installBlas.getDestinationDirectory(), "lib"))
        blasPath = os.path.join(installBlas.getDestinationDirectory(), "lib")
        shutil.copyfile(os.path.join(installBlas.getSourceDirectory(), libblas), os.path.join(blasPath, libblas))

################################################################################
# END of Blas / Start of Lapack 
################################################################################
        if lapackPath is None:
            EDVerbose.screen("Checking for Lapack %s : not found, so I have to compile it myself" % liblapack)
        else:
            EDVerbose.screen("Checking for Lapack %s : Found on %s, but I don't trust it because Blas was missing" % (liblapack, lapackPath))
        liblapack = os.path.splitext(liblapack)[0] + ".a"