Python EDUtilsParallel.initializeNbThread方法代码示例

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import initializeNbThread [as 别名]
    def __init__(self, _strPluginName, _functXMLin, \
                  _functXMLout=None, _functXMLerr=None, \
                  _iNbThreads=None, _fDelay=1.0, _bVerbose=None, _bDebug=None):
        """
        This is the constructor of the edna plugin launcher.
        
        @param _strPluginName: the name of the ENDA plugin
        @type  _strPluginName: python string
        
        @param _functXMLin: a function taking a path in input and returning the XML string for input in the EDNA plugin. 
        @type  _functXMLin: python function
        
        @param _functXMLOut: a function to be called each time a plugin gas finished his job sucessfully, it should take two option: strXMLin an strXMLout
        @type  _functXMLOut: python function
         
        @param _functXMLErr: a function to be called each time a plugin gas finished his job and crashed, it should take ONE option: strXMLin
        @type  _functXMLErr: python function 
        
        @param _iNbThreads: The number of parallel threads to be used by EDNA, usually the number of Cores of the computer. If 0 or None, the number of cores  will be auto-detected. 
        @type  _iNbThreads: python integer
        
        @param _fDelay: The delay in seconds between two directories analysis 
        @type  _fDelay: python float
        
        @param _bVerbose:  Do you want the EDNA plugin execution to be verbose ?
        @type  _bVerbose: boolean

        @param _bDebug:  Do you want EDNA plugin execution debug output (OBS! very verbose) ?
        @type  _bDebug: boolean
        """
        EDLogging.__init__(self)
        self.__iNbThreads = EDUtilsParallel.detectNumberOfCPUs(_iNbThreads)
        EDUtilsParallel.initializeNbThread(self.__iNbThreads)
################################################################################
# #We are not using the one from EDUtilsParallel to leave it to control the number of  execPlugins.
################################################################################
        self.__semaphoreNbThreads = Semaphore(self.__iNbThreads)
        self.__strPluginName = _strPluginName
        self.__functXMLin = _functXMLin
        self.__functXMLout = _functXMLout
        self.__functXMLerr = _functXMLerr
        self.__strCurrWorkDir = os.getcwd()
        self.__strTempDir = None
        self.__listInputPaths = []
        self.__dictCurrentlyRunning = {}
        if _bVerbose is not None:
            if _bVerbose:
                self.setVerboseDebugOn()
            else:
                self.setVerboseOff()
        if _bDebug is not None:
            if _bDebug:
                self.setVerboseDebugOn()
            else:
                self.setVerboseDebugOff()
        self.__fDelay = _fDelay #default delay between two directory checks.
        self.__bQuit = False    # To check if we should quit the application
        self.__bIsFirstExecute = True
        signal.signal(signal.SIGTERM, self.handleKill)
        signal.signal(signal.SIGINT, self.handleKill)

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import initializeNbThread [as 别名]
    def preProcess(self):
        """
        PreProcess of the execution test: download a set of images  from http://www.edna-site.org
        and remove any existing output file 
        """
        EDTestCasePluginExecute.preProcess(self)
        self.loadTestImage([ "bioSaxsMask.edf", "bioSaxsNormalized.edf", "bioSaxsIntegratedv1_3.dat", "bioSaxsCorrected.edf"])
        strExpectedOutput = self.readAndParseFile (self.getReferenceDataOutputFile())
        EDVerbose.DEBUG("strExpectedOutput:" + strExpectedOutput)
        xsDataResultReference = XSDataResultBioSaxsAzimutIntv1_0.parseString(strExpectedOutput)
        self.integratedCurve = xsDataResultReference.getIntegratedCurve().getPath().value
        EDVerbose.DEBUG("Output file is %s" % self.integratedCurve)
        if not os.path.isdir(os.path.dirname(self.integratedCurve)):
            os.makedirs(os.path.dirname(self.integratedCurve))
        if os.path.isfile(self.integratedCurve):
            EDVerbose.DEBUG(" Output Integrated Curve file exists %s, I will remove it" % self.integratedCurve)
            os.remove(self.integratedCurve)

        EDUtilsParallel.initializeNbThread()
#        self.correctedImage = xsDataResultReference.getCorrectedImage().getPath().value
#        EDVerbose.DEBUG("Output Corrected Image file is %s" % self.correctedImage)
#        if not os.path.isdir(os.path.dirname(self.correctedImage)):
#            os.makedirs(os.path.dirname(self.correctedImage))
#        if os.path.isfile(self.correctedImage):
#            EDVerbose.DEBUG(" Output Corrected Image file exists %s, I will remove it" % self.correctedImage)
#            os.remove(self.correctedImage)
        if not pyFAI.version.startswith("0.7"):
            EDVerbose.ERROR('pyFAI is not the right version, tested only with 0.7.2, here running version %s' % pyFAI.version)

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import initializeNbThread [as 别名]
    def __init__(self, _strPluginName, _strPluginDir=None, _strTestName=None):
        """
        Initialize the Plugin Unit test
        """
        EDTestCasePlugin.__init__(self, _strPluginName, _strPluginDir, _strTestName)
        self.__edPlugin = None
#        To track dead locks
        EDUtilsParallel.initializeNbThread(1)

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import initializeNbThread [as 别名]
    def __init__(self, strPluginName, iNbCpu=None):
        EDLogging.__init__(self)
        self.pluginName = strPluginName
        self.startTime = time.time()
        try:
            self.iNbCpu = int(iNbCpu)
        except:
            self.iNbCpu = EDUtilsParallel.detectNumberOfCPUs()

        self.screen("Initializing Reprocess with max %i jobs in parallel." % self.iNbCpu)
        self.__semaphoreNbThreads = Semaphore(self.iNbCpu)
        EDUtilsParallel.initializeNbThread(self.iNbCpu)
        self.jobQueue = Queue()
        self.processingSem = Semaphore()
        self.statLock = Semaphore()
        self.lastStatistics = "No statistics collected yet, please use the 'collectStatistics' method first"
        self.lastFailure = "No job Failed (yet)"
        self.lastSuccess = "No job succeeded (yet)"

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import initializeNbThread [as 别名]
 def preProcess(self):
     """
     PreProcess of the execution test: download a set of images  from http://www.edna-site.org
     and remove any existing output file 
     """
     EDTestCasePluginExecute.preProcess(self)
     self.loadTestImage(["bioSaxsRaw.edf", "bioSaxsMask.edf",
                         "bioSaxsCorrected.edf", "bioSaxsProcessNormalized.edf",
                         "bioSaxsProcessIntegrated.edf", "bioSaxsProcessIntegrated1_2.dat"])
     strExpectedOutput = self.readAndParseFile (self.getReferenceDataOutputFile())
     EDVerbose.DEBUG("strExpectedOutput:" + strExpectedOutput)
     xsDataResultReference = XSDataResultBioSaxsProcessOneFilev1_0.parseString(strExpectedOutput)
     self.refNormImg = xsDataResultReference.normalizedImage.path.value
     self.refIntCrv = xsDataResultReference.integratedCurve.path.value
     if not os.path.isdir(os.path.dirname(self.refNormImg)):
         os.makedirs(os.path.dirname(self.refNormImg))
     if os.path.isfile(self.refNormImg):
         os.remove(self.refNormImg)
     if os.path.isfile(self.refIntCrv):
         os.remove(self.refIntCrv)
     EDUtilsParallel.initializeNbThread()

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import initializeNbThread [as 别名]
    def process(self, _edPlugin=None):
        """
        Executes the execution plugins
        """
        EDPluginControl.process(self, _edPlugin)
        self.DEBUG("EDPluginControlImageQualityIndicatorsv1_2.process")
        EDUtilsParallel.initializeNbThread()
        # Check if we should do indexing:
        bDoIndexing = False
        if self.dataInput.doIndexing is not None:
            if self.dataInput.doIndexing.value:
                bDoIndexing = True
        # Loop through all the incoming reference images
        listXSDataImage = self.dataInput.image
        xsDataInputMXWaitFile = XSDataInputMXWaitFile()
        self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators(
        )
        listPlugin = []
        for xsDataImage in listXSDataImage:
            self.edPluginMXWaitFile = self.loadPlugin(
                self.strPluginMXWaitFileName)
            xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
            xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
            xsDataInputMXWaitFile.setTimeOut(
                XSDataTime(self.fMXWaitFileTimeOut))
            self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
            self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
            self.edPluginMXWaitFile.executeSynchronous()
            if not os.path.exists(xsDataImage.path.value):
                strError = "Time-out while waiting for image %s" % xsDataImage.path.value
                self.error(strError)
                self.addErrorMessage(strError)
                self.setFailure()
            else:
                if self.bUseThinClient:
                    strPluginName = self.strPluginNameThinClient
                else:
                    strPluginName = self.strPluginName
                edPluginPluginExecImageQualityIndicator = self.loadPlugin(
                    strPluginName)
                listPlugin.append(edPluginPluginExecImageQualityIndicator)
                self.listPluginExecImageQualityIndicator.append(
                    edPluginPluginExecImageQualityIndicator)
                xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength(
                )
                xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage)
                edPluginPluginExecImageQualityIndicator.setDataInput(
                    xsDataInputDistlSignalStrength)
                edPluginPluginExecImageQualityIndicator.execute()
        listIndexing = []
        # Synchronize all image quality indicator plugins and upload to ISPyB
        xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators(
        )
        for edPluginPluginExecImageQualityIndicator in listPlugin:
            edPluginPluginExecImageQualityIndicator.synchronize()
            xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString( \
                                             edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
            self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(
                xsDataImageQualityIndicators)
            xsDataISPyBImageQualityIndicators = \
                XSDataISPyBImageQualityIndicators.parseString(xsDataImageQualityIndicators.marshal())
            xsDataInputStoreListOfImageQualityIndicators.addImageQualityIndicators(
                xsDataISPyBImageQualityIndicators)
#        print xsDataInputStoreListOfImageQualityIndicators.marshal()
        if self.bDoISPyBUpload:
            self.edPluginISPyB = self.loadPlugin(self.strISPyBPluginName)
            self.edPluginISPyB.dataInput = xsDataInputStoreListOfImageQualityIndicators
            self.edPluginISPyB.execute()
        #
        if bDoIndexing:
            # Find the 5 most intensive images (TIS):
            listImage = []
            listSorted = sorted(
                self.xsDataResultControlImageQualityIndicators.
                imageQualityIndicators,
                key=
                lambda imageQualityIndicators: imageQualityIndicators.totalIntegratedSignal.value
            )
            for xsDataResultControlImageQualityIndicator in listSorted[-5:]:
                if xsDataResultControlImageQualityIndicator.goodBraggCandidates.value > 30:
                    xsDataInputReadImageHeader = XSDataInputReadImageHeader()
                    xsDataInputReadImageHeader.image = XSDataFile(
                        xsDataResultControlImageQualityIndicator.image.path)
                    self.edPluginReadImageHeader = self.loadPlugin(
                        self.strPluginReadImageHeaderName)
                    self.edPluginReadImageHeader.dataInput = xsDataInputReadImageHeader
                    self.edPluginReadImageHeader.executeSynchronous()
                    xsDataResultReadImageHeader = self.edPluginReadImageHeader.dataOutput
                    if xsDataResultReadImageHeader is not None:
                        xsDataSubWedge = xsDataResultReadImageHeader.subWedge
                        self.xsDataCollection = XSDataCollection()
                        self.xsDataCollection.addSubWedge(xsDataSubWedge)
                        xsDataIndexingInput = XSDataIndexingInput()
                        xsDataIndexingInput.setDataCollection(
                            self.xsDataCollection)
                        xsDataMOSFLMIndexingInput = EDHandlerXSDataMOSFLMv10.generateXSDataMOSFLMInputIndexing(
                            xsDataIndexingInput)
                        edPluginMOSFLMIndexing = self.loadPlugin(
                            self.strIndexingMOSFLMPluginName)
                        self.listPluginMOSFLM.append([
#.........这里部分代码省略.........

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import initializeNbThread [as 别名]
    def process(self, _edPlugin=None):
        """
        Executes the execution plugins
        """
        EDPluginControl.process(self, _edPlugin)
        self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process")
        EDUtilsParallel.initializeNbThread()
        # Check batch size
        if self.dataInput.batchSize is None:
            batchSize = 1
        else:
            batchSize = self.dataInput.batchSize.value
        # Check if we should do distlSignalStrength:
        bDoDistlSignalStrength = True
        if self.dataInput.doDistlSignalStrength is not None:
            if not self.dataInput.doDistlSignalStrength.value:
                bDoDistlSignalStrength = False
        # Check if we should do indexing:
        bDoIndexing = False
        if self.dataInput.doIndexing is not None:
            if self.dataInput.doIndexing.value:
                 bDoIndexing = True
        # Loop through all the incoming reference images
        listXSDataImage = self.dataInput.image
        xsDataInputMXWaitFile = XSDataInputMXWaitFile()
        self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators()
        listPluginDistl = []
        listPluginDozor = []
        listBatch = []
        indexBatch = 0
        for xsDataImage in listXSDataImage:
            self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
            xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
            xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
            xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
            self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
            self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
            self.edPluginMXWaitFile.executeSynchronous()
            if not os.path.exists(xsDataImage.path.value):
                self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
                xsDataInputMXWaitFile.file = XSDataFile(xsDataImage.path)
                xsDataInputMXWaitFile.setSize(XSDataInteger(5000000))
                xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
                self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
                self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
                self.edPluginMXWaitFile.executeSynchronous()
            if not os.path.exists(xsDataImage.path.value):
                strError = "Time-out while waiting for image %s" % xsDataImage.path.value
                self.error(strError)
                self.addErrorMessage(strError)
                self.setFailure()
            else:
                # Check if we should run distl.signalStrength
                edPluginPluginExecImageQualityIndicator = None
                if bDoDistlSignalStrength:
                    if self.bUseThinClient:
                        strPluginName = self.strPluginNameThinClient
                    else:
                        strPluginName = self.strPluginName
                    edPluginPluginExecImageQualityIndicator = self.loadPlugin(strPluginName)
                    self.listPluginExecImageQualityIndicator.append(edPluginPluginExecImageQualityIndicator)
                    xsDataInputDistlSignalStrength = XSDataInputDistlSignalStrength()
                    xsDataInputDistlSignalStrength.setReferenceImage(xsDataImage)
                    edPluginPluginExecImageQualityIndicator.setDataInput(xsDataInputDistlSignalStrength)
                    edPluginPluginExecImageQualityIndicator.execute()
                listPluginDistl.append((xsDataImage.copy(), edPluginPluginExecImageQualityIndicator))
                listBatch.append(xsDataImage.copy())
                if len(listBatch) == batchSize:
                    edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor)
                    xsDataInputControlDozor = XSDataInputControlDozor()
                    for image in listBatch:
                        xsDataInputControlDozor.addImage(XSDataFile(image.path))
                    xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
                    edPluginControlDozor.dataInput = xsDataInputControlDozor
                    edPluginControlDozor.execute()
                    listPluginDozor.append((edPluginControlDozor, listBatch))
                    listBatch = []
        if len(listBatch) > 0:
            # Process the remaining images...
            edPluginControlDozor = self.loadPlugin(self.strPluginNameControlDozor)
            xsDataInputControlDozor = XSDataInputControlDozor()
            for image in listBatch:
                xsDataInputControlDozor.addImage(XSDataFile(image.path))
            xsDataInputControlDozor.batchSize = XSDataInteger(batchSize)
            edPluginControlDozor.dataInput = xsDataInputControlDozor
            edPluginControlDozor.execute()
            listPluginDozor.append([edPluginControlDozor, listBatch])
        listIndexing = []
        # Synchronize all image quality indicator plugins and upload to ISPyB
        xsDataInputStoreListOfImageQualityIndicators = XSDataInputStoreListOfImageQualityIndicators()

        for (xsDataImage, edPluginPluginExecImageQualityIndicator) in listPluginDistl:
            xsDataImageQualityIndicators = XSDataImageQualityIndicators()
            xsDataImageQualityIndicators.image = xsDataImage.copy()
            if edPluginPluginExecImageQualityIndicator is not None:
                edPluginPluginExecImageQualityIndicator.synchronize()
                if edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators is not None:
                    xsDataImageQualityIndicators = XSDataImageQualityIndicators.parseString(\
                            edPluginPluginExecImageQualityIndicator.dataOutput.imageQualityIndicators.marshal())
            self.xsDataResultControlImageQualityIndicators.addImageQualityIndicators(xsDataImageQualityIndicators)
#.........这里部分代码省略.........

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import initializeNbThread [as 别名]
    def process(self, _edObject=None):
        EDPluginControl.process(self)
        self.DEBUG("EDPluginControlDozorv1_0.process")
        self.sendMessageToMXCuBE("Processing started...", "info")
        EDUtilsParallel.initializeNbThread()
        xsDataResultControlDozor = XSDataResultControlDozor()
        # Check if connection to ISPyB needed
        if self.dataInput.dataCollectionId is not None:
            edPluginRetrieveDataCollection = self.loadPlugin("EDPluginISPyBRetrieveDataCollectionv1_4")
            xsDataInputRetrieveDataCollection = XSDataInputRetrieveDataCollection()
            xsDataInputRetrieveDataCollection.dataCollectionId = self.dataInput.dataCollectionId
            edPluginRetrieveDataCollection.dataInput = xsDataInputRetrieveDataCollection
            edPluginRetrieveDataCollection.executeSynchronous()
            ispybDataCollection = edPluginRetrieveDataCollection.dataOutput.dataCollection
            if self.batchSize is None:
                batchSize = ispybDataCollection.numberOfImages
            else:
                batchSize = self.batchSize
            if batchSize > self.maxBatchSize:
                batchSize = self.maxBatchSize
            if abs(ispybDataCollection.overlap) > 1:
                self.hasOverlap = True
                self.overlap = ispybDataCollection.overlap
            dictImage = self.createImageDictFromISPyB(ispybDataCollection)
        else:
            # No connection to ISPyB, take parameters from input
            if self.dataInput.batchSize is None:
                batchSize = self.maxBatchSize
            else:
                batchSize = self.dataInput.batchSize.value
            dictImage = self.createImageDict(self.dataInput)
        self.screen("Dozor batch size: {0}".format(batchSize))
        if self.dataInput.hdf5BatchSize is not None:
            self.hdf5BatchSize = self.dataInput.hdf5BatchSize.value
        listAllBatches = self.createListOfBatches(dictImage.keys(), batchSize)
        if dictImage[listAllBatches[0][0]].path.value.endswith("h5"):
            # Convert HDF5 images to CBF
            self.screen("HDF5 converter batch size: {0}".format(self.batchSize))
            if self.doRadiationDamage:
                self.cbfTempDir = None
            else:
                self.cbfTempDir = tempfile.mkdtemp(prefix="CbfTemp_")
            listHdf5Batches = self.createListOfBatches(dictImage.keys(), self.batchSize)
            dictImage, self.hasHdf5Prefix = self.convertToCBF(dictImage, listHdf5Batches, self.doRadiationDamage)
        for listBatch in listAllBatches:
            # Read the header from the first image in the batch
            xsDataFile = dictImage[listBatch[0]]
            edPluginControlReadImageHeader = self.loadPlugin(self.strEDPluginControlReadImageHeaderName)
            xsDataInputReadImageHeader = XSDataInputReadImageHeader()
            xsDataInputReadImageHeader.image = xsDataFile
            edPluginControlReadImageHeader.dataInput = xsDataInputReadImageHeader
            edPluginControlReadImageHeader.executeSynchronous()
            subWedge = edPluginControlReadImageHeader.dataOutput.subWedge
            xsDataInputDozor = XSDataInputDozor()
            beam = subWedge.experimentalCondition.beam
            detector = subWedge.experimentalCondition.detector
            goniostat = subWedge.experimentalCondition.goniostat
            xsDataInputDozor.detectorType = detector.type
            xsDataInputDozor.exposureTime = XSDataDouble(beam.exposureTime.value)
            xsDataInputDozor.spotSize = XSDataInteger(3)
            xsDataInputDozor.detectorDistance = XSDataDouble(detector.distance.value)
            xsDataInputDozor.wavelength = XSDataDouble(beam.wavelength.value)
#            xsDataInputDozor.fractionPolatization : XSDataDouble optional
            orgx = detector.beamPositionY.value / detector.pixelSizeY.value
            orgy = detector.beamPositionX.value / detector.pixelSizeX.value
            xsDataInputDozor.orgx = XSDataDouble(orgx)
            xsDataInputDozor.orgy = XSDataDouble(orgy)
            xsDataInputDozor.oscillationRange = XSDataDouble(goniostat.oscillationWidth.value)
#            xsDataInputDozor.imageStep : XSDataDouble optional
            xsDataInputDozor.startingAngle = XSDataDouble(goniostat.rotationAxisStart.value)
            xsDataInputDozor.firstImageNumber = subWedge.image[0].number
            xsDataInputDozor.numberImages = XSDataInteger(len(listBatch))
            if self.hasOverlap:
                xsDataInputDozor.overlap = XSDataAngle(self.overlap)
            strFileName = subWedge.image[0].path.value
            strPrefix = EDUtilsImage.getPrefix(strFileName)
            strSuffix = EDUtilsImage.getSuffix(strFileName)
            if EDUtilsPath.isEMBL():
                strXDSTemplate = "%s_?????.%s" % (strPrefix, strSuffix)
            elif self.hasHdf5Prefix and not self.hasOverlap:
                strXDSTemplate = "%s_??????.%s" % (strPrefix, strSuffix)
            else:
                strXDSTemplate = "%s_????.%s" % (strPrefix, strSuffix)
            xsDataInputDozor.nameTemplateImage = XSDataString(os.path.join(os.path.dirname(strFileName), strXDSTemplate))
            xsDataInputDozor.wedgeNumber = self.dataInput.wedgeNumber
            xsDataInputDozor.radiationDamage = self.dataInput.radiationDamage
            edPluginDozor = self.loadPlugin(self.strEDPluginDozorName, "Dozor_%05d" % subWedge.image[0].number.value)
            edPluginDozor.dataInput = xsDataInputDozor
            edPluginDozor.execute()
            edPluginDozor.synchronize()
            indexImage = 0
            imageDozorBatchList = []

            for xsDataResultDozor in edPluginDozor.dataOutput.imageDozor:
                xsDataControlImageDozor = XSDataControlImageDozor()
                xsDataControlImageDozor.number = xsDataResultDozor.number
                xsDataControlImageDozor.image = dictImage[listBatch[indexImage]]
                xsDataControlImageDozor.spotsNumOf = xsDataResultDozor.spotsNumOf
                xsDataControlImageDozor.spotsIntAver = xsDataResultDozor.spotsIntAver
                xsDataControlImageDozor.spotsResolution = xsDataResultDozor.spotsResolution
#.........这里部分代码省略.........

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import initializeNbThread [as 别名]
strEdnaKernel = os.path.join(pyStrEdnaHomePath, "kernel", "src")
sys.path.append(strEdnaKernel)
if not "PYTHONPATH" in os.environ:
    os.environ["PYTHONPATH"] = strEdnaKernel
else:
    os.environ["PYTHONPATH"] += strEdnaKernel
os.environ["PYTHONPATH"] += os.pathsep + os.path.join(pyStrEdnaHomePath, "kernel", "tests", "src")
os.environ["PYTHONPATH"] += os.pathsep + os.path.join(pyStrEdnaHomePath, "kernel", "tests", "testsuite")


from EDJob import EDJob
from EDUtilsParallel import EDUtilsParallel
from EDVerbose import EDVerbose
from EDFactoryPluginStatic import EDFactoryPluginStatic

EDUtilsParallel.initializeNbThread()
EDFactoryPluginStatic.loadModule("XSDataDocumentation")
from XSDataDocumentation import XSDataString, XSDataFile, XSDataInputEpydoc, XSDataInteger

dictJobs = {}


def findPlugins(EDNAHome, EDPluginPrefix="EDPlugin", XSDataPrefix="XSData", pythonExtension=".py"):
    """
    This function is the walker that goes through all directories in EDNA_HOME directory and searches for EDNA plugins ...
    
    @param EDNAHome: the path of EDNA_HOME
    @type EDNAHome: string
    @param EDPluginPrefix: the start of the name of an EDNA Plugin
    @type EDPluginPrefix: string
    @param pythonExtension: the extension of an EDNA plugin, usually .py

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import initializeNbThread [as 别名]
    def process(self, _edPlugin=None):
        """
        Executes the execution plugins
        """
        EDPluginControl.process(self, _edPlugin)
        self.DEBUG("EDPluginControlImageQualityIndicatorsv1_4.process")
        EDUtilsParallel.initializeNbThread()
        # Check batch size
        if self.dataInput.batchSize is None:
            batchSize = 1
        else:
            batchSize = self.dataInput.batchSize.value
        self.screen("Batch size: {0}".format(batchSize))
        # Check if we should do distlSignalStrength:
        bDoDistlSignalStrength = True
        if self.dataInput.doDistlSignalStrength is not None:
            if not self.dataInput.doDistlSignalStrength.value:
                bDoDistlSignalStrength = False
        # Check if we should do indexing:
        bDoIndexing = False
        if self.dataInput.doIndexing is not None:
            if self.dataInput.doIndexing.value:
                 bDoIndexing = True
        # Loop through all the incoming reference images
        listXSDataImage = self.dataInput.image
        xsDataInputMXWaitFile = XSDataInputMXWaitFile()
        self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators()
        listPluginDistl = []
        listPluginDozor = []
        listBatch = []
        indexBatch = 0
        listH5FilePath = []
        ispybDataCollection = None
        for xsDataImage in listXSDataImage:
            strPathToImage = xsDataImage.path.value
            # If Eiger, just wait for the h5 file
            if "id30a3" in strPathToImage:
                h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(strPathToImage, batchSize)
#                print(h5FilePath)
#                print(hdf5ImageNumber)
                if not h5DataFilePath in listH5FilePath:
                    self.screen("ID30a3 Eiger data, waiting for master and data files...")
                    listH5FilePath.append(h5DataFilePath)
                    self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
                    xsDataInputMXWaitFile.file = XSDataFile(XSDataString(h5DataFilePath))
                    xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
                    xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
                    self.screen("Waiting for file {0}".format(h5DataFilePath))
                    self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
                    self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
                    self.edPluginMXWaitFile.executeSynchronous()
#                    hdf5FilePath = strPathToImage.replace(".cbf", ".h5")
                    ispybDataCollection = None
                    time.sleep(1)
                indexLoop = 1
                continueLoop = True
                while continueLoop:
                    xsDataInputControlH5ToCBF = XSDataInputControlH5ToCBF()
                    xsDataInputControlH5ToCBF.hdf5File = XSDataFile(XSDataString(strPathToImage))
                    imageNumber = EDUtilsImage.getImageNumber(strPathToImage)
                    xsDataInputControlH5ToCBF.imageNumber = XSDataInteger(imageNumber)
                    xsDataInputControlH5ToCBF.hdf5ImageNumber = XSDataInteger(hdf5ImageNumber)
                    xsDataInputControlH5ToCBF.ispybDataCollection = ispybDataCollection
                    edPluginControlH5ToCBF = self.loadPlugin(self.strPluginControlH5ToCBF, "ControlH5ToCBF_%04d_%02d" % (imageNumber, indexLoop))
                    edPluginControlH5ToCBF.dataInput = xsDataInputControlH5ToCBF
                    edPluginControlH5ToCBF.executeSynchronous()
                    cbfFile = edPluginControlH5ToCBF.dataOutput.outputCBFFile
#                    print(cbfFile)
#                    print(indexLoop)
                    if cbfFile is not None:
                        strPathToImage = cbfFile.path.value
#                        print(cbfFile.path.value)
                        if os.path.exists(strPathToImage):
                            self.screen("Image has been converted to CBF file: {0}".format(strPathToImage))
                            continueLoop = False
#                    print(continueLoop)
                    if continueLoop:
                        self.screen("Still waiting for converting to CBF file: {0}".format(strPathToImage))
                        indexLoop += 1
                        time.sleep(5)
                        if indexLoop > 10:
                            continueLoop = False

                ispybDataCollection = edPluginControlH5ToCBF.dataOutput.ispybDataCollection


            elif not os.path.exists(strPathToImage):
                self.screen("Waiting for file {0}".format(strPathToImage))
                self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
                xsDataInputMXWaitFile.file = XSDataFile(XSDataString(strPathToImage))
                xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
                xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
                self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
                self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
                self.edPluginMXWaitFile.executeSynchronous()
            if not os.path.exists(strPathToImage):
                strError = "Time-out while waiting for image %s" % strPathToImage
                self.error(strError)
                self.addErrorMessage(strError)
                self.setFailure()
#.........这里部分代码省略.........

# 需要导入模块: from EDUtilsParallel import EDUtilsParallel [as 别名]
# 或者: from EDUtilsParallel.EDUtilsParallel import initializeNbThread [as 别名]
    def process(self, _edPlugin=None):
        """
        Executes the execution plugins
        """
        EDPluginControl.process(self, _edPlugin)
        self.DEBUG("EDPluginControlImageQualityIndicatorsv1_5.process")
        EDUtilsParallel.initializeNbThread()
        # Check batch size
        if self.dataInput.batchSize is None:
            batchSize = 1
        else:
            batchSize = self.dataInput.batchSize.value
        self.screen("Image quality indicators batch size: {0}".format(batchSize))
        # Check if we should do distlSignalStrength:
        bDoDistlSignalStrength = True
        if self.dataInput.doDistlSignalStrength is not None:
            if not self.dataInput.doDistlSignalStrength.value:
                bDoDistlSignalStrength = False
        # Check if we should do indexing:
        bDoIndexing = False
        if self.dataInput.doIndexing is not None:
            if self.dataInput.doIndexing.value:
                 bDoIndexing = True
        # Check if fast mesh (for HDF5)
        isFastMesh = False
        if self.dataInput.fastMesh:
            isFastMesh = self.dataInput.fastMesh.value
        # Loop through all the incoming reference images
        if len(self.dataInput.image) == 0:
            directory = self.dataInput.directory.path.value
            template = self.dataInput.template.value
            startNo = self.dataInput.startNo.value
            endNo = self.dataInput.endNo.value
            listXSDataImage = []
            for index in range(startNo, endNo + 1):
                imageName = template.replace("####", "{0:04d}".format(index))
                imagePath = os.path.join(directory, imageName)
                xsDataImage = XSDataImage(path=XSDataString(imagePath), number=XSDataInteger(index))
                listXSDataImage.append(xsDataImage)
        else:
            listXSDataImage = self.dataInput.image
        xsDataInputMXWaitFile = XSDataInputMXWaitFile()
        self.xsDataResultControlImageQualityIndicators = XSDataResultControlImageQualityIndicators()
        listPluginDistl = []
        listPluginDozor = []
        listOfImagesInBatch = []
        listOfAllBatches = []
        indexBatch = 0
        listH5FilePath = []
        # Process data in batches
        for xsDataImage in listXSDataImage:
            listOfImagesInBatch.append(xsDataImage.copy())
            if len(listOfImagesInBatch) == batchSize:
                listOfAllBatches.append(listOfImagesInBatch)
                listOfImagesInBatch = []
        if len(listOfImagesInBatch) > 0:
            listOfAllBatches.append(listOfImagesInBatch)
            listOfImagesInBatch = []
        # Loop over batches
        for listOfImagesInBatch in listOfAllBatches:
            # First wait for images
            for image in listOfImagesInBatch:
                strPathToImage = image.path.value
                # If Eiger, just wait for the h5 file
                if strPathToImage.endswith(".h5"):
                    h5MasterFilePath, h5DataFilePath, hdf5ImageNumber = self.getH5FilePath(strPathToImage,
                                                                                           batchSize=batchSize,
                                                                                           isFastMesh=isFastMesh)
    #                print(h5FilePath)
    #                print(hdf5ImageNumber)
                    if not h5DataFilePath in listH5FilePath:
                        self.screen("ID30a3 Eiger data, waiting for master and data files...")
                        listH5FilePath.append(h5DataFilePath)
                        self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
                        xsDataInputMXWaitFile.file = XSDataFile(XSDataString(h5DataFilePath))
                        xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
                        xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
                        self.screen("Waiting for file {0}".format(h5DataFilePath))
                        self.DEBUG("Wait file timeOut set to %f" % self.fMXWaitFileTimeOut)
                        self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
                        self.edPluginMXWaitFile.executeSynchronous()
    #                    hdf5FilePath = strPathToImage.replace(".cbf", ".h5")
                        time.sleep(1)
                    if not os.path.exists(h5DataFilePath):
                        strError = "Time-out while waiting for image %s" % h5DataFilePath
                        self.error(strError)
                        self.addErrorMessage(strError)
                        self.setFailure()
                else:
                    if not os.path.exists(strPathToImage):
                        # self.screen("Waiting for file {0}".format(strPathToImage))
                        self.edPluginMXWaitFile = self.loadPlugin(self.strPluginMXWaitFileName)
                        xsDataInputMXWaitFile.file = XSDataFile(XSDataString(strPathToImage))
                        xsDataInputMXWaitFile.setSize(XSDataInteger(self.minImageSize))
                        xsDataInputMXWaitFile.setTimeOut(XSDataTime(self.fMXWaitFileTimeOut))
                        self.screen("Wait file timeOut set to %.0f s" % self.fMXWaitFileTimeOut)
                        self.edPluginMXWaitFile.setDataInput(xsDataInputMXWaitFile)
                        self.edPluginMXWaitFile.executeSynchronous()
                    if not os.path.exists(strPathToImage):
                        strError = "Time-out while waiting for image %s" % strPathToImage
#.........这里部分代码省略.........