Python DBSBufferUtil.createBlocks方法代码示例

# 需要导入模块: from WMComponent.DBS3Buffer.DBSBufferUtil import DBSBufferUtil [as 别名]
# 或者: from WMComponent.DBS3Buffer.DBSBufferUtil.DBSBufferUtil import createBlocks [as 别名]
    def testDualUpload(self):
        """
        _testDualUpload_

        Verify that the dual upload mode works correctly.
        """
        self.dbsApi = DbsApi(url = self.dbsUrl)
        config = self.getConfig(dbs3UploadOnly = True)
        dbsUploader = DBSUploadPoller(config = config)
        dbsUtil = DBSBufferUtil()

        # First test verifies that uploader will poll and then not do anything
        # as the database is empty.
        dbsUploader.algorithm()

        acqEra = "Summer%s" % (int(time.time()))
        parentFiles = self.createParentFiles(acqEra)
        (moreParentFiles, childFiles) = \
                          self.createFilesWithChildren(parentFiles, acqEra)

        allFiles = parentFiles + moreParentFiles
        allBlocks = []
        for i in range(4):
            blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID()
            dbsBlock = DBSBlock(blockName, "malpaquet", 1)
            dbsBlock.status = "Open"                
            dbsUtil.createBlocks([dbsBlock])
            for file in allFiles[i * 5 : (i * 5) + 5]:
                dbsBlock.addFile(file)
                dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
                if i < 2:
                    dbsBlock.status = "InDBS"
                dbsUtil.updateBlocks([dbsBlock])
            dbsUtil.updateFileStatus([dbsBlock], "InDBS")
            allBlocks.append(dbsBlock)            

        blockName = childFiles[0]["datasetPath"] + "#" + makeUUID()
        dbsBlock = DBSBlock(blockName, "malpaquet", 1)
        dbsBlock.status = "InDBS"
        dbsUtil.createBlocks([dbsBlock])
        for file in childFiles:
            dbsBlock.addFile(file)
            dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})

        dbsUtil.updateFileStatus([dbsBlock], "InDBS")

        dbsUploader.algorithm()
        time.sleep(5)
        dbsUploader.algorithm()
        time.sleep(5)

        self.verifyData(parentFiles[0]["datasetPath"], parentFiles)

        # Change the status of the rest of the parent blocks so we can upload
        # them and the children.
        for dbsBlock in allBlocks:
            dbsBlock.status = "InDBS"            
            dbsUtil.updateBlocks([dbsBlock])

        dbsUploader.algorithm()
        time.sleep(5)

        self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles)

        # Run the uploader one more time to upload the children.
        dbsUploader.algorithm()
        time.sleep(5)        
    
        self.verifyData(childFiles[0]["datasetPath"], childFiles)
        return

# 需要导入模块: from WMComponent.DBS3Buffer.DBSBufferUtil import DBSBufferUtil [as 别名]
# 或者: from WMComponent.DBS3Buffer.DBSBufferUtil.DBSBufferUtil import createBlocks [as 别名]

#.........这里部分代码省略.........
           written to DBSBuffer.  We don't have to do anything with it.
        """
        myThread = threading.currentThread()

        createInDBS = []
        createInDBSBuffer = []
        updateInDBSBuffer = []
        for block in self.blockCache.values():
            if block.getName() in self.queuedBlocks:
                # Block is already being dealt with by another process.  We'll
                # ignore it here.
                continue
            if block.status == 'Pending':
                # All pending blocks need to be injected into DBS.
                createInDBS.append(block)

                # If this is a new block it needs to be added to DBSBuffer
                # otherwise it just needs to be updated in DBSBuffer.
                if not block.inBuff:
                    createInDBSBuffer.append(block)
                else:
                    updateInDBSBuffer.append(block)
            if block.status == 'Open' and not block.inBuff:
                # New block that needs to be added to DBSBuffer.
                createInDBSBuffer.append(block)

        # Build the pool if it was closed
        if len(self.pool) == 0:
            self.setupPool()

        # First handle new and updated blocks
        try:
            myThread.transaction.begin()
            self.dbsUtil.createBlocks(blocks = createInDBSBuffer)
            self.dbsUtil.updateBlocks(blocks = updateInDBSBuffer)
            myThread.transaction.commit()
        except WMException:
            myThread.transaction.rollback()
            raise
        except Exception as ex:
            msg =  "Unhandled exception while writing new blocks into DBSBuffer\n"
            msg += str(ex)
            logging.error(msg)
            logging.debug("Blocks for DBSBuffer: %s\n" % createInDBSBuffer)
            logging.debug("Blocks for Update: %s\n" % updateInDBSBuffer)
            myThread.transaction.rollback()
            raise DBSUploadException(msg)

        # Update block status in the block cache.  Mark the blocks that we have
        # added to DBSBuffer as being in DBSBuffer.
        for block in createInDBSBuffer:
            self.blockCache.get(block.getName()).inBuff = True

        # Record new file/block associations in DBSBuffer.
        try:
            myThread.transaction.begin()
            self.dbsUtil.setBlockFiles(binds = self.filesToUpdate)
            self.filesToUpdate = []
            myThread.transaction.commit()
        except WMException:
            myThread.transaction.rollback()
            raise
        except Exception as ex:
            msg =  "Unhandled exception while setting blocks in files.\n"
            msg += str(ex)
            logging.error(msg)