Python CouchCollection.CouchCollection类代码示例

    def failedJobs(self, failedJobs, useMask = True):
        """
        _failedJobs_

        Given a list of failed jobs, sort them into Filesets and record them

        NOTE: jobs must have a non-standard task, workflow, owner and group
        attributes assigned to them.
        """
        for job in failedJobs:
            try:
                taskName = job['task']
                workflow = job['workflow']
            except KeyError as ex:
                msg =  "Missing required, non-standard key %s in job in ACDC.DataCollectionService" % (str(ex))
                logging.error(msg)
                raise ACDCDCSException(msg)

            coll = CouchCollection(database = self.database, url = self.url,
                                   name = workflow,
                                   type = CollectionTypes.DataCollection)
            owner = self.newOwner(job.get("group", "cmsdataops"),
                                  job.get("owner", "cmsdataops"))
            coll.setOwner(owner)
            fileset = CouchFileset(database = self.database, url = self.url,
                                    name = taskName)
            coll.addFileset(fileset)
            if useMask:
                fileset.add(files = job['input_files'], mask = job['mask'])
            else:
                fileset.add(files = job['input_files'])

        return

    def setupACDCDatabase(self, collectionName, taskPath,
                          user, group):
        """
        _setupACDCDatabase_

        Populate an ACDC database with bogus records
        associated to certain collection name, user and task path.
        """
        acdcServer = CouchService(url = self.testInit.couchUrl,
                                  database = "%s_acdc" % self.couchDBName)
        owner = acdcServer.newOwner(group, user)
        testCollection = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl,
                                          name = collectionName)
        testCollection.setOwner(owner)
        testFileset = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = taskPath)
        testCollection.addFileset(testFileset)

        testFiles = []
        for _ in range(5):
            testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
                            events = random.randint(1024, 4096))
            testFiles.append(testFile)

        testFileset.add(testFiles)

    def testFileset(self):
        """
        _testFileset_

        Verify that converting an ACDC fileset to a DataStructs fileset works
        correctly.
        """
        testCollection = CouchCollection(database = self.testInit.couchDbName,
                                         url = self.testInit.couchUrl, 
                                         name = "Thunderstruck")
        testCollection.setOwner(self.owner)
        testFileset = CouchFileset(database = self.testInit.couchDbName,
                                   url = self.testInit.couchUrl,
                                   name = "TestFileset")
        testCollection.addFileset(testFileset)

        testFiles = {}
        for i in range(5):
            lfn = makeUUID()
            testFile = File(lfn = lfn, size = random.randint(1024, 4096),
                            events = random.randint(1024, 4096))
            testFiles[lfn] = testFile
            testFileset.add([testFile])

        for file in testFileset.fileset().files:
            self.assertTrue(file["lfn"] in testFiles.keys(),
                            "Error: File missing.")
            self.assertEqual(file["events"], testFiles[file["lfn"]]["events"],
                             "Error: Wrong number of events.")
            self.assertEqual(file["size"], testFiles[file["lfn"]]["size"],
                             "Error: Wrong file size.")
        return

    def getDataCollection(self, collName):
        """
        _getDataCollection_

        Get a data collection by name
        """
        coll = CouchCollection(name=collName, database=self.database, url=self.url)

        coll.populate()
        return coll

    def failedJobs(self, failedJobs, useMask=True):
        """
        _failedJobs_

        Given a list of failed jobs, sort them into Filesets and record them

        NOTE: jobs must have a non-standard task, workflow attributes assigned to them.
        """
        # first we sort the list of dictionary by two keys: workflow then task
        failedJobs.sort(key=itemgetter('workflow'))
        failedJobs.sort(key=itemgetter('task'))

        previousWorkflow = ""
        previousTask = ""
        for job in failedJobs:
            try:
                workflow = job['workflow']
                taskName = job['task']
            except KeyError as ex:
                msg = "Missing required, non-standard key %s in job in ACDC.DataCollectionService" % (str(ex))
                logging.error(msg)
                raise ACDCDCSException(msg)

            if workflow != previousWorkflow:
                coll = CouchCollection(database=self.database, url=self.url,
                                       name=workflow,
                                       type=CollectionTypes.DataCollection)
            if taskName != previousTask:
                fileset = CouchFileset(database=self.database, url=self.url,
                                       name=taskName)
            coll.addFileset(fileset)
            inputFiles = job['input_files']
            for fInfo in inputFiles:
                if int(fInfo["merged"]) == 1:  # Looks like Oracle and MySQL return diff type
                    fInfo["parents"] = []
                elif fInfo.get("parents", []):
                    firstParent = next(iter(fInfo["parents"]))
                    if "/store/unmerged/" in firstParent:
                        # parents and input files are unmerged files - need to find merged ascendant
                        fInfo["parents"] = list(getMergedParents(fInfo["parents"]))
                    elif "MCFakeFile" in firstParent:
                        fInfo["parents"] = []
                    # other case, fInfo["parents"] all or merged parents
            if useMask:
                fileset.add(files=inputFiles, mask=job['mask'])
            else:
                fileset.add(files=inputFiles)

            previousWorkflow = workflow
            previousTask = taskName

        return

    def getDataCollection(self, collName, user = "cmsdataops",
                          group = "cmsdataops"):
        """
        _getDataCollection_

        Get a data collection by name
        """
        coll = CouchCollection(name = collName, database = self.database,
                               url = self.url)

        coll.owner = self.newOwner(group, user)
        coll.populate()
        return coll

    def listCollections(self, owner):
        """
        _listCollections_

        List the collections belonging to an owner.
        """
        params = {"startkey": [owner.group.name, owner.name],
                  "endkey": [owner.group.name, owner.name, {}],
                  "reduce": True, "group_level": 3}

        result = self.couchdb.loadView("ACDC", "owner_coll_fileset_docs",
                                       params)

        for row in result["rows"]:
            coll = CouchCollection(name = row["key"][2],
                                   database = self.database, url = self.url)
            coll.setOwner(owner)
            coll.populate()
            yield coll

    def createCollection(self, collectionName, userName, groupName):
        """
        _createCollection_

        Create an empty AnalysisCollection
        """

        if userName == None:
            msg = "WMSpec does not contain an owner.name parameter"
            raise RuntimeError(msg)
        if groupName == None:
            msg = "WMSpec does not contain an owner.group parameter"
            raise RuntimeError(msg)

        user = self.newOwner(groupName, userName)
        collection = CouchCollection(
            name=collectionName, type=CollectionTypes.AnalysisCollection, url=self.url, database=self.database
        )
        collection.setOwner(user)
        collection.create()

        return collection

#.........这里部分代码省略.........
                         'lumis' : intersectionDiff,
                         'outMod' : outputModulesToRecover,
                         'outputs' : datasetsToRecover}
        requests.append(requestObject)
    # Now go through all the output datasets, creating a single request for
    # each
    for dataset in differenceInformation:
        datasetsToRecover = [dataset]
        diffedLumis = differenceInformation[dataset]
        if (inputDataset, dataset) in edges:
            taskToRecover = edges[(inputDataset, dataset)]['task']
            outputModulesToRecover = edges[(inputDataset,dataset)]['outMod']
            parentDataset = inputDataset
        else:
            for parentDataset in nodes:
                if dataset in nodes[parentDataset]:
                    taskToRecover = edges[(parentDataset, dataset)]['task']
                    outputModulesToRecover = edges[(parentDataset,dataset)]['outMod']
                    break
        requestObject = {'task' : taskToRecover,
                         'input' : parentDataset,
                         'lumis' : diffedLumis,
                         'outMod' : outputModulesToRecover,
                         'outputs' : datasetsToRecover}
        requests.append(requestObject)

    logging.info("About to upload ACDC records to: %s/%s" % (acdcCouchUrl, acdcCouchDb))
    pprint(requests)
    # With the request objects we need to build ACDC records and
    # request JSONs
    for idx, requestObject in enumerate(requests):
        collectionName = '%s_%s' % (workload.name(), str(uuid.uuid1()))
        filesetName = requestObject['task']
        collection = CouchCollection(**{"url" : acdcCouchUrl,
                                      "database" : acdcCouchDb,
                                      "name" : collectionName})
        owner = makeUser(workload.getOwner()['group'], workload.getOwner()['name'], acdcCouchUrl, acdcCouchDb)
        collection.setOwner(owner)
        files = 0
        lumis = 0
        for lfn in datasetInformation[requestObject['input']]:
            fileInfo = datasetInformation[requestObject['input']][lfn]
            fileRuns = {}
            for run in fileInfo['runs']:
                if run in requestObject['lumis']:
                    for lumi in fileInfo['runs'][run][0]:
                        if lumi in requestObject['lumis'][run]:
                            if run not in fileRuns:
                                fileRuns[run] = []
                            fileRuns[run].append(lumi)
                            lumis += 1
            if fileRuns:
                files += 1
                fileset = CouchFileset(**{"url" : acdcCouchUrl,
                                        "database" : acdcCouchDb,
                                        "name" : filesetName})
                fileset.setCollection(collection)
                acdcRuns = []
                for run in fileRuns:
                    runObject = {}
                    runObject['run_number'] = int(run)
                    runObject['lumis'] = fileRuns[run]
                    acdcRuns.append(runObject)
                acdcFile = {"lfn" : lfn,
                            "first_event" : 0,
                            "last_event" : 0,

    def testDropCount(self):
        """
        _testDropCount_

        Verify that dropping a fileset and counting the files in a fileset works
        correctly.
        """
        testCollectionA = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl, 
                                          name = "Thunderstruck")
        testCollectionB = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl, 
                                          name = "StruckThunder")
        testCollectionA.setOwner(self.owner)
        testCollectionB.setOwner(self.owner)        

        testFiles = []
        for i in range(5):
            testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
                            events = random.randint(1024, 4096))
            testFiles.append(testFile)

        testFilesetA = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = "TestFilesetA")
        testFilesetB = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = "TestFilesetB")
        testFilesetC = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = "TestFilesetC")
        testCollectionA.addFileset(testFilesetA)
        testCollectionB.addFileset(testFilesetB)
        testCollectionB.addFileset(testFilesetC)
        testFilesetA.add(testFiles)
        testFilesetB.add(testFiles)
        testFilesetC.add(testFiles)        

        testFilesetC.drop()

        testCollectionC = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl, 
                                          name = "StruckThunder")
        testCollectionC.setOwner(self.owner)
        testCollectionC.populate()

        self.assertEqual(len(testCollectionC["filesets"]), 1,
                         "Error: There should be one fileset in this collection.")
        self.assertEqual(testCollectionC["filesets"][0].fileCount(), 5,
                         "Error: Wrong number of files in fileset.")

        testCollectionD = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl, 
                                          name = "Thunderstruck")
        testCollectionD.setOwner(self.owner)
        testCollectionD.populate()

        self.assertEqual(len(testCollectionD["filesets"]), 1,
                         "Error: There should be one fileset in this collection.")
        self.assertEqual(testCollectionD["filesets"][0].fileCount(), 5,
                         "Error: Wrong number of files in fileset.")        
        return

    def testListCollectionsFilesets(self):
        """
        _testListCollectionsFilesets_

        Verify that collections and filesets in ACDC can be listed.
        """
        svc = CouchService(url = self.testInit.couchUrl,
                           database = self.testInit.couchDbName)

        ownerA = svc.newOwner("somegroup", "someuserA")
        ownerB = svc.newOwner("somegroup", "someuserB")
        
        testCollectionA = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl, 
                                          name = "Thunderstruck")
        testCollectionA.setOwner(ownerA)
        testCollectionB = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl, 
                                          name = "Struckthunder")
        testCollectionB.setOwner(ownerA)
        testCollectionC = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl, 
                                          name = "Thunderstruck")
        testCollectionC.setOwner(ownerB)
        testCollectionD = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl, 
                                          name = "Thunderstruck")
        testCollectionD.setOwner(ownerB)        
        
        testFilesetA = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = "TestFilesetA")
        testCollectionA.addFileset(testFilesetA)
        testFilesetB = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = "TestFilesetB")
        testCollectionB.addFileset(testFilesetB)
        testFilesetC = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = "TestFilesetC")
        testCollectionC.addFileset(testFilesetC)
        testFilesetD = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = "TestFilesetD")
        testCollectionC.addFileset(testFilesetD)        

        testFiles = []
        for i in range(5):
            testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
                            events = random.randint(1024, 4096))
            testFiles.append(testFile)
            
        testFilesetA.add(testFiles)
        testFilesetB.add(testFiles)
        testFilesetC.add(testFiles)
        testFilesetD.add(testFiles)

        goldenCollectionNames = ["Thunderstruck", "Struckthunder"]
        for collection in svc.listCollections(ownerA):
            self.assertTrue(collection["name"] in goldenCollectionNames,
                            "Error: Missing collection name.")
            goldenCollectionNames.remove(collection["name"])
        self.assertEqual(len(goldenCollectionNames), 0,
                         "Error: Missing collections.")

        goldenFilesetNames = ["TestFilesetC", "TestFilesetD"]
        for fileset in svc.listFilesets(testCollectionD):
            self.assertTrue(fileset["name"] in goldenFilesetNames,
                            "Error: Missing fileset.")
            goldenFilesetNames.remove(fileset["name"])
        self.assertEqual(len(goldenFilesetNames), 0,
                         "Error: Missing filesets.")

        return

    def testL_CascadeCloseOutAnnnouncement(self):
        """
        _testL_CascadeCloseOutAnnouncement_

        Test the cascade closeout REST call, also
        check that when announced a request deletes all ACDC records in the system.
        """
        userName     = 'Taizong'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")

        result = self.jsonSender.put("request", schema)[0]
        originalRequest = result['RequestName']
        self.setupACDCDatabase(originalRequest, "/%s/DataProcessing" % originalRequest,
                               result['Requestor'], result['Group'])
        depth = 2
        nReq = 3
        requests = [originalRequest]
        def createChildrenRequest(parentRequest, i, nReq):
            createdRequests = []
            resubSchema = utils.getResubmissionSchema(parentRequest, "/%s/DataProcessing" % parentRequest,
                                                      groupName, userName)
            result = self.jsonSender.put("request", resubSchema)[0]
            requestName = result['RequestName']
            self.setupACDCDatabase(requestName, "/%s/DataProcessing" % requestName, result['Requestor'], result['Group'])
            createdRequests.append(requestName)
            if i:
                for _ in range(nReq):
                    createdRequests.extend(createChildrenRequest(requestName, i - 1, nReq))
            return createdRequests
        requests.extend(createChildrenRequest(originalRequest, depth, nReq))
        for request in requests:
            self.changeStatusAndCheck(request, 'assignment-approved')
        for request in requests:
            self.jsonSender.put("assignment?team=%s&requestName=%s" % (teamName, request))
        for status in ['acquired',
                       'running-open', 'running-closed',
                       'completed']:
            for request in requests:
                self.changeStatusAndCheck(request, status)
        self.jsonSender.post('closeout?requestName=%s&cascade=True' % originalRequest)
        svc = CouchService(url = self.testInit.couchUrl,
                                  database = "%s_acdc" % self.couchDBName)

        owner = svc.newOwner(groupName, userName)
        for request in requests:
            result = self.jsonSender.get('request/%s' % request)
            self.assertEqual(result[0]['RequestStatus'], 'closed-out')
            testCollection = CouchCollection(database = self.testInit.couchDbName,
                                             url = self.testInit.couchUrl,
                                             name = request)
            testCollection.setOwner(owner)
            testCollection.populate()
            self.assertNotEqual(len(testCollection["filesets"]), 0)

        self.jsonSender.post('announce?requestName=%s&cascade=True' % originalRequest)
        for request in requests:

            result = self.jsonSender.get('request/%s' % request)
            self.assertEqual(result[0]['RequestStatus'], 'announced')
            testCollection = CouchCollection(database = self.testInit.couchDbName,
                                             url = self.testInit.couchUrl,
                                             name = request)
            testCollection.setOwner(owner)
            testCollection.populate()
            self.assertEqual(len(testCollection["filesets"]), 0)

    def populateCouchDB(self):
        """
        _populateCouchDB_

        Populate the ACDC records
        """
        svc = CouchService(url=self.testInit.couchUrl, database=self.testInit.couchDbName)

        ownerA = svc.newOwner("somegroup", "someuserA")
        ownerB = svc.newOwner("somegroup", "someuserB")

        testCollectionA = CouchCollection(
            database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="Thunderstruck"
        )
        testCollectionA.setOwner(ownerA)
        testCollectionB = CouchCollection(
            database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="Struckthunder"
        )
        testCollectionB.setOwner(ownerA)
        testCollectionC = CouchCollection(
            database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="Thunderstruck"
        )
        testCollectionC.setOwner(ownerB)
        testCollectionD = CouchCollection(
            database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="Thunderstruck"
        )
        testCollectionD.setOwner(ownerB)

        testFilesetA = CouchFileset(database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="TestFilesetA")
        testCollectionA.addFileset(testFilesetA)
        testFilesetB = CouchFileset(database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="TestFilesetB")
        testCollectionB.addFileset(testFilesetB)
        testFilesetC = CouchFileset(database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="TestFilesetC")
        testCollectionC.addFileset(testFilesetC)
        testFilesetD = CouchFileset(database=self.testInit.couchDbName, url=self.testInit.couchUrl, name="TestFilesetD")
        testCollectionC.addFileset(testFilesetD)

        testFiles = []
        for i in range(5):
            testFile = File(lfn=makeUUID(), size=random.randint(1024, 4096), events=random.randint(1024, 4096))
            testFiles.append(testFile)

        testFilesetA.add(testFiles)
        time.sleep(1)
        testFilesetB.add(testFiles)
        time.sleep(1)
        testFilesetC.add(testFiles)
        time.sleep(2)
        testFilesetD.add(testFiles)

    def testCreatePopulateDrop(self):
        """
        _testCreatePopulateDrop_

        Test creating, populating and dropping a collection.
        """
        testCollectionA = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl,
                                          name = "Thunderstruck")
        testCollectionB = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl,
                                          name = "StruckThunder")
        testCollectionA.setOwner(self.owner)
        testCollectionB.setOwner(self.owner)
        testCollectionA.create()
        testCollectionB.create()

        # There should be nothing in couch.  Documents are only added for
        # filesets and files.

        testFilesA = []
        for i in range(5):
            testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
                            events = random.randint(1024, 4096))
            testFilesA.append(testFile)
        testFilesB = []
        for i in range(10):
            testFile = File(lfn = makeUUID(), size = random.randint(1024, 4096),
                            events = random.randint(1024, 4096))
            testFilesB.append(testFile)

        testFilesetA = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = "TestFilesetA")
        testFilesetB = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = "TestFilesetB")
        testFilesetC = CouchFileset(database = self.testInit.couchDbName,
                                    url = self.testInit.couchUrl,
                                    name = "TestFilesetC")
        testCollectionA.addFileset(testFilesetA)
        testCollectionB.addFileset(testFilesetB)
        testCollectionB.addFileset(testFilesetC)
        testFilesetA.add(testFilesA)
        testFilesetB.add(testFilesA)
        testFilesetC.add(testFilesA)
        testFilesetC.add(testFilesB)

        # Drop testCollectionA
        testCollectionA.drop()

        # Try to populate testFilesetA
        testCollectionC = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl,
                                          name = "ThunderStruck")
        testCollectionC.setOwner(self.owner)
        testCollectionC.populate()

        self.assertEqual(len(testCollectionC["filesets"]), 0,
                         "Error: There should be no filesets in this collect.")

        # Try to populate testFilesetB
        testCollectionD = CouchCollection(database = self.testInit.couchDbName,
                                          url = self.testInit.couchUrl,
                                          name = "StruckThunder")
        testCollectionD.setOwner(self.owner)
        testCollectionD.populate()

        for fileset in testCollectionD["filesets"]:
            testFiles = testFilesA
            if fileset["name"] == "TestFilesetC":
                testFiles.extend(testFilesB)

            self.assertEqual(len(testFiles), len(fileset.files.keys()),
                             "Error: Wrong number of files in fileset.")
            for testFile in testFiles:
                self.assertTrue(testFile["lfn"] in fileset.files.keys(),
                                "Error: File is missing.")
                self.assertEqual(testFile["events"],
                                 fileset.files[testFile["lfn"]]["events"],
                                 "Error: Wrong number of events.")
                self.assertEqual(testFile["size"],
                                 fileset.files[testFile["lfn"]]["size"],
                                 "Error: Wrong file size.")

        return