Python policy.Policy类代码示例

 def testPolicyCopy(self):
     p = Policy.createPolicy(os.path.join(proddir, "examples", "EventTransmitter_policy.paf"))
     pp = Policy(p, True)
     self.assertEqual(p.get("transmitter.serializationFormat"), "deluxe")
     self.assertEqual(pp.getString("transmitter.serializationFormat"), "deluxe")
     p = None
     self.assertEqual(pp.getString("transmitter.serializationFormat"), "deluxe")

class BigBoolTestCase(unittest.TestCase):

    def setUp(self):
        self.policy = Policy()

    def tearDown(self):
        del self.policy

    def testBigBoolArray(self):
        biglen = 1000
        self.policy.isBool("true")
        for i in range(biglen):
            self.policy.add("true", True)

        v = self.policy.getArray("true")
        self.assertEqual(len(v), biglen, "wrong big number of values in array")
        for i in range(biglen):
            self.assertTrue(v[i], "big array with bad value")

        del v
        self.assertTrue(True, "Blew up True")

        fd = open("/dev/null", "w")
        print("look: %s" % True, file=fd)
        fd.close()

    def setup(self):
        deffile = DefaultPolicyFile("ctrl_sched","GetAJob_dict.paf","policies")
        defpol = Policy.createPolicy(deffile, deffile.getRepositoryPath())

        if not hasattr(self,"policy") or not self.policy:
            self.policy = Policy()
        self.policy.mergeDefaults(defpol.getDictionary())

        self.jobid = None
        self.tagLogger(None)

#        self.mode = self.policy.getString("mode")
#        if self.mode not in "parallel serial":
#            raise RuntimeError("Stage %s: Unsupported mode: %s" %
#                               (self.getName(), self.mode))

        self.clipboardKeys = {}
        self.clipboardKeys["jobIdentity"] = \
           self.policy.getString("outputKeys.jobIdentity")
        self.clipboardKeys["inputDatasets"] = \
           self.policy.getString("outputKeys.inputDatasets")
        self.clipboardKeys["outputDatasets"] = \
           self.policy.getString("outputKeys.outputDatasets")
        self.clipboardKeys["completedDatasets"] = \
           self.policy.getString("outputKeys.completedDatasets")
        self.log.log(Log.INFO-1, "clipboard keys: " + str(self.clipboardKeys))

        topic = self.policy.getString("pipelineEvent")
        self.client = GetAJobClient(self.getRun(), self.getName(), topic,
                                    self.getEventBrokerHost(), self.log)
        self.log.log(Log.INFO-1,
                     "Using OriginatorId = %d" % self.client.getOriginatorId())

 def testSetNothing(self):
     p = Policy()
     try:
         p.set("foo", None)
         self.assert_(False, "Setting value to None succeeded.")
     except RuntimeError:
         self.assertFalse(p.exists("foo"))

 def testPolicyCopy(self):
     p = Policy.createPolicy("examples/EventTransmitter_policy.paf")
     pp = Policy(p, True)
     self.assertEquals(p.get("transmitter.serializationFormat"), "deluxe")
     self.assertEquals(pp.getString("transmitter.serializationFormat"), "deluxe")
     p = None
     self.assertEquals(pp.getString("transmitter.serializationFormat"), "deluxe")

    def main(self, argv=None):
        self.parseArgs(argv)

        # 1. load policy
        policy = self.tryThis(Policy,
                              "Reading policy file \"" + self.policyFile + "\"",
                              self.policyFile)

        # resolve policy file references
        polLoadDir = self.options.loadPolicy
        polLoadDesc = polLoadDir
        if polLoadDir is None:
            if self.policyFile.find("/") >= 0:
                polLoadDir = self.policyFile.rpartition("/")[0]
                polLoadDesc = polLoadDir
            else:
                polLoadDir = ""
                polLoadDesc = "current directory; " \
                              "try -l DIR or --load-policy-references=DIR"
            if self.verbose:
                print("No policy load dir specified; defaulting to " + polLoadDesc)
        message = "Resolving references in " + self.policyFile \
                  + ",\n    using " + polLoadDesc
        self.tryThis(policy.loadPolicyFiles, message, polLoadDir, True)

        # derive short name by trimming off path & suffix
        shortname = self.policyFile
        if shortname.endswith(".paf"):
            shortname = shortname.rpartition(".paf")[0]
        if shortname.find("/") >= 0:
            shortname = shortname.rpartition("/")[2]

        if self.verbose:
            print("Short name =", shortname)

        # 2. create a dictionary from it
        dictionary = Policy()

        dictionary.set("target", shortname)
        self.generateDict(policy, dictionary)

        # TODO: remove commas from lists
        if self.verbose:
            print("Generating Dictionary for Policy " + self.policyFile + ":")
            print("---------------------------Policy---------------------------")
            print(policy)
            print("-------------------------Dictionary-------------------------")

        realDict = Dictionary(dictionary)
        try:
            realDict.validate(policy)
            print("#<?cfg paf dictionary ?>")
            writer = PAFWriter()
            writer.write(dictionary)
            print(writer.toString())
            # print(dictionary)
        except lsst.pex.exceptions.Exception as e:
            ve = e.args[0]
            print("Internal error: validation fails against extrapolated dictionary:")
            print(ve.describe("  - "))

    def testProcessDataset(self):
        with self.bb.queues:
            self.assertEquals(self.bb.queues.dataAvailable.length(), 0)

        policy = Policy.createPolicy(os.path.join(exampledir,
                                                  "ccdassembly-joboffice.paf"))
        spolicy = policy.getPolicy("schedule")

        # manipulate the policy
        idp = Policy.createPolicy(PolicyString(idpolicy))
        spolicy.set("job.identity", idp)
        
        self.sched = DataTriggeredScheduler(self.bb, spolicy, self.logger)

        # pdb.set_trace()
        ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=15)
        self.sched.processDataset(ds)

        with self.bb.queues:
            self.assertEquals(self.bb.queues.dataAvailable.length(), 1)
            self.assertEquals(self.bb.queues.jobsPossible.length(), 1)
            job = self.bb.queues.jobsPossible.get(0)
            self.assertEquals(job.getName(), "Job-1")
            self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 15)
            self.assertEquals(self.sched.nameNumber, 2)
    
        ds = Dataset("PostISR", visitid=95, ccdid=22, snapid=0, ampid=15)
        self.sched.processDataset(ds)

        with self.bb.queues:
            self.assertEquals(self.bb.queues.dataAvailable.length(), 2)
            self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
            job = self.bb.queues.jobsPossible.get(1)
            self.assertEquals(job.getName(), "Job-2")
            self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 15)
            inputs = job.getInputDatasets()
            self.assertEquals(len(inputs), 16)
            self.assertEquals(inputs[0].type, "PostISR")
            self.assertEquals(self.sched.nameNumber, 3)

        ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=14)
        self.sched.processDataset(ds)

        with self.bb.queues:
            self.assertEquals(self.bb.queues.dataAvailable.length(), 3)
            self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
            job = self.bb.queues.jobsPossible.get(0)
            self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 14)

        # pdb.set_trace()
        for i in xrange(14):
            ds = Dataset("PostISR", visitid=88, ccdid=22, snapid=0, ampid=i)
            self.sched.processDataset(ds)

        with self.bb.queues:
            self.assertEquals(self.bb.queues.dataAvailable.length(), 17)
            self.assertEquals(self.bb.queues.jobsPossible.length(), 2)
            job = self.bb.queues.jobsPossible.get(0)
            self.assertEquals(job.triggerHandler.getNeededDatasetCount(), 0)
            self.assert_(job.isReady())

def main():
    """
    run the job office
    """
    (cl.opts, cl.args) = cl.parse_args()

    if len(cl.args) == 0:
        fail("Missing policy file")
    if not os.path.exists(cl.args[0]):
        fail("%s: policy file not found" % cl.args[0])
    if not os.path.exists(cl.opts.rootdir):
        fail("%s: root directory not found" % cl.opts.rootdir)
    if not cl.opts.runid:
        logger.log(Log.WARN, "No RunID given (via -r)")

    defpolf = DefaultPolicyFile("ctrl_sched", "JobOffice_dict.paf", "policies")
    policy = Policy.createPolicy(cl.args[0])
    policy.mergeDefaults(Policy.createPolicy(defpolf,
                                             defpolf.getRepositoryPath()))
    name = policy.getString("name")

    
    # set job office root directory
    if not os.path.isabs(cl.opts.rootdir):
        cl.opts.rootdir = os.path.abspath(cl.opts.rootdir)
    persistdir = os.path.join(cl.opts.rootdir, name)
    if policy.exists("persist.dir"):
        persistdir = policy.get("persist.dir") % \
                     {"schedroot": cl.opts.rootdir, "name": name }

    # create the logger(s)
    logfile = cl.opts.logfile
    if not logfile:
        logfile = os.path.join(persistdir, "joboffice.log")
    if not os.path.exists(logfile):
        if not os.path.exists(os.path.dirname(logfile)):
            os.makedirs(os.path.dirname(logfile))
    
    if not cl.opts.asdaemon or cl.opts.toscreen:
        ofclogger = DualLog(logfile, Log.DEBUG, Log.DEBUG, False)
        # logging bug workaround
        ofclogger.setScreenVerbose(False)
    else:
        ofclogger = Log()
        ofclogger.addDestination(logfile)
    ofclogger.setThreshold(run.verbosity2threshold(cl.opts.logverb, 0))
    ofclogger.log(-2,"office threshold: %i" % ofclogger.getThreshold())

    try:
        # create the JobOffice instance
        office = createJobOffice(cl.opts.rootdir, policy, ofclogger, 
                                 cl.opts.runid, cl.opts.brokerhost,
                                 cl.opts.brokerport)
    except Exception, ex:
        logger.log(Log.FATAL, "Failed to create job office: " + str(ex))
        raise
        sys.exit(1)

    def testFromPolicy(self):
        type = "CalExp"
        path = "goob/CalExp-v88-c12.fits"
        ccdid = 12
        visitid = 88

        p = Policy()
        p.set("type", type)
        # pdb.set_trace()
        ds = Dataset.fromPolicy(p)
        self.assertEquals(ds.type, type)
        self.assert_(ds.path is None)
        self.assert_(ds.ids is None)

        p.set("path", path)
        ds = Dataset.fromPolicy(p)
        self.assertEquals(ds.type, type)
        self.assertEquals(ds.path, path)
        self.assert_(ds.ids is None)

        p.set("ids.ccdid", ccdid)
        p.set("ids.visitid", visitid)
        ds = Dataset.fromPolicy(p)
        self.assertEquals(ds.type, type)
        self.assertEquals(ds.path, path)
        self.assert_(ds.ids is not None)
        self.assertEquals(ds.ids["ccdid"], ccdid)
        self.assertEquals(ds.ids["visitid"], visitid)

 def __init__(self, policyfile=None):
     """
     create an item with the given properties.
     @param policyfile    A policy
     """
     # the properties attached to this items
     self._props = None
     if policyfile:
         self._props = Policy.createPolicy(policyfile)
     else:
         self._props = Policy()

    def testRepos(self):
        # when the repository is mis-specified, local files cannot be loaded
        upf = UrnPolicyFile("pex_policy:tests:urn/indirect_parent_good.paf")
        # we expect it to look in <package>/tests/simple.paf
        expectedFile = os.environ["PEX_POLICY_DIR"] + "/tests/simple.paf"
        self.assertRaisesEx(IoErrorException,
                            "failure opening Policy file: " + expectedFile,
                            upf.load, "Wrong repository dir.", Policy())

        # a PAF file designed to have "tests" as it repository
        p = Policy()
        UrnPolicyFile("pex_policy:tests:urn/local_tests_repos.paf").load(p)
        self.assert_(p.get("local.polish") == "fancy")

    def testRepos(self):
        # when the repository is mis-specified, local files cannot be loaded
        upf = UrnPolicyFile("pex_policy:tests:urn/indirect_parent_good.paf")
        # we expect it to look in <package>/tests/simple.paf
        pexPolicyDir = lsst.utils.getPackageDir('pex_policy')
        expectedFile = pexPolicyDir + "/tests/simple.paf"
        self.assertRaiseLCE(lsst.pex.exceptions.IoError,
                            "failure opening Policy file: " + expectedFile,
                            upf.load, "Wrong repository dir.", Policy())

        # a PAF file designed to have "tests" as it repository
        p = Policy()
        UrnPolicyFile("pex_policy:tests:urn/local_tests_repos.paf").load(p)
        self.assertEqual(p.get("local.polish"), "fancy")

    def testWrongType(self):
        d = Dictionary(self.getTestDictionary("types_dictionary.paf"))

        p = Policy(self.getTestDictionary("types_policy_bad_bool.paf"))
        ve = ValidationError("Dictionary_1.py", 0, "testWrongType")
        d.validate(p, ve.cpp)
        self.assert_(ve.getErrors() == ValidationError.WRONG_TYPE, "wrong type")
        self.assert_(ve.getParamCount() == 5, "number of errors")
        self.assert_(ve.getErrors("bool_type") == 0, "correct type")
        
        p = Policy(self.getTestDictionary("types_policy_bad_int.paf"))
        ve = ValidationError("Dictionary_1.py", 1, "testWrongType")
        d.validate(p, ve.cpp)
        self.assert_(ve.getErrors() == ValidationError.WRONG_TYPE, "wrong type")
        self.assert_(ve.getParamCount() == 5, "number of errors")
        self.assert_(ve.getErrors("int_type") == 0, "correct type")
        self.assert_(ve.getErrors("double_type") == ValidationError.WRONG_TYPE,
                     "int can't pass as double")
        
        p = Policy(self.getTestDictionary("types_policy_bad_double.paf"))
        ve = ValidationError("Dictionary_1.py", 2, "testWrongType")
        d.validate(p, ve.cpp)
        self.assert_(ve.getErrors() == ValidationError.WRONG_TYPE, "wrong type")
        self.assert_(ve.getParamCount() == 5, "number of errors")
        self.assert_(ve.getErrors("double_type") == 0, "correct type")

        p = Policy(self.getTestDictionary("types_policy_bad_string.paf"))
        ve = ValidationError("Dictionary_1.py", 3, "testWrongType")
        d.validate(p, ve.cpp)
        self.assert_(ve.getErrors() == ValidationError.WRONG_TYPE, "wrong type")
        self.assert_(ve.getParamCount() == 5, "number of errors")
        self.assert_(ve.getErrors("string_type") == 0, "correct type")

        p = Policy(self.getTestDictionary("types_policy_bad_policy.paf"))
        ve = ValidationError("Dictionary_1.py", 4, "testWrongType")
        d.validate(p, ve.cpp)
        self.assert_(ve.getErrors() == ValidationError.WRONG_TYPE, "wrong type")
        self.assert_(ve.getParamCount() == 4, "number of errors")
        self.assert_(ve.getErrors("policy_type") == 0, "correct type")
        self.assert_(ve.getErrors("file_type") == 0, "correct type")

        p = Policy(self.getTestDictionary("types_policy_bad_file.paf"))
        ve = ValidationError("Dictionary_1.py", 5, "testWrongType")
        d.validate(p, ve.cpp)
        self.assert_(ve.getErrors() == ValidationError.NOT_LOADED, "not loaded")
        self.assert_(ve.getParamCount() == 6, "number of errors")
        self.assert_(ve.getErrors("bool_type") == ValidationError.NOT_LOADED,
                     "not loaded")
        self.assert_(ve.getErrors("file_type") == ValidationError.NOT_LOADED,
                     "not loaded")
        self.assert_(ve.getErrors("policy_type") == ValidationError.NOT_LOADED,
                     "not loaded")
        p.loadPolicyFiles(self.getTestDictionary(), True)
        ve = ValidationError("Dictionary_1.py", 6, "testWrongType")
        d.validate(p, ve.cpp)
        self.assert_(ve.getErrors() == ValidationError.WRONG_TYPE, "wrong type")
        self.assert_(ve.getErrors("file_type") == 0, "correct type")
        self.assert_(ve.getErrors("policy_type") == 0, "correct type")
        self.assert_(ve.getParamCount() == 4, "wrong type")

 def setUp(self):
     p = Policy()
     p.set("foo", "bar")
     p.set("count", 3)
     p.set("files", "goob")
     p.add("files", "gurn")
     impl = bb.PolicyBlackboardItem()
     impl._props = p
     self.bbi = bb.ImplBlackboardItem(impl)
         
     self.initCount = 3

    def testReference(self):
        addr = "pex_policy:examples:EventTransmitter_policy.paf"
        p = Policy()

        p.set("transmitter.logVerbosity", "not")
        UrnPolicyFile(addr).load(p)
        self.assertEqual(p.get("transmitter.logVerbosity"), "debug")

        p.set("transmitter.logVerbosity", "not")
        UrnPolicyFile("urn:eupspkg:" + addr).load(p)
        self.assertEqual(p.get("transmitter.logVerbosity"), "debug")

        p.set("transmitter.logVerbosity", "not")
        UrnPolicyFile("@@" + addr).load(p)
        self.assertEqual(p.get("transmitter.logVerbosity"), "debug")