Python Utils.Parameters类代码示例

 def addClassifierModel(self, model, classifierModelPath, classifierParameters, threshold=None):
     classifierModel = model.get(self.tag+"classifier-model", True)
     shutil.copy2(classifierModelPath, classifierModel)
     model.addStr(self.tag+"classifier-parameter", Parameters.toString(Parameters.get(classifierParameters)))
     if threshold != None:
         model.addStr(self.tag+"threshold", str(threshold))
     return classifierModel

def getConnection(connection): #, account=None, workDirBase=None, remoteSettingsPath=None):
    if connection == None: # return a "dummy" local connection
        return getConnection("connection=Unix:jobLimit=1")
    elif type(connection) in types.StringTypes and hasattr(Settings, connection): # connection is a Settings key
        print >> sys.stderr, "Using connection", connection
        return getConnection(getattr(Settings, connection))
        #return getConnection(*getattr(Settings, connection))
    else: # connection is a parameter string or dictionary
        defaultParams = dict.fromkeys(["connection", "account", "workdir", "settings", "memory", "cores", "modules", "wallTime", "jobLimit", "preamble", "debug"])
        defaultParams["debug"] = False
        connection = Parameters.get(connection, valueListKey="connection", valueTypes={"debug":[bool]}, defaults=defaultParams)
        if connection["connection"] == None:
            connection["connection"] = "Unix"
        if connection["account"] == None:
            assert connection["workdir"] == None
            #assert remoteSettingsPath == None
            print >> sys.stderr, "New local connection", Parameters.toString(connection)
        else: 
            print >> sys.stderr, "New remote connection:", Parameters.toString(connection)
        # Make the connection
        exec "ConnectionClass = " + connection["connection"] + "Connection"
        connectionArgs = {}
        for key in connection:
            if key != "connection" and connection[key] != None:
                connectionArgs[key] = connection[key]
        return ConnectionClass(**connectionArgs)

def getSteps(step, omitSteps, mainSteps):
    # Determine substep to start from, for the main step from which processing starts
    step = Parameters.get(step, mainSteps)
    fromMainStep = None
    fromSubStep = {} # The substep to start from, for the main step to start from
    for mainStep in step.keys():
        fromSubStep[mainStep] = step[mainStep] # the sub step to start from
        if step[mainStep] != None:
            assert fromMainStep == None # processing can start from one place only
            fromMainStep = mainStep
            if step[mainStep] == True:
                fromSubStep[mainStep] = None
            else:
                assert type(step[mainStep]) in types.StringTypes # no list allowed, processing can start from one place only
    # Determine steps to omit
    omitSubSteps = {} # Skip these substeps. If the value is True, skip the entire main step.
    omitMainSteps = []
    omitSteps = Parameters.get(omitSteps, mainSteps)
    for mainStep in omitSteps.keys():
        omitSubSteps[mainStep] = omitSteps[mainStep]
        if omitSteps[mainStep] == True:
            omitMainSteps.append(mainStep)
            omitSubSteps[mainStep] = None
    # Initialize main step selector
    if fromMainStep != None:
        if fromSubStep[fromMainStep] != None:
            print >> sys.stderr, "Starting process from step", fromMainStep + ", substep", fromSubStep[fromMainStep]
        else:
            print >> sys.stderr, "Starting process from step", fromMainStep
    selector = StepSelector(mainSteps, fromStep=fromMainStep, omitSteps=omitMainSteps)
    return selector, fromSubStep, omitSubSteps

 def saveModel(self, teesModel, tag=""):
     if hasattr(self, "model") and self.model != None:
         teesModelPath = teesModel.get(tag+"classifier-model", True)
         shutil.copy2(self.model, teesModelPath)
     if hasattr(self, "parameters") and self.parameters != None:
         teesModel.addStr(tag+"classifier-parameter", Parameters.toString(Parameters.get(self.parameters)))
     if hasattr(self, "threshold") and self.threshold != None:
         teesModel.addStr(tag+"threshold", str(self.threshold))

 def optimize(self, examples, outDir, parameters, classifyExamples, classIds, step="BOTH", evaluator=None, determineThreshold=False, timeout=None, downloadAllModels=False):
     assert step in ["BOTH", "SUBMIT", "RESULTS"], step
     outDir = os.path.abspath(outDir)
     # Initialize training (or reconnect to existing jobs)
     combinations = Parameters.getCombinations(Parameters.get(parameters, valueListKey="c")) #Core.OptimizeParameters.getParameterCombinations(parameters)
     trained = []
     for combination in combinations:
         trained.append( self.train(examples, outDir, combination, classifyExamples, replaceRemoteExamples=(len(trained) == 0), dummy=(step == "RESULTS")) )
     if step == "SUBMIT": # Return already
         classifier = copy.copy(self)
         classifier.setState("OPTIMIZE")
         return classifier
     
     # Wait for the training to finish
     finalJobStatus = self.connection.waitForJobs([x.getJob() for x in trained])
     # Evaluate the results
     print >> sys.stderr, "Evaluating results"
     #Stream.setIndent(" ")
     bestResult = None
     if evaluator == None:
         evaluator = self.defaultEvaluator
     for i in range(len(combinations)):
         id = trained[i].parameterIdStr
         #Stream.setIndent(" ")
         # Get predictions
         predictions = None
         if trained[i].getStatus() == "FINISHED":
             predictions = trained[i].downloadPredictions()
         else:
             print >> sys.stderr, "No results for combination" + id
             continue
         if downloadAllModels:
             trained[i].downloadModel()
         # Compare to other results
         print >> sys.stderr, "*** Evaluating results for combination" + id + " ***"
         threshold = None
         if determineThreshold:
             print >> sys.stderr, "Thresholding, original micro =",
             evaluation = evaluator.evaluate(classifyExamples, predictions, classIds, os.path.join(outDir, "evaluation-before-threshold" + id + ".csv"), verbose=False)
             print >> sys.stderr, evaluation.microF.toStringConcise()
             threshold, bestF = evaluator.threshold(classifyExamples, predictions)
             print >> sys.stderr, "threshold =", threshold, "at binary fscore", str(bestF)[0:6]
         evaluation = evaluator.evaluate(classifyExamples, ExampleUtils.loadPredictions(predictions, threshold=threshold), classIds, os.path.join(outDir, "evaluation" + id + ".csv"))
         if bestResult == None or evaluation.compare(bestResult[0]) > 0: #: averageResult.fScore > bestResult[1].fScore:
             bestResult = [evaluation, trained[i], combinations[i], threshold]
         if not self.connection.isLocal():
             os.remove(predictions) # remove predictions to save space
     #Stream.setIndent()
     if bestResult == None:
         raise Exception("No results for any parameter combination")
     print >> sys.stderr, "*** Evaluation complete", finalJobStatus, "***"
     print >> sys.stderr, "Selected parameters", bestResult[2]
     classifier = copy.copy(bestResult[1])
     classifier.threshold = bestResult[3]
     classifier.downloadModel()
     return classifier

    def doGrid(self):
        print >> sys.stderr, "--------- Booster parameter search ---------"
        # Build trigger examples
        self.triggerDetector.buildExamples(self.model, [self.optData], [self.workDir+"grid-trigger-examples.gz"])

        if self.fullGrid:
            # Parameters to optimize
            ALL_PARAMS={
                "trigger":[int(i) for i in Parameters.get(self.triggerClassifierParameters, valueListKey="c")["c"]], 
                "booster":[float(i) for i in self.recallAdjustParameters.split(",")], 
                "edge":[int(i) for i in Parameters.get(self.edgeClassifierParameters, valueListKey="c")["c"]] }
        else:
            ALL_PARAMS={"trigger":Parameters.get(self.model.getStr(self.triggerDetector.tag+"classifier-parameter"), valueListKey="c")["c"],
                        "booster":[float(i) for i in self.recallAdjustParameters.split(",")],
                        "edge":Parameters.get(self.model.getStr(self.edgeDetector.tag+"classifier-parameter"), valueListKey="c")["c"]}
        
        paramCombinations = Parameters.getCombinations(ALL_PARAMS, ["trigger", "booster", "edge"])
        prevParams = None
        EDGE_MODEL_STEM = os.path.join(self.edgeDetector.workDir, os.path.normpath(self.model.path)+"-edge-models/model-c_")
        TRIGGER_MODEL_STEM = os.path.join(self.triggerDetector.workDir, os.path.normpath(self.model.path)+"-trigger-models/model-c_")
        bestResults = None
        for i in range(len(paramCombinations)):
            params = paramCombinations[i]
            print >> sys.stderr, "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
            print >> sys.stderr, "Processing params", str(i+1) + "/" + str(len(paramCombinations)), params
            print >> sys.stderr, "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
            # Triggers and Boost
            if prevParams == None or prevParams["trigger"] != params["trigger"] or prevParams["booster"] != params["booster"]:
                print >> sys.stderr, "Classifying trigger examples for parameters", "trigger:" + str(params["trigger"]), "booster:" + str(params["booster"])
                xml = self.triggerDetector.classifyToXML(self.optData, self.model, self.workDir+"grid-trigger-examples.gz", self.workDir+"grid-", classifierModel=TRIGGER_MODEL_STEM+str(params["trigger"]), recallAdjust=params["booster"])
            prevParams = params
            # Build edge examples
            self.edgeDetector.buildExamples(self.model, [xml], [self.workDir+"grid-edge-examples.gz"], [self.optData])
            # Classify with pre-defined model
            edgeClassifierModel=EDGE_MODEL_STEM+str(params["edge"])
            xml = self.edgeDetector.classifyToXML(xml, self.model, self.workDir+"grid-edge-examples.gz", self.workDir+"grid-", classifierModel=edgeClassifierModel)
            bestResults = self.evaluateGrid(xml, params, bestResults)
        print >> sys.stderr, "Booster search complete"
        print >> sys.stderr, "Tested", len(paramCombinations), "combinations"
        print >> sys.stderr, "Best parameters:", bestResults[0]
        print >> sys.stderr, "Best result:", bestResults[2] # f-score
        # Save grid model
        self.saveStr("recallAdjustParameter", str(bestResults[0]["booster"]), self.model)
        self.saveStr("recallAdjustParameter", str(bestResults[0]["booster"]), self.combinedModel, False)
        if self.fullGrid: # define best models
            self.triggerDetector.addClassifierModel(self.model, TRIGGER_MODEL_STEM+str(bestResults[0]["trigger"]), bestResults[0]["trigger"])
            self.edgeDetector.addClassifierModel(self.model, EDGE_MODEL_STEM+str(bestResults[0]["edge"]), bestResults[0]["edge"])
        # Remove work files
        for stepTag in [self.workDir+"grid-trigger", self.workDir+"grid-edge", self.workDir+"grid-unmerging"]:
            for fileStem in ["-classifications", "-classifications.log", "examples.gz", "pred.xml.gz"]:
                if os.path.exists(stepTag+fileStem):
                    os.remove(stepTag+fileStem)

 def trainUnmergingDetector(self):
     xml = None
     if not self.unmerging:
         print >> sys.stderr, "No unmerging"
     if self.checkStep("SELF-TRAIN-EXAMPLES-FOR-UNMERGING", self.unmerging) and self.unmerging:
         # Self-classified train data for unmerging
         if self.doUnmergingSelfTraining:
             # This allows limiting to a subcorpus
             triggerStyle = copy.copy(Parameters.get(self.triggerExampleStyle))
             edgeStyle = copy.copy(Parameters.get(self.edgeExampleStyle))
             unmergingStyle = Parameters.get(self.unmergingExampleStyle)
             if "sentenceLimit" in unmergingStyle and unmergingStyle["sentenceLimit"]:
                 triggerStyle["sentenceLimit"] = unmergingStyle["sentenceLimit"]
                 edgeStyle["sentenceLimit"] = unmergingStyle["sentenceLimit"]
             # Build the examples
             xml = self.triggerDetector.classifyToXML(self.trainData, self.model, None, self.workDir+"unmerging-extra-", exampleStyle=triggerStyle)#, recallAdjust=0.5)
             xml = self.edgeDetector.classifyToXML(xml, self.model, None, self.workDir+"unmerging-extra-", exampleStyle=edgeStyle)#, recallAdjust=0.5)
             assert xml != None
             EvaluateInteractionXML.run(self.edgeDetector.evaluator, xml, self.trainData, self.parse)
         else:
             print >> sys.stderr, "No self-training for unmerging"
     if self.checkStep("UNMERGING-EXAMPLES", self.unmerging) and self.unmerging:
         # Unmerging example generation
         GOLD_TEST_FILE = self.optData.replace("-nodup", "")
         GOLD_TRAIN_FILE = self.trainData.replace("-nodup", "")
         if self.doUnmergingSelfTraining:
             if xml == None: 
                 xml = self.workDir+"unmerging-extra-edge-pred.xml.gz"
             self.unmergingDetector.buildExamples(self.model, [self.optData.replace("-nodup", ""), [self.trainData.replace("-nodup", ""), xml]], 
                                                  [self.workDir+"unmerging-opt-examples.gz", self.workDir+"unmerging-train-examples.gz"], 
                                                  [GOLD_TEST_FILE, [GOLD_TRAIN_FILE, GOLD_TRAIN_FILE]], 
                                                  exampleStyle=self.unmergingExampleStyle, saveIdsToModel=True)
             xml = None
         else:
             self.unmergingDetector.buildExamples(self.model, [self.optData.replace("-nodup", ""), self.trainData.replace("-nodup", "")], 
                                                  [self.workDir+"unmerging-opt-examples.gz", self.workDir+"unmerging-train-examples.gz"], 
                                                  [GOLD_TEST_FILE, GOLD_TRAIN_FILE], 
                                                  exampleStyle=self.unmergingExampleStyle, saveIdsToModel=True)
             xml = None
         #UnmergingExampleBuilder.run("/home/jari/biotext/EventExtension/TrainSelfClassify/test-predicted-edges.xml", GOLD_TRAIN_FILE, UNMERGING_TRAIN_EXAMPLE_FILE, PARSE, TOK, UNMERGING_FEATURE_PARAMS, UNMERGING_IDS, append=True)
     if self.checkStep("BEGIN-UNMERGING-MODEL", self.unmerging) and self.unmerging:
         self.unmergingDetector.beginModel(None, self.model, self.workDir+"unmerging-train-examples.gz", self.workDir+"unmerging-opt-examples.gz")
     if self.checkStep("END-UNMERGING-MODEL", self.unmerging) and self.unmerging:
         self.unmergingDetector.endModel(None, self.model, self.workDir+"unmerging-opt-examples.gz")
         print >> sys.stderr, "Adding unmerging classifier model to test-set event model"
         if self.combinedModel != None:
             self.combinedModel.addStr("unmerging-example-style", self.model.getStr("unmerging-example-style"))
             self.combinedModel.insert(self.model.get("unmerging-ids.classes"), "unmerging-ids.classes")
             self.combinedModel.insert(self.model.get("unmerging-ids.features"), "unmerging-ids.features")
             self.unmergingDetector.addClassifierModel(self.combinedModel, self.model.get("unmerging-classifier-model", True), 
                                                       self.model.getStr("unmerging-classifier-parameter"))
             self.combinedModel.save()

    def classifyToXML(self, data, model, exampleFileName=None, tag="", classifierModel=None, goldData=None, parse=None, recallAdjust=None, compressExamples=True, exampleStyle=None):
        model = self.openModel(model, "r")
        if parse == None:
            parse = self.getStr(self.tag+"parse", model)
        if exampleFileName == None:
            exampleFileName = tag+self.tag+"examples"
            if compressExamples:
                exampleFileName += ".gz"
        self.buildExamples(model, [data], [exampleFileName], [goldData], parse=parse, exampleStyle=exampleStyle)
        if classifierModel == None:
            classifierModel = model.get(self.tag+"classifier-model")
        else:
            assert os.path.exists(classifierModel), classifierModel
        classifier = self.Classifier()
        classifier.classify(exampleFileName, tag+self.tag+"classifications", classifierModel, finishBeforeReturn=True)
        threshold = model.getStr(self.tag+"threshold", defaultIfNotExist=None, asType=float)
        predictions = ExampleUtils.loadPredictions(tag+self.tag+"classifications", recallAdjust, threshold=threshold)
        evaluator = self.evaluator.evaluate(exampleFileName, predictions, model.get(self.tag+"ids.classes"))
        #outputFileName = tag+"-"+self.tag+"pred.xml.gz"
        #exampleStyle = self.exampleBuilder.getParameters(model.getStr(self.tag+"example-style"))
        if exampleStyle == None:
            exampleStyle = Parameters.get(model.getStr(self.tag+"example-style")) # no checking, but these should already have passed the ExampleBuilder
        return self.exampleWriter.write(exampleFileName, predictions, data, tag+self.tag+"pred.xml.gz", model.get(self.tag+"ids.classes"), parse, exampleStyle=exampleStyle)
#        if evaluator.getData().getTP() + evaluator.getData().getFP() > 0:
#            return self.exampleWriter.write(exampleFileName, predictions, data, outputFileName, model.get(self.tag+"ids.classes"), parse)
#        else:
#            # TODO: e.g. interactions must be removed if task does unmerging
#            print >> sys.stderr, "No positive", self.tag + "predictions, XML file", outputFileName, "unchanged from input"
#            if type(data) in types.StringTypes: # assume its a file
#                shutil.copy(data, outputFileName)
#            else: # assume its an elementtree
#                ETUtils.write(data, outputFileName)
#            #print >> sys.stderr, "No positive predictions, XML file", tag+self.tag+"pred.xml", "not written"
#            return data #None

 def train(self, trainData=None, optData=None, model=None, combinedModel=None, exampleStyle=None, 
     classifierParameters=None, parse=None, tokenization=None, task=None, fromStep=None, toStep=None, 
     workDir=None):
     
     exampleStyle = Parameters.cat(exampleStyle, "keep_neg:no_features")
     EdgeDetector.train(self, trainData, optData, model, combinedModel, exampleStyle, classifierParameters, parse, tokenization, fromStep, toStep)
     self.classify(trainData, model, "classification-train/train", goldData=trainData, workDir="classification-train")

 def train(self, examples, outDir, parameters, classifyExamples=None, dummy=False):
     outDir = os.path.abspath(outDir)
     
     examples = self.getExampleFile(examples, dummy=dummy)
     classifyExamples = self.getExampleFile(classifyExamples, dummy=dummy)
     
     # Return a new classifier instance for following the training process and using the model
     classifier = copy.copy(self)
     classifier.parameters = parameters
     classifier._filesToRelease = [examples, classifyExamples]
     
     if not os.path.exists(outDir):
         os.makedirs(outDir)
     
     trainFeatures, trainClasses = datasets.load_svmlight_file(examples)
     if classifyExamples != None:
         develFeatures, develClasses = datasets.load_svmlight_file(classifyExamples, trainFeatures.shape[1])
     binarizer = preprocessing.LabelBinarizer()
     binarizer.fit(trainClasses)
     trainClasses = binarizer.transform(trainClasses)
     if classifyExamples != None:
         develClasses = binarizer.transform(develClasses)
     
     print >> sys.stderr, "Training Keras model with parameters:", parameters
     parameters = Parameters.get(parameters, {"TEES.classifier":"KerasClassifier", "layers":5, "lr":0.001, "epochs":1, "batch_size":64, "patience":10})
     np.random.seed(10)
     classifier.kerasModel = classifier._defineModel(outDir, parameters, trainFeatures, trainClasses, develFeatures, develClasses)
     classifier._fitModel(outDir, parameters, trainFeatures, trainClasses, develFeatures, develClasses)

 def getBioNLPSharedTaskParams(self, parameters=None, model=None):
     if parameters == None:
         if model != None:
             model = self.openModel(model, "r")
             parameters = model.getStr("BioNLPSTParams", defaultIfNotExist=None)
         else:
             parameters = {}
     return Parameters.get(parameters, ["convert", "evaluate", "scores", "a2Tag"])

 def getClassifier(self, parameters):
     #parameters = Parameters.get(parameters, ["TEES.threshold", "TEES.classifier", "c"], valueListKey="c")
     parameters = Parameters.get(parameters, ["TEES.threshold", "TEES.classifier"], allowNew=True, valueListKey="c")
     if parameters["TEES.classifier"] == None:
         return self.Classifier
     else:
         exec "from Classifiers." + parameters["TEES.classifier"] + " import " + parameters["TEES.classifier"] + " as " + parameters["TEES.classifier"]
         return eval(parameters["TEES.classifier"])

 def getParameters(self, parameters=None, model=None, defaultValue=None, modelParameterStringName=None):
     if modelParameterStringName == None:
         modelParameterStringName = self.modelParameterStringName
     if parameters == None and model != None:
         model = self.openModel(model, "r")
         parameters = model.getStr(modelParameterStringName, defaultIfNotExist=None)
     defaultStepNames = [x[0] for x in self.getDefaultSteps()]
     valueLimits={"omitSteps":defaultStepNames + [None], "intermediateFiles":defaultStepNames + [True, None]}
     defaults = self.getDefaultParameters(defaultValue=defaultValue)
     return Parameters.get(parameters, defaults, valueLimits=valueLimits)

 def train(self, examples, outDir, parameters, classifyExamples=None, finishBeforeReturn=False, replaceRemoteExamples=True, dummy=False):
     outDir = os.path.abspath(outDir)
     
     examples = self.getExampleFile(examples, replaceRemote=replaceRemoteExamples, dummy=dummy)
     classifyExamples = self.getExampleFile(classifyExamples, replaceRemote=replaceRemoteExamples, dummy=dummy)
     parameters = Parameters.get(parameters, valueListKey="c")
     trainDir = self.connection.getSetting(self.trainDirSetting)
     
     # Return a new classifier instance for following the training process and using the model
     classifier = copy.copy(self)
     classifier.setState("TRAIN")
     classifier.parameters = parameters
     classifier._filesToRelease = [examples, classifyExamples]
     # Train
     if not os.path.exists(outDir):
         os.makedirs(outDir)
     trainCommand = os.path.join(trainDir, self.trainCommand)
     paramKeys = sorted(parameters.keys())
     idStr = ""
     paramString = ""
     for key in paramKeys:
         if key.startswith("TEES."):
             continue
         if len(paramString) > 0 and not paramString.endswith(" "):
             paramString += " "
         if parameters[key] != None:
             paramString += self.parameterFormat.replace("%k", key).replace("%v", str(parameters[key])).strip()
             idStr += "-" + str(key) + "_" + str(parameters[key])
         else:
             paramString += self.parameterFormat.replace("%k", key).replace("%v", "").strip()
             idStr += "-" + str(key)
     classifier.parameterIdStr = idStr
     classifier.model = self.connection.getRemotePath(outDir + "/model" + idStr, True)
     modelPath = self.connection.getRemotePath(outDir + "/model" + idStr, False)
     trainCommand = trainCommand.replace("%p", paramString).replace("%e", examples).replace("%m", modelPath).strip()
     self.connection.addCommand(trainCommand)
     # Classify with the trained model (optional)
     if classifyExamples != None:
         classifier.predictions = self.connection.getRemotePath(outDir + "/predictions" + idStr, True)
         predictionsPath = self.connection.getRemotePath(outDir + "/predictions" + idStr, False)
         classifyDir = self.connection.getSetting(self.classifyDirSetting)
         classifyCommand = os.path.join(classifyDir, self.classifyCommand).replace("%e", classifyExamples).replace("%m", modelPath).replace("%c", predictionsPath).strip()
         self.connection.addCommand(classifyCommand)
     # Run the process
     jobName = self.trainCommand.split()[0] + idStr
     logPath = outDir + "/" + jobName
     if dummy: # return a classifier that connects to an existing job
         self.connection.clearCommands()
         classifier._job = self.connection.getJob(jobDir=outDir, jobName=jobName)
     else: # submit the job
         classifier._job = self.connection.submit(jobDir=outDir, jobName=jobName, stdout=logPath+".stdout")
         if finishBeforeReturn:
             self.connection.waitForJob(classifier._job)
             self.getStatus()
     return classifier

 def initModel(self, model, saveParams=[]):
     if model == None:
         return model
     elif type(model) in types.StringTypes:
         model = self.openModel(model, "w")
     else:
         assert model.mode in ["a", "w"]
     for param in saveParams:
         model.addStr(param[1], Parameters.toString(getattr(self, param[0])))
     model.save()
     return model