Python Classifier.Classifier类代码示例

def expected_case(cli: Classifier, percept: list) -> Classifier:
    """

    :rtype: Classifier
    """
    diff = get_differences(cli.mark, percept)
    if diff == [cons.symbol] * cons.lenCondition:
        cli.q += cons.beta * (1 - cli.q)
        return None
    else:
        spec = number_of_spec(cli.condition)
        spec_new = number_of_spec(diff)
        child = Classifier(cli)
        if spec == cons.uMax:
            remove_random_spec_att(child.condition)
            spec -= 1
            while spec + spec_new > cons.beta:
                if spec > 0 and random() < 0.5:
                    remove_random_spec_att(child.condition)
                    spec -= 1
                else:
                    remove_random_spec_att(diff)
                    spec_new -= 1
        else:
            while spec + spec_new > cons.beta:
                remove_random_spec_att(diff)
                spec_new -= 1
        child.condition = diff
        if child.q < 0.5:
            child.q = 0.5
        child.exp = 1
        assert isinstance(child, Classifier), 'Should be a Classifier'
        return child

 def __init__( self, k, mode = 0, distanceFunction = None) :
     self.k = k
     Classifier.__init__( self)
     self.logger.setDebugLevel( 0 )
     self.logger.setFileDebugLevel( 3 )
     self.distances = {}
     self.mode = mode
     self.dist = distanceFunction
     if(self.dist == None):
         self.dist = self.calculateDistance

def main():
    try:
        trainingData, tuningData, testData, priorSpam = buildDataSets()
        nbc = Classifier(priorSpam, COUNT_THRESHOLD, SMOOTHING_FACTOR, DEFAULT_PROBABILITY)
        # nbc = Classifier2(priorSpam, 0, .01, None)
        nbc.train(trainingData)

        nbc.classify(testData)
        report(testData)

    except Exception as e:
        print e
        return 5

def run(procId, procCount):
    connection = PgSQL.connect(user = "postgres", database = DatabaseName);
    memDb = redis.Redis( host='localhost', port=6379 );
    TrainDbConfig = DbBuildConfig['train'];
    TestDbConfig = DbBuildConfig['test'];
    trainDocDb = DocumentsDatabase(connection, 
                                   TrainDbConfig['DocTagsTable'], 
                                   TrainDbConfig['RawDocTable'], 
                                   TrainDbConfig['TagsTable'], 
                                   TrainDbConfig['DocumentsTable'] );
    testDocDb = DocumentsDatabase(connection, 
                                  TestDbConfig['DocTagsTable'], 
                                  TestDbConfig['RawDocTable'], 
                                  TestDbConfig['TagsTable'], 
                                  TestDbConfig['DocumentsTable'] );
    trainFeatureDb = FeatureDatabase(connection, 
                                     memDb, 
                                     trainDocDb, 
                                     TrainDbConfig['FeaturesTable'], 
                                     TrainDbConfig['DocFeaturesTable'],
                                     TrainDbConfig['TagSpecificFeatureTable']);
    testFeatureDb = FeatureDatabase(connection, 
                                    memDb, 
                                    testDocDb, 
                                    TestDbConfig['FeaturesTable'], 
                                    TestDbConfig['DocFeaturesTable'],
                                    TestDbConfig['TagSpecificFeatureTable']);

    classifier = Classifier(connection, trainFeatureDb, testFeatureDb, 
                   ClassifierTableConfig['predictedTrain'],
                   ClassifierTableConfig['predictedTest'], trainDocDb);

#    if procId == 0:
 #       classifier.createTables();
  #      classifier.createTagPredictTables();
   #     classifier.cleanClassificationTables();

    tags = trainDocDb.getTagsList();
    count = 0;
    for tag in tags:
        count = count + 1;
        if count % procCount != procId:
            continue;
        if count < 9000:
            continue;
        print "Processing ", tag, " ", count;
        c1 = trainDocDb.getTagCount(tag);
        if c1 <= 23:
            continue;
        classifier.predictForTag( tag );

    def captureFrameforAnalysis(self):
        try:
            self.mydatasetlist.get(self.mydatasetlist.curselection())
            datasetName = self.mydatasetlist.get(self.mydatasetlist.curselection())
            dataset_path = "binData/"+datasetName+".npz"
            print dataset_path

            img = cv2.cvtColor(self.current_frame, cv2.COLOR_BGR2RGB)
            cv2.namedWindow("CurrentFrame",cv2.WINDOW_NORMAL)
            cv2.imshow("CurrentFrame",img)

            cl = Classifier(img)
            cl.classifieSample(dataset_path)
        except:
            tkMessageBox.showerror("Error","Please pick a Dataset")

class Classifier_controller:

    def __init__(self):
        self.tf_idf = self.create_tf_idf()
        self.df_list = self.create_df_list()
        self.classes = self.create_classes()
        self.classifier = Classifier()

    def create_tf_idf(self):
        tf_idf = []
        os.system("pwd")
        path1 = './tutorial/data/tf_idf'
        classes = os.listdir(path1)
        for each_class in classes:
            path2 = path1 + '/' + each_class
            files = os.listdir(path2)
            for each_file in files:
                path3 = path2 + '/' + each_file     
                vector = dict()
                f = open(path3)
                dimes = f.readlines()
                f.close()
                i = 1
                for dime in dimes:
                    if float(dime) != 0.0:
                        vector[i] = float(dime)
                    i += 1
                tf_idf.append( (int(each_class), vector.items()) )
                print "creating class sample_vector...\n"
        print "finished..."
        return tf_idf

    def create_df_list(self):
        df_list = []
        f1 = open('./tutorial/data/df.dat')
        f2 = open('./tutorial/data/attribute.dat')
        df_records = f1.readlines()
        att_records = f2.readlines()
        f1.close()
        f2.close()
        i = 0
        for df in df_records:
            attribute = att_records[i].strip('\n')
            i += 1
            df_list.append((attribute, int(df)))
            print "reading %s %d\n" %(attribute, int(df))
        print "finished..."
        return df_list

    def create_classes(self):
        classes = []
        f = open('./tutorial/data/classes.dat')
        for each in f.readlines():
            classes.append(each.strip('\n'))
        f.close()
        return classes

    def get_classes(self, text, k):
        i = self.classifier.fun(text, self.df_list, len(self.tf_idf), self.tf_idf, k)
        return self.classes[i]

def apply_mutation(cl: Classifier, perception: list):
    """
    :type cl: Classifier
    :param cl:
    :type perception: list
    :param perception:
    :return:
    """
    for i in range(len(cl.condition)):
        if rd.random() < cons.nu:
            if cl.condition[i] == cons.dontCare:
                cl.condition[i] = perception[i]
            else:
                cl.condition[i] = cons.dontCare
    if rd.random() < cons.nu:
        c = rd.choice([i for i in range(0, (cons.nbAction - 1))])
        cl.action = c

def gen_match_set(pop: list, percept: list):
    """
    Generate a list of Classifier thats match current perception
    :param pop:
    :type pop: list
    :param percept:
    :type percept: list
    :return:
    :rtype: list
    """
    ma = []
    if time == 0 or len(pop) == 0:
        for i in range(cons.nbAction):
            newcl = Classifier()
            newcl.condition = [cons.symbol] * cons.lenCondition
            newcl.action = i
            newcl.effect = [cons.symbol] * cons.lenCondition
            newcl.exp = 0
            newcl.t = time
            newcl.q = 0.5
            pop.append(newcl)
    for c in pop:
        if does_match(c, percept):
            ma.append(c)
    return ma

def main(sc):
    start = timer()

    #### 1) Recuperando os produtos da base de dados
    #categs = ["Computers & Tablets", "Video Games", "TV & Home Theater"]# , ]
    stpwrds = stopwords.words('portuguese')
    products = findProductsByCategory([])
    
    print '####### Creating product rdd with {} product'.format(len(products))
    
    productRDD = sc.parallelize(products)
    #productRDD, discardedProductRDD = entiryProductRDD.randomSplit([2, 8], seed=0L)
   

    #### 2) Criadno o corpus de documento utilizando 
    corpusRDD = productRDD.map(lambda s: (s[0], word_tokenize(s[1].lower()), s[2], s[3])).map(lambda s: (s[0], [PorterStemmer().stem(x) for x in s[1] if x not in stpwrds], s[2], s[3] )).map(lambda s: (s[0], [x[0] for x in pos_tag(s[1]) if x[1] == 'NN' or x[1] == 'NNP'], s[2], s[3])).cache()

    idfsRDD = idfs(corpusRDD)
    idfsRDDBroadcast = sc.broadcast(idfsRDD.collectAsMap())
    tfidfRDD = corpusRDD.map(lambda x: (x[0], tfidf(x[1], idfsRDDBroadcast.value), x[2], x[3]))
    category = productRDD.map(lambda x: x[2]).distinct().collect()
    categoryAndSubcategory = productRDD.map(lambda x: (x[2], x[3])).distinct().collect()
    tokens = corpusRDD.flatMap(lambda x: x[1]).distinct().collect()

    insertTokensAndCategories(tokens, category, categoryAndSubcategory)
    
    classifier = Classifier(sc, 'NaiveBayes')   
    
    
    trainingVectSpaceCategoryRDD, testVectSpaceCategoryRDD = classifier.createVectSpaceCategory(tfidfRDD, category, tokens).randomSplit([8, 2], seed=0L)
    modelNaiveBayesCategory = classifier.trainModel(trainingVectSpaceCategoryRDD, '/dados/models/naivebayes/category_new')
    predictionAndLabelCategoryRDD = testVectSpaceCategoryRDD.map(lambda p : (category[int(modelNaiveBayesCategory.predict(p.features))], category[int(p.label)]))
    acuraccyCategory = float(predictionAndLabelCategoryRDD.filter(lambda (x, v): x[0] == v[0]).count())/float(predictionAndLabelCategoryRDD.count())
    print 'the accuracy of the Category Naive Bayes model is %f' % acuraccyCategory

    trainingVectSpaceSubcategory, testVectSpaceSubcategory = classifier.createVectSpaceSubcategory(tfidfRDD, categoryAndSubcategory, tokens).randomSplit([8, 2], seed=0L)
    modelNaiveBayesSubcategory = classifier.trainModel(trainingVectSpaceSubcategory, '/dados/models/naivebayes/subcategory_new')

    predictionAndLabelSubcategory = testVectSpaceSubcategory.map(lambda p : (categoryAndSubcategory[int(modelNaiveBayesSubcategory.predict(p.features))], categoryAndSubcategory[int(p.label)]))
    acuraccySubcategory = float(predictionAndLabelSubcategory.filter(lambda (x, v): x[0] == v[0]).count())/float(predictionAndLabelSubcategory.count())
    print 'the accuracy of the Subcategory Naive Bayes model is %f' % acuraccySubcategory

    #test with DecisionTree Model
    classifierDT = Classifier(sc, 'DecisionTree')
    trainingVectSpaceCategory, testVectSpaceCategory = classifierDT.createVectSpaceCategory(tfidfRDD, category, tokens).randomSplit([8, 2], seed=0L)
    modelDecisionTreeCategory = classifierDT.trainModel(trainingVectSpaceCategory, '/dados/models/dt/category_new')

    predictions = modelDecisionTreeCategory.predict(testVectSpaceCategory.map(lambda x: x.features))
    predictionAndLabelCategory = testVectSpaceCategory.map(lambda lp: lp.label).zip(predictions)
    acuraccyDecisionTree = float(predictionAndLabelCategory.filter(lambda (x, v): x == v).count())/float(predictionAndLabelCategory.count())   
    print 'the accuracy of the Decision Tree model is %f' % acuraccyDecisionTree

    elap = timer()-start
    print 'it tooks %d seconds' % elap

def validateClassifier():
    cl = Classifier(dataset="binData/classificationTrainingPalmahim100.npz",regression=False)
    path_list = ["../data/training_classification/positive", "../data/training_classification/negative"]
    kmeans_path = 'binData/KmeansBlobsPalmahim100.pkl'
    #cl.classificationValidation(path_list, kmeans,kernel='linear',gamma=None,C=1)            
    Cs = [0.001,0.002,0.003,0.004]
    gammas = [0.1]
    kernels = ["rbf","linear"]
    for kernel in kernels:
        if kernel == 'linear':
            gamma = None
            for C in Cs:
                cl.classificationValidation(path_list, kmeans,kernel=kernel,gamma=gamma,C=C)        
        else:
            for gamma in gammas:
                for C in Cs:
                    cl.classificationValidation(path_list, kmeans,kernel=kernel,gamma=gamma,C=C)
    
    cv.waitKey()

 def confirmPush(self):
     limbList = []
     for p in self.selection:
         if self.selection[p] == 1:
             limbList.append(p)
     self.pbar.setValue(0)
     homedir = os.getcwd()
     filt = Filter(homedir)
     filt.dataProcess()
     self.pbar.setValue(25)
     select = RandomSelector(homedir)
     select.dataProcess()
     self.pbar.setValue(50)
     st = StaticAnalyzer(homedir,limbList)
     st.dataProcess()
     self.pbar.setValue(75)
     c = Classifier(homedir)
     count,rate,total,result = c.staticClassify()
     self.pbar.setValue(100)
     reply = QtGui.QMessageBox.question(self, 'Static Analysis Result',"Total number is %d"%(total)+"\nCorrect number is %d"%(count)+"\nCorrect rate is %f"%(100*rate)+"%", QtGui.QMessageBox.Yes)

 def __init__(self,files,chanNum):
     self.signal = []
     self.stimulusCode = []
     self.phaseInSequence = []
     self.targetLetters = []
     self.firsttrain = 1
     self.cl = Classifier()
     self.sf = SpatialFilter(chanNum)
     self.rate = 0
     self.files = files
     self.chanNum = chanNum

def update_application_average(cli: Classifier, t: int):
    """
    Update Classifier's parameters aav
    :type t: int
    :param t: Time
    :type cli: Classifier
    """
    if cli.exp < 1 / cons.beta:
        cli.aav += (t - cli.tga - cli.aav) / cli.exp
    else:
        cli.aav += cons.beta * (t - cli.tga - cli.aav)
    cli.tga = t

def load_classifier(neighbours, blur_scale, c=None, gamma=None, verbose=0):
    classifier_file = 'classifier_%s_%s.dat' \
            % (blur_scale, neighbours)
    classifier_path = DATA_FOLDER + classifier_file

    if exists(classifier_file):
        if verbose:
            print 'Loading classifier...'

        classifier = Classifier(filename=classifier_path, \
                neighbours=neighbours, verbose=verbose)
    elif c != None and gamma != None:
        if verbose:
            print 'Training new classifier...'

        classifier = Classifier(c=c, gamma=gamma, neighbours=neighbours, \
                verbose=verbose)
        learning_set = load_learning_set(neighbours, blur_scale, \
                verbose=verbose)
        classifier.train(learning_set)
        classifier.save(classifier_path)
    else:
        raise Exception('No soft margin and gamma specified.')

    return classifier

def run():
    connection = PgSQL.connect(user = "postgres", database = DatabaseName);
    memDb = redis.Redis( host='localhost', port=6379 );
    TrainDbConfig = DbBuildConfig['train'];
    TestDbConfig = DbBuildConfig['test'];
    trainDocDb = DocumentsDatabase(connection, 
                                   TrainDbConfig['DocTagsTable'], 
                                   TrainDbConfig['RawDocTable'], 
                                   TrainDbConfig['TagsTable'], 
                                   TrainDbConfig['DocumentsTable'] );
    trainFeatureDb = FeatureDatabase(connection, 
                                     memDb, 
                                     trainDocDb, 
                                     TrainDbConfig['FeaturesTable'], 
                                     TrainDbConfig['DocFeaturesTable'],
                                     TrainDbConfig['TagSpecificFeatureTable']);
    testFeatureDb = FeatureDatabase(connection, 
                                    memDb, 
                                    None, 
                                    TestDbConfig['FeaturesTable'], 
                                    TestDbConfig['DocFeaturesTable'],
                                    TestDbConfig['TagSpecificFeatureTable']);

    classifier = Classifier(connection, trainFeatureDb, testFeatureDb, 
                   ClassifierTableConfig['predictedTrain'],
                   ClassifierTableConfig['predictedTest'], trainDocDb);

#    classifier.createTables();
    classifier.createTagPredictTables();
    classifier.cleanClassificationTables();
    tags = trainDocDb.getTagsList();
    s1 = 0;
    s2 = 0;
    for tag in tags:
        features = trainFeatureDb.getTagSpecificFeatures( tag );
        testTag = tag;
        hashes = trainFeatureDb.getTagSpecificFeatures(testTag);
        if not hashes:
            continue;
        c1 = trainDocDb.getTagCount(testTag);
        if c1 <= 25:
            continue;
        s1 += c1;
        print classifier.predictForTag( tag );
    classifier.saveClassificationResults();