Python A类代码示例

def classify(X_train, X_test, y_train):
    '''
    Train the best classifier on (X_train, and y_train) then predict X_test labels

    :param X_train: A dictionary with the following structure
            { instance_id: [w_1 count, w_2 count, ...],
            ...
            }

    :param X_test: A dictionary with the following structure
            { instance_id: [w_1 count, w_2 count, ...],
            ...
            }

    :param y_train: A dictionary with the following structure
            { instance_id : sense_id }

    :return: results: a list of tuples (instance_id, label) where labels are predicted by the best classifier
    '''

# create x, y lists from training datas

    x_train_list, y_train_list = A.x_y_lists_from_training(X_train, y_train)

    # train svm
    print 'training svm...'
    svm_clf = svm.LinearSVC()
    svm_clf.fit(x_train_list, y_train_list)

    # predict svm results
    print 'predicting svm...'
    svm_results = A.predictions_from_data(svm_clf, X_test)

    return svm_results

def run(train, test, language, answer):
    results = {}
    if language == 'English':
        _POS_TAGGER = 'taggers/maxent_treebank_pos_tagger/english.pickle'
        tagger = load(_POS_TAGGER)
    elif language == 'Spanish':
        tagger = ut(cess_esp.tagged_sents())
    elif language == 'Catalan':
        tagger  = ut(cess_cat.tagged_sents())

    for lexelt in train:

        train_features, y_train = extract_features(train[lexelt],language,tagger)
        test_features, _ = extract_features(test[lexelt],language,tagger)

        X_train, X_test = vectorize(train_features,test_features)
        X_train_new, X_test_new = feature_selection(X_train, X_test,y_train)
        results[lexelt] = classify(X_train_new, X_test_new,y_train)
    """
    B1.c
    for lexelt in train:
        features = getBestWords(train[lexelt], 30)
        train_features = countFeature(features, train[lexelt])
        _, y_train = extract_features(train[lexelt], language)
        test_features = countFeature(features, test[lexelt])

        X_train, X_test = vectorize(train_features, test_features)
        results[lexelt] = classify(X_train, X_test, y_train)
    B1.c
    """
    A.print_results(results, answer)

def main(aligned_sents):
    ba = BerkeleyAligner(aligned_sents, 10)
    A.save_model_output(aligned_sents, ba, "ba.txt")
    avg_aer = A.compute_avg_aer(aligned_sents, ba, 50)
    print ('Berkeley Aligner')
    print ('---------------------------')
    print('Average AER: {0:.3f}\n'.format(avg_aer))

def run(train, test, language, answer):
    results = {}

    total = len(train)
    counter = 1

    s = build_s(train, language)
    #s = {}

    # if language == 'English':
    #     tagger = set_tagger(language)
    # else:
    tagger = None
    #tagger = set_tagger(language)
    stemmer = set_stemmer(language)

    for lexelt in train:
        train_features, y_train = extract_features(train[lexelt], language, tagger, stemmer, s[lexelt])
        test_features, _ = extract_features(test[lexelt], language, tagger, stemmer, s[lexelt])

        X_train, X_test = vectorize(train_features,test_features)
        X_train_new, X_test_new = feature_selection(X_train, X_test,y_train, language)
        results[lexelt] = classify(X_train_new, X_test_new,y_train)

        print str(counter) + ' out of ' + str(total) + ' completed'
        counter += 1

    A.print_results(results, answer)

def new(line):
    line = line.strip()
    if A.accept(line):
        return A.new(line)
    elif C.accept(line):
        return C.new(line)
    else:
        raise SyntaxError("Unknown instruction", (None, -1, 0, line))

def main(aligned_sents):
    time.clock()
    ba = BerkeleyAligner(aligned_sents, 10)
    A.save_model_output(aligned_sents, ba, "ba.txt")
    avg_aer = A.compute_avg_aer(aligned_sents, ba, 50)

    print ('Berkeley Aligner')
    print ('---------------------------')
    print('Average AER: {0:.3f}\n'.format(avg_aer))
    print "Part B time: " + str(time.clock()) + ' sec'

def main(aligned_sents):
    print 'training regular berkeley model'
    iters = 10
    ba = BerkeleyAligner(aligned_sents, iters)
    A.save_model_output(aligned_sents, ba, "ba.txt")
    avg_aer = A.compute_avg_aer(aligned_sents, ba, 50, 'berk_errs.txt')

    print ('Berkeley Aligner')
    print ('iterations:' + str(iters))
    print ('---------------------------')
    print('Average AER: {0:.3f}\n\n\n'.format(avg_aer))

def main(aligned_sents):
    ba = BerkeleyAligner(aligned_sents, 10)
    A.save_model_output(aligned_sents, ba, "ba.txt")
    avg_aer = A.compute_avg_aer(aligned_sents, ba, 50)
    #Report aer for each sentence of first 20 sentences
    for i,aligned_sent in enumerate(aligned_sents[:20]):
	print "ba , aer of sentence "+str(i)+" "+str(A.compute_avg_aer([aligned_sent],ba,1))

    print ('Berkeley Aligner')
    print ('---------------------------')
    print('Average AER: {0:.3f}\n'.format(avg_aer))

    def testCode (self):

        x = A.a2('a2m-value', 'legal')
        self.assertEqual('a2m-value', x.a2member)
        self.assertEqual(B.bst.legal, x.a2b)

        myobj = B.b1(x, 'legal')
        self.assertEqual(myobj.a2elt, x)

        x2 = A.a2('anotherValue', 'legal')
        myobj.a2elt = x2
        self.assertEqual('anotherValue', myobj.a2elt.a2member)
        self.assertEqual(B.bst.legal, myobj.a2elt.a2b)

def run(train, test, language, answer):
    results = {}

    for lexelt in train:

        train_features, y_train = extract_features(train[lexelt])
        test_features, _ = extract_features(test[lexelt])

        X_train, X_test = vectorize(train_features,test_features)
        X_train_new, X_test_new = feature_selection(X_train, X_test,y_train)
        results[lexelt] = classify(X_train_new, X_test_new,y_train)

    A.print_results(results, answer)

def run(train, test, language, answer):
    results = {}
    #calc_high_frequency_words(train)
    print 'Calling A'
    s = A.build_s(train)
    for lexelt in train:

        train_features, y_train = extract_features(train[lexelt],language,lexelt,s[lexelt])
        test_features, _ = extract_features(test[lexelt],language,lexelt,s[lexelt])

        X_train, X_test = vectorize(train_features,test_features)
        X_train_new, X_test_new = feature_selection(X_train, X_test,y_train,language)
        results[lexelt] = classify(X_train_new, X_test_new,y_train)
    A.print_results(results, answer)
    print 'ended'

def run(train, test, language, answer):
    results = {}
    l = len(train)
    for i, lexelt in enumerate(train):
        sys.stdout.write('\r{} / {} ({}%)'.format(i, l, int(float(i) / l * 100)))
        sys.stdout.flush()

        train_features, y_train = extract_features(train[lexelt], language)
        test_features, _ = extract_features(test[lexelt], language)

        X_train, X_test = vectorize(train_features,test_features)
        X_train_new, X_test_new = feature_selection(X_train, X_test,y_train)
        results[lexelt] = classify(X_train_new, X_test_new,y_train)

    A.print_results(results, answer)

def add_k_word_features_count_to_vector(vector, left_tokens, right_tokens, window_size, head=None):
    words = A.k_nearest_words_vector_from_tokens(left_tokens, right_tokens, window_size)
    for word in words:
        vector[word] = vector[word] + 1 if word in vector else 1

    if head:
        vector[head] = 1

def main():
    if len(sys.argv) != 7:
        print 'Usage: python main.py <input_training file> <input test file> <output KNN file> <output SVM file> <output best file> <language>'
        sys.exit(0)

    train_file = sys.argv[1]
    test_file = sys.argv[2]
    knn_answer = sys.argv[3]
    svm_answer = sys.argv[4]
    best_answer = sys.argv[5]
    language = sys.argv[6]

    train_set = parse_data(train_file)
    test_set = parse_data(test_file)

    A.run(train_set, test_set, language, knn_answer, svm_answer)
    B.run(train_set, test_set, language, best_answer)

def run(train, test, language, answer):
    results = {}

    if language == 'English': language = 'en'
    if language == 'Spanish': language = 'spa'
    if language == 'Catalan': language = 'cat'

    for lexelt in train:
        rel_dict = relevance(train[lexelt])
        train_features, y_train = extract_features(train[lexelt], language, rel_dict=rel_dict)
        test_features, _ = extract_features(test[lexelt], language, rel_dict=rel_dict)

        X_train, X_test = vectorize(train_features,test_features)
        X_train_new, X_test_new = feature_selection(X_train, X_test,y_train)
        results[lexelt] = classify(X_train_new, X_test_new,y_train)

    A.print_results(results, answer)