Python NaiveBayesClassifier.train方法代码示例

# 需要导入模块: from nltk.classify import NaiveBayesClassifier [as 别名]
# 或者: from nltk.classify.NaiveBayesClassifier import train [as 别名]
def split_train_test(all_instances, n=None):
    """
    Randomly split `n` instances of the dataset into train and test sets.

    :param all_instances: a list of instances (e.g. documents) that will be split.
    :param n: the number of instances to consider (in case we want to use only a
        subset).
    :return: two lists of instances. Train set is 8/10 of the total and test set
        is 2/10 of the total.
    """
    random.seed(12345)
    random.shuffle(all_instances)
    if not n or n > len(all_instances):
        n = len(all_instances)
    train_set = all_instances[:int(.8*n)]
    test_set = all_instances[int(.8*n):n]

    return train_set, test_set 

# 需要导入模块: from nltk.classify import NaiveBayesClassifier [as 别名]
# 或者: from nltk.classify.NaiveBayesClassifier import train [as 别名]
def demo_sent_subjectivity(text):
    """
    Classify a single sentence as subjective or objective using a stored
    SentimentAnalyzer.

    :param text: a sentence whose subjectivity has to be classified.
    """
    from nltk.classify import NaiveBayesClassifier
    from nltk.tokenize import regexp
    word_tokenizer = regexp.WhitespaceTokenizer()
    try:
        sentim_analyzer = load('sa_subjectivity.pickle')
    except LookupError:
        print('Cannot find the sentiment analyzer you want to load.')
        print('Training a new one using NaiveBayesClassifier.')
        sentim_analyzer = demo_subjectivity(NaiveBayesClassifier.train, True)

    # Tokenize and convert to lower case
    tokenized_text = [word.lower() for word in word_tokenizer.tokenize(text)]
    print(sentim_analyzer.classify(tokenized_text)) 

# 需要导入模块: from nltk.classify import NaiveBayesClassifier [as 别名]
# 或者: from nltk.classify.NaiveBayesClassifier import train [as 别名]
def __init__(
        self,
        train=None,
        model=None,
        affix_length=-3,
        min_stem_length=2,
        backoff=None,
        cutoff=0,
        verbose=False,
    ):

        self._check_params(train, model)

        ContextTagger.__init__(self, model, backoff)

        self._affix_length = affix_length
        self._min_word_length = min_stem_length + abs(affix_length)

        if train:
            self._train(train, cutoff, verbose) 

# 需要导入模块: from nltk.classify import NaiveBayesClassifier [as 别名]
# 或者: from nltk.classify.NaiveBayesClassifier import train [as 别名]
def split_train_test(all_instances, n=None):
    """
    Randomly split `n` instances of the dataset into train and test sets.

    :param all_instances: a list of instances (e.g. documents) that will be split.
    :param n: the number of instances to consider (in case we want to use only a
        subset).
    :return: two lists of instances. Train set is 8/10 of the total and test set
        is 2/10 of the total.
    """
    random.seed(12345)
    random.shuffle(all_instances)
    if not n or n > len(all_instances):
        n = len(all_instances)
    train_set = all_instances[: int(0.8 * n)]
    test_set = all_instances[int(0.8 * n) : n]

    return train_set, test_set 

# 需要导入模块: from nltk.classify import NaiveBayesClassifier [as 别名]
# 或者: from nltk.classify.NaiveBayesClassifier import train [as 别名]
def demo_sent_subjectivity(text):
    """
    Classify a single sentence as subjective or objective using a stored
    SentimentAnalyzer.

    :param text: a sentence whose subjectivity has to be classified.
    """
    from nltk.classify import NaiveBayesClassifier
    from nltk.tokenize import regexp

    word_tokenizer = regexp.WhitespaceTokenizer()
    try:
        sentim_analyzer = load('sa_subjectivity.pickle')
    except LookupError:
        print('Cannot find the sentiment analyzer you want to load.')
        print('Training a new one using NaiveBayesClassifier.')
        sentim_analyzer = demo_subjectivity(NaiveBayesClassifier.train, True)

    # Tokenize and convert to lower case
    tokenized_text = [word.lower() for word in word_tokenizer.tokenize(text)]
    print(sentim_analyzer.classify(tokenized_text)) 

# 需要导入模块: from nltk.classify import NaiveBayesClassifier [as 别名]
# 或者: from nltk.classify.NaiveBayesClassifier import train [as 别名]
def __init__(self, n, train=None, model=None,
                 backoff=None, cutoff=0, verbose=False):
        self._n = n
        self._check_params(train, model)

        ContextTagger.__init__(self, model, backoff)

        if train:
            self._train(train, cutoff, verbose) 

# 需要导入模块: from nltk.classify import NaiveBayesClassifier [as 别名]
# 或者: from nltk.classify.NaiveBayesClassifier import train [as 别名]
def train(self, graphs):
        """
        :type graphs: list(DependencyGraph)
        :param graphs: A list of dependency graphs to train the scorer.
        Typically the edges present in the graphs can be used as
        positive training examples, and the edges not present as negative
        examples.
        """
        raise NotImplementedError() 

# 需要导入模块: from nltk.classify import NaiveBayesClassifier [as 别名]
# 或者: from nltk.classify.NaiveBayesClassifier import train [as 别名]
def hall_demo():
    npp = ProbabilisticNonprojectiveParser()
    npp.train([], DemoScorer())
    for parse_graph in npp.parse(['v1', 'v2', 'v3'], [None, None, None]):
        print(parse_graph) 

# 需要导入模块: from nltk.classify import NaiveBayesClassifier [as 别名]
# 或者: from nltk.classify.NaiveBayesClassifier import train [as 别名]
def nonprojective_conll_parse_demo():
    from nltk.parse.dependencygraph import conll_data2

    graphs = [
        DependencyGraph(entry) for entry in conll_data2.split('\n\n') if entry
    ]
    npp = ProbabilisticNonprojectiveParser()
    npp.train(graphs, NaiveBayesDependencyScorer())
    for parse_graph in npp.parse(['Cathy', 'zag', 'hen', 'zwaaien', '.'], ['N', 'V', 'Pron', 'Adj', 'N', 'Punc']):
        print(parse_graph) 

# 需要导入模块: from nltk.classify import NaiveBayesClassifier [as 别名]
# 或者: from nltk.classify.NaiveBayesClassifier import train [as 别名]
def train(cls,
              docs: Collection[Document],
              cutoff: float = 0.1,
              idf_table: Optional[Mapping[Word, float]] = None,
              ) -> 'NaiveBayesSummarizer':
        """Train the model on a collection of documents.

        Args:
            docs (Collection[Document]): The collection of documents to train on.
            cutoff (float): Cutoff for signature words.
            idf_table (Mapping[Word, float]): Precomputed IDF table. If not given, the IDF
                will be computed from ``docs``.

        Returns:
            NaiveBayes: The trained model.
        """
        # Find signature words
        idf = cls._compute_idf(docs) if idf_table is None else idf_table
        n_cutoff = int(cutoff * len(idf))
        signature_words = set(sorted(
            idf.keys(), key=lambda w: idf[w], reverse=True)[:n_cutoff])

        train_data = []  # type: list
        for doc in docs:
            featuresets = cls._extract_featuresets(doc, signature_words)
            labels = [sent.label for sent in doc.sentences]
            train_data.extend(zip(featuresets, labels))
        return cls(
            NaiveBayesClassifier.train(train_data), signature_words=signature_words) 

# 需要导入模块: from nltk.classify import NaiveBayesClassifier [as 别名]
# 或者: from nltk.classify.NaiveBayesClassifier import train [as 别名]
def nonprojective_conll_parse_demo():
    from nltk.parse.dependencygraph import conll_data2

    graphs = [DependencyGraph(entry) for entry in conll_data2.split('\n\n') if entry]
    npp = ProbabilisticNonprojectiveParser()
    npp.train(graphs, NaiveBayesDependencyScorer())
    for parse_graph in npp.parse(
        ['Cathy', 'zag', 'hen', 'zwaaien', '.'], ['N', 'V', 'Pron', 'Adj', 'N', 'Punc']
    ):
        print(parse_graph) 

# 需要导入模块: from nltk.classify import NaiveBayesClassifier [as 别名]
# 或者: from nltk.classify.NaiveBayesClassifier import train [as 别名]
def demo_movie_reviews(trainer, n_instances=None, output=None):
    """
    Train classifier on all instances of the Movie Reviews dataset.
    The corpus has been preprocessed using the default sentence tokenizer and
    WordPunctTokenizer.
    Features are composed of:
        - most frequent unigrams

    :param trainer: `train` method of a classifier.
    :param n_instances: the number of total reviews that have to be used for
        training and testing. Reviews will be equally split between positive and
        negative.
    :param output: the output file where results have to be reported.
    """
    from nltk.corpus import movie_reviews
    from sentiment_analyzer import SentimentAnalyzer

    if n_instances is not None:
        n_instances = int(n_instances/2)

    pos_docs = [(list(movie_reviews.words(pos_id)), 'pos') for pos_id in movie_reviews.fileids('pos')[:n_instances]]
    neg_docs = [(list(movie_reviews.words(neg_id)), 'neg') for neg_id in movie_reviews.fileids('neg')[:n_instances]]
    # We separately split positive and negative instances to keep a balanced
    # uniform class distribution in both train and test sets.
    train_pos_docs, test_pos_docs = split_train_test(pos_docs)
    train_neg_docs, test_neg_docs = split_train_test(neg_docs)

    training_docs = train_pos_docs+train_neg_docs
    testing_docs = test_pos_docs+test_neg_docs

    sentim_analyzer = SentimentAnalyzer()
    all_words = sentim_analyzer.all_words(training_docs)

    # Add simple unigram word features
    unigram_feats = sentim_analyzer.unigram_word_feats(all_words, min_freq=4)
    sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
    # Apply features to obtain a feature-value representation of our datasets
    training_set = sentim_analyzer.apply_features(training_docs)
    test_set = sentim_analyzer.apply_features(testing_docs)

    classifier = sentim_analyzer.train(trainer, training_set)
    try:
        classifier.show_most_informative_features()
    except AttributeError:
        print('Your classifier does not provide a show_most_informative_features() method.')
    results = sentim_analyzer.evaluate(test_set)

    if output:
        extr = [f.__name__ for f in sentim_analyzer.feat_extractors]
        output_markdown(output, Dataset='Movie_reviews', Classifier=type(classifier).__name__,
                        Tokenizer='WordPunctTokenizer', Feats=extr, Results=results,
                        Instances=n_instances) 

# 需要导入模块: from nltk.classify import NaiveBayesClassifier [as 别名]
# 或者: from nltk.classify.NaiveBayesClassifier import train [as 别名]
def demo_subjectivity(trainer, save_analyzer=False, n_instances=None, output=None):
    """
    Train and test a classifier on instances of the Subjective Dataset by Pang and
    Lee. The dataset is made of 5000 subjective and 5000 objective sentences.
    All tokens (words and punctuation marks) are separated by a whitespace, so
    we use the basic WhitespaceTokenizer to parse the data.

    :param trainer: `train` method of a classifier.
    :param save_analyzer: if `True`, store the SentimentAnalyzer in a pickle file.
    :param n_instances: the number of total sentences that have to be used for
        training and testing. Sentences will be equally split between positive
        and negative.
    :param output: the output file where results have to be reported.
    """
    from sentiment_analyzer import SentimentAnalyzer
    from nltk.corpus import subjectivity

    if n_instances is not None:
        n_instances = int(n_instances/2)

    subj_docs = [(sent, 'subj') for sent in subjectivity.sents(categories='subj')[:n_instances]]
    obj_docs = [(sent, 'obj') for sent in subjectivity.sents(categories='obj')[:n_instances]]

    # We separately split subjective and objective instances to keep a balanced
    # uniform class distribution in both train and test sets.
    train_subj_docs, test_subj_docs = split_train_test(subj_docs)
    train_obj_docs, test_obj_docs = split_train_test(obj_docs)

    training_docs = train_subj_docs+train_obj_docs
    testing_docs = test_subj_docs+test_obj_docs

    sentim_analyzer = SentimentAnalyzer()
    all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_docs])

    # Add simple unigram word features handling negation
    unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg, min_freq=4)
    sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)

    # Apply features to obtain a feature-value representation of our datasets
    training_set = sentim_analyzer.apply_features(training_docs)
    test_set = sentim_analyzer.apply_features(testing_docs)

    classifier = sentim_analyzer.train(trainer, training_set)
    try:
        classifier.show_most_informative_features()
    except AttributeError:
        print('Your classifier does not provide a show_most_informative_features() method.')
    results = sentim_analyzer.evaluate(test_set)

    if save_analyzer == True:
        save_file(sentim_analyzer, 'sa_subjectivity.pickle')

    if output:
        extr = [f.__name__ for f in sentim_analyzer.feat_extractors]
        output_markdown(output, Dataset='subjectivity', Classifier=type(classifier).__name__,
                        Tokenizer='WhitespaceTokenizer', Feats=extr,
                        Instances=n_instances, Results=results)

    return sentim_analyzer