Python metrics.CategoricalAccuracy方法代码示例

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def __init__(self, vocab: Vocabulary,
                 text_field_embedder: TextFieldEmbedder,
                 verbose_metrics: bool = False,
                 dropout: float = 0.2,
                 initializer: InitializerApplicator = InitializerApplicator(),
                 regularizer: Optional[RegularizerApplicator] = None,
                 ) -> None:
        super(TextClassifier, self).__init__(vocab, regularizer)

        self.text_field_embedder = text_field_embedder
        self.dropout = torch.nn.Dropout(dropout)
        self.num_classes = self.vocab.get_vocab_size("labels")
        self.classifier_feedforward = torch.nn.Linear(self.text_field_embedder.get_output_dim()  , self.num_classes)

        self.label_accuracy = CategoricalAccuracy()
        self.label_f1_metrics = {}

        self.verbose_metrics = verbose_metrics

        for i in range(self.num_classes):
            self.label_f1_metrics[vocab.get_token_from_index(index=i, namespace="labels")] = F1Measure(positive_label=i)
        self.loss = torch.nn.CrossEntropyLoss()

        initializer(self) 

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def test_top_k_categorical_accuracy_works_for_sequences(self, device: str):
        accuracy = CategoricalAccuracy(top_k=2)
        predictions = torch.tensor(
            [
                [[0.35, 0.25, 0.1, 0.1, 0.2], [0.1, 0.6, 0.1, 0.2, 0.0], [0.1, 0.6, 0.1, 0.2, 0.0]],
                [[0.35, 0.25, 0.1, 0.1, 0.2], [0.1, 0.6, 0.1, 0.2, 0.0], [0.1, 0.6, 0.1, 0.2, 0.0]],
            ],
            device=device,
        )
        targets = torch.tensor([[0, 3, 4], [0, 1, 4]], device=device)
        accuracy(predictions, targets)
        actual_accuracy = accuracy.get_metric(reset=True)
        assert_allclose(actual_accuracy, 0.6666666)

        # Test the same thing but with a mask:
        mask = torch.tensor([[False, True, True], [True, False, True]], device=device)
        accuracy(predictions, targets, mask)
        actual_accuracy = accuracy.get_metric(reset=True)
        assert_allclose(actual_accuracy, 0.50) 

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def __init__(self, vocab            ,
                 text_field_embedder                   ,
                 encoder                ,
                 initializer                        = InitializerApplicator(),
                 regularizer                                  = None)        :
        super(SimpleTagger, self).__init__(vocab, regularizer)

        self.text_field_embedder = text_field_embedder
        self.num_classes = self.vocab.get_vocab_size(u"labels")
        self.encoder = encoder
        self.tag_projection_layer = TimeDistributed(Linear(self.encoder.get_output_dim(),
                                                           self.num_classes))

        check_dimensions_match(text_field_embedder.get_output_dim(), encoder.get_input_dim(),
                               u"text field embedding dim", u"encoder input dim")
        self.metrics = {
                u"accuracy": CategoricalAccuracy(),
                u"accuracy3": CategoricalAccuracy(top_k=3)
        }

        initializer(self)

    #overrides 

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def test_top_k_categorical_accuracy_works_for_sequences(self):
        accuracy = CategoricalAccuracy(top_k=2)
        predictions = torch.Tensor([[[0.35, 0.25, 0.1, 0.1, 0.2],
                                     [0.1, 0.6, 0.1, 0.2, 0.0],
                                     [0.1, 0.6, 0.1, 0.2, 0.0]],
                                    [[0.35, 0.25, 0.1, 0.1, 0.2],
                                     [0.1, 0.6, 0.1, 0.2, 0.0],
                                     [0.1, 0.6, 0.1, 0.2, 0.0]]])
        targets = torch.Tensor([[0, 3, 4],
                                [0, 1, 4]])
        accuracy(predictions, targets)
        actual_accuracy = accuracy.get_metric(reset=True)
        numpy.testing.assert_almost_equal(actual_accuracy, 0.6666666)

        # Test the same thing but with a mask:
        mask = torch.Tensor([[0, 1, 1],
                             [1, 0, 1]])
        accuracy(predictions, targets, mask)
        actual_accuracy = accuracy.get_metric(reset=True)
        numpy.testing.assert_almost_equal(actual_accuracy, 0.50) 

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def __init__(self, vocab: Vocabulary,
                       regularizer: RegularizerApplicator = None):
        super().__init__(vocab, regularizer)

        self.nsp_loss_function = torch.nn.CrossEntropyLoss(ignore_index=-1)
        self.lm_loss_function = torch.nn.CrossEntropyLoss(ignore_index=0)

        self._metrics = {
            "total_loss_ema": ExponentialMovingAverage(alpha=0.5),
            "nsp_loss_ema": ExponentialMovingAverage(alpha=0.5),
            "lm_loss_ema": ExponentialMovingAverage(alpha=0.5),
            "total_loss": Average(),
            "nsp_loss": Average(),
            "lm_loss": Average(),
            "lm_loss_wgt": WeightedAverage(),
            "mrr": MeanReciprocalRank(),
        }
        self._accuracy = CategoricalAccuracy() 

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def __init__(self, vocab: Vocabulary,
                 text_field_embedder: TextFieldEmbedder,
                 encoder: Seq2SeqEncoder,
                 # binary_feature_dim: int,
                 embedding_dropout: float = 0.0,
                 initializer: InitializerApplicator = InitializerApplicator(),
                 regularizer: Optional[RegularizerApplicator] = None) -> None:
        super(LstmSwag, self).__init__(vocab, regularizer)

        self.text_field_embedder = text_field_embedder

        # For the span based evaluation, we don't want to consider labels
        # for verb, because the verb index is provided to the model.
        self.encoder = encoder
        self.embedding_dropout = Dropout(p=embedding_dropout)
        self.output_prediction = Linear(self.encoder.get_output_dim(), 1, bias=False)

        check_dimensions_match(text_field_embedder.get_output_dim(),
                               encoder.get_input_dim(),
                               "text embedding dim", "eq encoder input dim")
        self._accuracy = CategoricalAccuracy()
        self._loss = torch.nn.CrossEntropyLoss()

        initializer(self) 

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def __init__(self, vocab: Vocabulary,
                 input_dim: int,
                 num_classes: int,
                 label_namespace: str = "labels",
                 feedforward: Optional[FeedForward] = None,
                 dropout: Optional[float] = None,
                 verbose_metrics: bool = False,
                 initializer: InitializerApplicator = InitializerApplicator(),
                 regularizer: Optional[RegularizerApplicator] = None) -> None:
        super().__init__(vocab, regularizer)
        self.label_namespace = label_namespace
        self.input_dim = input_dim
        self.num_classes = num_classes 
        self._verbose_metrics = verbose_metrics
        if dropout:
            self.dropout = torch.nn.Dropout(dropout)
        else:
            self.dropout = None
        self._feedforward = feedforward

        if self._feedforward is not None: 
            self.projection_layer = Linear(feedforward.get_output_dim(), self.num_classes)
        else:
            self.projection_layer = Linear(self.input_dim, self.num_classes)

        self.metrics = {
                "accuracy": CategoricalAccuracy(),
                "accuracy3": CategoricalAccuracy(top_k=3),
                "accuracy5": CategoricalAccuracy(top_k=5)
        }
        self._loss = torch.nn.CrossEntropyLoss()

        initializer(self) 

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def __init__(self, vocab: Vocabulary,
                 text_field_embedder: TextFieldEmbedder,
                 text_encoder: Seq2SeqEncoder,
                 classifier_feedforward: FeedForward,
                 verbose_metrics: False,
                 initializer: InitializerApplicator = InitializerApplicator(),
                 regularizer: Optional[RegularizerApplicator] = None,
                 ) -> None:
        super(TextClassifier, self).__init__(vocab, regularizer)

        self.text_field_embedder = text_field_embedder
        self.num_classes = self.vocab.get_vocab_size("labels")
        self.text_encoder = text_encoder
        self.classifier_feedforward = classifier_feedforward
        self.prediction_layer = torch.nn.Linear(self.classifier_feedforward.get_output_dim()  , self.num_classes)

        self.label_accuracy = CategoricalAccuracy()
        self.label_f1_metrics = {}

        self.verbose_metrics = verbose_metrics

        for i in range(self.num_classes):
            self.label_f1_metrics[vocab.get_token_from_index(index=i, namespace="labels")] = F1Measure(positive_label=i)
        self.loss = torch.nn.CrossEntropyLoss()

        self.pool = lambda text, mask: util.get_final_encoder_states(text, mask, bidirectional=True)

        initializer(self) 

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def __init__(self, vocab: Vocabulary,
                 text_field_embedder: TextFieldEmbedder,
                 encoder: Seq2SeqEncoder,
                 include_start_end_transitions: bool = True,
                 dropout: Optional[float] = None,
                 initializer: InitializerApplicator = InitializerApplicator(),
                 regularizer: Optional[RegularizerApplicator] = None) -> None:
        super().__init__(vocab, regularizer)

        self.label_namespace = 'labels'
        self.num_tags = self.vocab.get_vocab_size(self.label_namespace)

        # encode text
        self.text_field_embedder = text_field_embedder
        self.encoder = encoder
        self.dropout = torch.nn.Dropout(dropout) if dropout else None

        # crf
        output_dim = self.encoder.get_output_dim()
        self.tag_projection_layer = TimeDistributed(Linear(output_dim, self.num_tags))
        self.crf = ConditionalRandomField(self.num_tags, constraints=None, include_start_end_transitions=include_start_end_transitions)

        self.metrics = {
            "accuracy": CategoricalAccuracy(),
            "accuracy3": CategoricalAccuracy(top_k=3)
        }
        for index, label in self.vocab.get_index_to_token_vocabulary(self.label_namespace).items():
            self.metrics['F1_' + label] = F1Measure(positive_label=index)

        initializer(self) 

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def __init__(self, vocab: Vocabulary,
                 text_field_embedder: TextFieldEmbedder,
                 predictor_dropout=0.0,
                 labels_namespace: str = "labels",
                 detect_namespace: str = "d_tags",
                 verbose_metrics: bool = False,
                 label_smoothing: float = 0.0,
                 confidence: float = 0.0,
                 initializer: InitializerApplicator = InitializerApplicator(),
                 regularizer: Optional[RegularizerApplicator] = None) -> None:
        super(Seq2Labels, self).__init__(vocab, regularizer)

        self.label_namespaces = [labels_namespace,
                                 detect_namespace]
        self.text_field_embedder = text_field_embedder
        self.num_labels_classes = self.vocab.get_vocab_size(labels_namespace)
        self.num_detect_classes = self.vocab.get_vocab_size(detect_namespace)
        self.label_smoothing = label_smoothing
        self.confidence = confidence
        self.incorr_index = self.vocab.get_token_index("INCORRECT",
                                                       namespace=detect_namespace)

        self._verbose_metrics = verbose_metrics
        self.predictor_dropout = TimeDistributed(torch.nn.Dropout(predictor_dropout))

        self.tag_labels_projection_layer = TimeDistributed(
            Linear(text_field_embedder._token_embedders['bert'].get_output_dim(), self.num_labels_classes))

        self.tag_detect_projection_layer = TimeDistributed(
            Linear(text_field_embedder._token_embedders['bert'].get_output_dim(), self.num_detect_classes))

        self.metrics = {"accuracy": CategoricalAccuracy()}

        initializer(self) 

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def test_categorical_accuracy(self, device: str):
        accuracy = CategoricalAccuracy()
        predictions = torch.tensor(
            [[0.35, 0.25, 0.1, 0.1, 0.2], [0.1, 0.6, 0.1, 0.2, 0.0]], device=device
        )
        targets = torch.tensor([0, 3], device=device)
        accuracy(predictions, targets)
        actual_accuracy = accuracy.get_metric()
        assert actual_accuracy == 0.50 

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def test_top_k_categorical_accuracy(self, device: str):
        accuracy = CategoricalAccuracy(top_k=2)
        predictions = torch.tensor(
            [[0.35, 0.25, 0.1, 0.1, 0.2], [0.1, 0.6, 0.1, 0.2, 0.0]], device=device
        )
        targets = torch.tensor([0, 3], device=device)
        accuracy(predictions, targets)
        actual_accuracy = accuracy.get_metric()
        assert actual_accuracy == 1.0 

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def test_top_k_categorical_accuracy_accumulates_and_resets_correctly(self, device: str):
        accuracy = CategoricalAccuracy(top_k=2)
        predictions = torch.tensor(
            [[0.35, 0.25, 0.1, 0.1, 0.2], [0.1, 0.6, 0.1, 0.2, 0.0]], device=device
        )
        targets = torch.tensor([0, 3], device=device)
        accuracy(predictions, targets)
        accuracy(predictions, targets)
        accuracy(predictions, torch.tensor([4, 4], device=device))
        accuracy(predictions, torch.tensor([4, 4], device=device))
        actual_accuracy = accuracy.get_metric(reset=True)
        assert actual_accuracy == 0.50
        assert accuracy.correct_count == 0.0
        assert accuracy.total_count == 0.0 

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def test_top_k_categorical_accuracy_catches_exceptions(self, device: str):
        accuracy = CategoricalAccuracy()
        predictions = torch.rand([5, 7], device=device)
        out_of_range_labels = torch.tensor([10, 3, 4, 0, 1], device=device)
        with pytest.raises(ConfigurationError):
            accuracy(predictions, out_of_range_labels) 

# 需要导入模块: from allennlp.training import metrics [as 别名]
# 或者: from allennlp.training.metrics import CategoricalAccuracy [as 别名]
def test_tie_break_categorical_accuracy(self, device: str):
        accuracy = CategoricalAccuracy(tie_break=True)
        predictions = torch.tensor(
            [[0.35, 0.25, 0.35, 0.35, 0.35], [0.1, 0.6, 0.1, 0.2, 0.2], [0.1, 0.0, 0.1, 0.2, 0.2]],
            device=device,
        )
        # Test without mask:
        targets = torch.tensor([2, 1, 4], device=device)
        accuracy(predictions, targets)
        assert accuracy.get_metric(reset=True) == (0.25 + 1 + 0.5) / 3.0

        # # # Test with mask
        mask = torch.tensor([True, False, True], device=device)
        targets = torch.tensor([2, 1, 4], device=device)
        accuracy(predictions, targets, mask)
        assert accuracy.get_metric(reset=True) == (0.25 + 0.5) / 2.0

        # # Test tie-break with sequence
        predictions = torch.tensor(
            [
                [
                    [0.35, 0.25, 0.35, 0.35, 0.35],
                    [0.1, 0.6, 0.1, 0.2, 0.2],
                    [0.1, 0.0, 0.1, 0.2, 0.2],
                ],
                [
                    [0.35, 0.25, 0.35, 0.35, 0.35],
                    [0.1, 0.6, 0.1, 0.2, 0.2],
                    [0.1, 0.0, 0.1, 0.2, 0.2],
                ],
            ],
            device=device,
        )
        targets = torch.tensor(
            [[0, 1, 3], [0, 3, 4]], device=device  # 0.25 + 1 + 0.5  # 0.25 + 0 + 0.5 = 2.5
        )
        accuracy(predictions, targets)
        actual_accuracy = accuracy.get_metric(reset=True)
        assert_allclose(actual_accuracy, 2.5 / 6.0)