Python Segmenter.recode方法代码示例

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
 def test_recode_overlapping_segmentation(self):
     """Does recode raise exception for overlapping segmentation?"""
     with self.assertRaises(
         ValueError,
         msg="recode doesn't raise exception for overlapping segmentation!"
     ):
         Segmenter.recode(
             self.overlapping_seg,
         )

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
 def test_recode_no_change(self):
     """Does recode return a Segmentation when no change is made?"""
     segmentation = Segmenter.recode(
         self.entire_text_seg,
     )
     self.assertTrue(
         isinstance(segmentation, Segmentation),
         msg="recode doesn't return a Segmentation when no change is made!"
     )

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
 def test_recode_segmentation_as_input(self):
     """Does recode return a Segmentation when input is one?"""
     segmentation = Segmenter.recode(
         self.letter_seg,
         case='upper',
     )
     self.assertTrue(
         isinstance(segmentation, Segmentation),
         msg="recode doesn't return a Segmentation when input is one!"
     )

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
 def test_recode_single_input(self):
     """Does recode return a single Input object when needed?"""
     segmentation = Segmenter.recode(
         self.entire_text_seg,
         case='upper',
     )
     self.assertTrue(
         isinstance(segmentation, Input),
         msg="recode doesn't return a single Input object when needed!"
     )

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
 def test_get_real_str_index_recoded(self):
     """Does get_real_str_index() work with actual str index?"""
     recoded_seg, _ = Segmenter.recode(
         self.char_seg,
         substitutions=[(re.compile(r'[bd]'), 'f')],
     )
     self.assertEqual(
         recoded_seg[-1].get_real_str_index(),
         self.char_seg[0].str_index,
         msg="get_real_str_index() doesn't work with redirected str index!"
     )

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
 def test_recode_remove_accents(self):
     """Does recode remove accents?"""
     segmentation = Segmenter.recode(
         self.second_word_seg,
         remove_accents=True,
     )
     self.assertEqual(
         [s.get_content() for s in segmentation],
         ['AB', 'cde'],
         msg="recode doesn't remove accents!"
     )

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
 def test_recode_lower_case(self):
     """Does recode change case to lower?"""
     segmentation = Segmenter.recode(
         self.second_word_seg,
         case='lower',
     )
     self.assertEqual(
         [s.get_content() for s in segmentation],
         ['ab', 'cd\xe9'],
         msg="recode doesn't change case to lower!"
     )

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
 def test_recode_upper_case(self):
     """Does recode change case to upper?"""
     segmentation = Segmenter.recode(
         self.word_seg,
         case='upper',
     )
     self.assertEqual(
         [s.get_content() for s in segmentation],
         ['AB', 'CDE'],
         msg="recode doesn't change case to upper!"
     )

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
    def test_recode_progress(self):
        """Does recode track progress?"""

        def progress_callback():
            """Mock progress callback"""
            self.count += 1

        Segmenter.recode(
            self.word_seg,
            case='upper',
            substitutions=[
                (re.compile(r'..'), 'x'),
                (re.compile(r'xe'), 'ex'),
            ],
            progress_callback=progress_callback,
        )
        self.assertEqual(
            self.count,
            len(self.word_seg),
            msg="recode doesn't track progress!"
        )

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
 def test_recode_copy_annotations_false(self):
     """Does recode skip copying annotations?"""
     segmentation = Segmenter.recode(
         self.word_seg,
         substitutions=[
             (re.compile(r'...'), 'test'),
         ],
         copy_annotations=False,
     )
     self.assertFalse(
         'a' in segmentation[0].annotations,
         msg="recode doesn't skip copying annotations!"
     )

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
 def test_recode_variable_interpolation(self):
     """Does recode interpolate variables for substitutions?"""
     segmentation = Segmenter.recode(
         self.word_seg,
         substitutions=[
             (re.compile(r'(.)(.)'), '&2&1'),
         ],
     )
     self.assertEqual(
         [s.get_content() for s in segmentation],
         ['ba', 'dce'],
         msg="recode doesn't interpolate variables for substitutions!"
     )

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
 def test_recode_copy_annotations(self):
     """Does recode copy annotations?"""
     segmentation = Segmenter.recode(
         self.word_seg,
         substitutions=[
             (re.compile(r'...'), 'test'),
         ],
         copy_annotations=True,
     )
     self.assertEqual(
         segmentation[0].annotations['a'],
         '1',
         msg="recode doesn't copy annotations!"
     )

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
 def test_recode_substitutions(self):
     """Does recode apply substitutions?"""
     segmentation = Segmenter.recode(
         self.word_seg,
         substitutions=[
             (re.compile(r'..'), 'x'),
             (re.compile(r'xe'), 'ex'),
         ],
     )
     self.assertEqual(
         [s.get_content() for s in segmentation],
         ['x', 'ex'],
         msg="recode doesn't apply substitutions!"
     )

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
 def test_recode_substitutions_after(self):
     """Does recode apply substitutions after preprocessing?"""
     segmentation = Segmenter.recode(
         self.word_seg,
         case='upper',
         substitutions=[
             (re.compile(r'..'), 'x'),
             (re.compile(r'xe'), 'ex'),
         ],
     )
     self.assertEqual(
         [s.get_content() for s in segmentation],
         ['x', 'xE'],
         msg="recode doesn't apply substitutions after preprocessing!"
     )

# 需要导入模块: from LTTL import Segmenter [as 别名]
# 或者: from LTTL.Segmenter import recode [as 别名]
    def sendData(self):
        
        # Si le lien vers treetagger n'est pas trouve
        if self.NoLink:
            self.infoBox.setText(
                u"Sorry, TreeTagger's link not found.",
                "error"
            )
            self.send('Text data', None)
        # Important: if input data is None, propagate this value to output...
        elif not self.inputData:
            self.infoBox.setText(
                u"Widget needs input",
                "warning"
            )
            self.send('Text data', None)
        # affiche que quelque chose se passe...
        else:
            self.infoBox.setText(
                u'TreeTagger is running...',
                "warning"
            )

            # Initialisation de variables
            total_tagged_text = list()
            new_segmentations = list()
            i = 0
            
            # Initialize progress bar.
            self.progressBar = OWGUI.ProgressBar(
                self,
                iterations = 5
            )
            
            # Copie de la segmentation avec ajout d'une annotation...
            copy_of_input_seg = Segmentation()
            copy_of_input_seg.label = self.inputData.label
            for seg_idx, segment in enumerate(self.inputData):
                attr = " ".join(
                    ["%s='%s'" % item for item in segment.annotations.items()]
                )
                segment.annotations["tt_xb"] = attr
                copy_of_input_seg.append(segment)
            
            # avancer la progressBar d'un cran
            self.progressBar.advance()

            concatenated_text = copy_of_input_seg.to_string(
                formatting="<xb_tt %(tt_xb)s>%(__content__)s</xb_tt>",
                display_all=True,
            )
            
            
            
            # avancer la progressBar d'un cran
            self.progressBar.advance()
            
            tagged_text = self.tag(concatenated_text)
            tagged_input = Input(tagged_text)
            tagged_segmentation = Segmenter.import_xml(tagged_input, "xb_tt")
            
            # avancer la progressBar d'un cran
            self.progressBar.advance()
            
            # Si checkBox xml active
            if self.activer_xml == True:
                xml_segmentation = Segmenter.recode(
                        tagged_segmentation,
                        substitutions = [
                            (re.compile(r"<unknown>"), '[unknown]'),
                            (re.compile(
                                r"(.+)\t(.+)\t(.+)"),
                                '<w lemma="&3" type="&2">&1</w>'
                            ),
                            (re.compile(r'"""'), '"&quot;"'),
                        ],
                    )
                final_segmentation = xml_segmentation
            # Si checkBox xml desactive
            else:
                xml_segmentation = Segmenter.recode(
                        tagged_segmentation,
                        substitutions=[
                            (re.compile(r"<unknown>"), '[unknown]'),
                            (re.compile(
                                r"(.+)\t(.+)\t(.+)"),
                                '<w lemma="&3" type="&2">&1</w>'
                            ),
                            (re.compile(r'"""'), '"&quot;"'),

                        ],
                    )
                final_segmentation = Segmenter.import_xml(
                    xml_segmentation,
                    "w"
                )

            self.infoBox.dataSent('')

            # Enregistrer le lien de treetagger...
#.........这里部分代码省略.........