Python template.TemplateMaker类代码示例

 def test_annotations(self):
     tm = TemplateMaker(self.PAGE)
     tm.annotate("field1", best_match("text to annotate"), best_match=False)
     annotations = [x[0] for x in tm.annotations()]
     self.assertEqual(
         annotations, [{u"annotations": {u"content": u"field1"}}, {u"annotations": {u"content": u"field1"}}]
     )

 def test_annotations(self):
     tm = TemplateMaker(self.PAGE)
     tm.annotate('field1', best_match('text to annotate'), best_match=False)
     annotations = [x[0] for x in tm.annotations()]
     self.assertEqual(annotations,
         [{u'annotations': {u'content': u'field1'}},
          {u'annotations': {u'content': u'field1'}}])

 def test_selected_data(self):
     tm = TemplateMaker(self.PAGE)
     indexes = tm.select(best_match('text to annotate'))
     data = [tm.selected_data(i) for i in indexes]
     self.assertEqual(data, \
         [u'<p>Some text to annotate here</p>', \
         u'<p>Another text to annotate there</p>'])

 def test_annotate_ignore_unpaired(self):
     tm = TemplateMaker(self.PAGE)
     tm.annotate('field1', best_match("and that's"), best_match=False)
     tpl = tm.get_template()
     ex = InstanceBasedLearningExtractor([(tpl, None)])
     self.assertEqual(ex.extract(self.PAGE)[0],
         [{u'field1': [u"More text with unpaired tag <img />and that's it"]}])

 def test_annotate_multiple(self):
     tm = TemplateMaker(self.PAGE)
     tm.annotate('field1', best_match('text to annotate'), best_match=False)
     tpl = tm.get_template()
     ex = InstanceBasedLearningExtractor([(tpl, None)])
     self.assertEqual(ex.extract(self.PAGE)[0],
         [{u'field1': [u'Some text to annotate here', u'Another text to annotate there']}])

 def test_annotate_multiple(self):
     tm = TemplateMaker(self.PAGE)
     tm.annotate("field1", best_match("text to annotate"), best_match=False)
     tpl = tm.get_template()
     ex = InstanceBasedLearningExtractor([tpl])
     self.assertEqual(
         ex.extract(self.PAGE)[0], [{u"field1": [u"Some text to annotate here", u"Another text to annotate there"]}]
     )

 def do_al(self, template_id):
     """al <template> - list annotations"""
     if assert_or_print(template_id, "missing template id"):
         return
     t = self._load_template(template_id)
     tm = TemplateMaker(t)
     for n, (a, i) in enumerate(tm.annotations()):
         print "[%s-%d] (%s) %r" % (template_id, n, a['annotations']['content'], 
             remove_annotation(tm.selected_data(i)))

 def do_t(self, line):
     """ts <template> <text> - test selection text"""
     template_id, criteria = line.split(' ', 1)
     t = self._load_template(template_id)
     criteria = parse_criteria(criteria)
     tm = TemplateMaker(t)
     selection = apply_criteria(criteria, tm)
     for n, i in enumerate(selection):
         print "[%d] %r" % (n, remove_annotation(tm.selected_data(i)))

 def train_from_htmlpage(self, htmlpage, data):
     assert data, "Cannot train with empty data"
     tm = TemplateMaker(htmlpage)
     for field, values in data.items():
         if (isinstance(values, (bytes, str)) or
                 not hasattr(values, '__iter__')):
             values = [values]
         for value in values:
             value = str_to_unicode(value, htmlpage.encoding)
             tm.annotate(field, best_match(value))
     self.add_template(tm.get_template())

 def train_from_htmlpage(self, htmlpage, data):
     assert data, "Cannot train with empty data"
     tm = TemplateMaker(htmlpage)
     for field, values in data.items():
         if not hasattr(values, '__iter__'):
             values = [values]
         for value in values:
             if isinstance(value, str):
                 value = value.decode(htmlpage.encoding or 'utf-8')
             tm.annotate(field, best_match(value))
     self.add_template(tm.get_template())

 def train(self, url=None, data=None, html=None, encoding='utf-8'):
     assert data, "Cannot train with empty data"
     page = self._get_page(url, encoding, html)
     tm = TemplateMaker(page)
     for field, values in data.items():
         if not hasattr(values, '__iter__'):
             values = [values]
         for value in values:
             if isinstance(value, str):
                 value = value.decode(encoding)
             tm.annotate(field, best_match(value))
     self.templates.append(tm.get_template())

    def do_annotate(self, line):
        """annotate <template_id> <data> [-n number] [-f field]- add or test annotation (aliases: a, t)

        Add a new annotation (if -f is passed) or test what would be annotated
        otherwise
        """
        if line.find(' ') < 0:
            print("You must provide a valid template identifier (check output of ls_templates)")
            print(IblTool.do_annotate.__doc__)
            return
        template_id, criteria = line.split(' ', 1)
        t = self._load_template(template_id)
        if not t:
            return
        criteria = self._parse_criteria(criteria)
        tm = TemplateMaker(t)
        selection = apply_criteria(criteria, tm)
        if criteria.field:
            for index in selection:
                index = selection[0]
                tm.annotate_fragment(index, criteria.field)
                self._save_template(template_id, tm.get_template())
                print("[new] (%s) %r" % (criteria.field,
                    remove_annotation(tm.selected_data(index))))
        else:
            for n, i in enumerate(selection):
                print("[%d] %r" % (n, remove_annotation(tm.selected_data(i))))

 def do_a(self, line):
     """a <template> <data> [-n number] [-f field]- add or test annotation
     
     Add a new annotation (if -f is passed) or test what would be annotated
     otherwise
     """
     template_id, criteria = line.split(' ', 1)
     t = self._load_template(template_id)
     criteria = parse_criteria(criteria)
     tm = TemplateMaker(t)
     selection = apply_criteria(criteria, tm)
     if criteria.field:
         for index in selection:
             index = selection[0]
             tm.annotate_fragment(index, criteria.field)
             self._save_template(template_id, tm.get_template())
             print "[new] (%s) %r" % (criteria.field,
                 remove_annotation(tm.selected_data(index)))
     else:
         for n, i in enumerate(selection):
             print "[%d] %r" % (n, remove_annotation(tm.selected_data(i)))

 def _load_annotations(self, template_id):
     t = self._load_template(template_id)
     tm = TemplateMaker(t)
     return [x[0] for x in tm.annotations()]

 def test_selected_data(self):
     tm = TemplateMaker(self.PAGE)
     indexes = tm.select(best_match("text to annotate"))
     data = [tm.selected_data(i) for i in indexes]
     self.assertEqual(data, [u"<p>Some text to annotate here</p>", u"<p>Another text to annotate there</p>"])