本文整理汇总了Python中markov.Markov.tokenize方法的典型用法代码示例。如果您正苦于以下问题：Python Markov.tokenize方法的具体用法？Python Markov.tokenize怎么用？Python Markov.tokenize使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类markov.Markov的用法示例。
在下文中一共展示了Markov.tokenize方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: TestParsing
# 需要导入模块: from markov import Markov [as 别名]
# 或者: from markov.Markov import tokenize [as 别名]
class TestParsing(unittest.TestCase):
def setUp(self):
self.markov = Markov()
def test_basic_tokenize(self):
result = self.markov.tokenize("grraaaaa brainss monkeybutter")
self.assertEquals(result, ["grraaaaa", "brainss", "monkeybutter"])
def test_basic_scan(self):
self.markov.scan(['a', 'b', 'c', 'd', 'e'])
self.assertTrue('a' in self.markov.chains)
self.assertTrue('b' in self.markov.chains)
self.assertTrue('c' in self.markov.chains)
self.assertTrue('b' in self.markov.chains['a']['next'])
self.assertTrue('c' in self.markov.chains['a']['next']['b']['next'])
self.assertTrue('d' in self.markov.chains['b']['next']['c']['next'])
def test_count(self):
m = Markov(2)
m.scan(['a', 'b', 'b', 'b', 'c'])
self.assertEquals(m.chains['a']['count'], 1)
self.assertEquals(m.chains['b']['count'], 3)
self.assertEquals(m.chains['a']['next']['b']['count'], 1)
self.assertEquals(m.chains['b']['next']['b']['count'], 2)
self.assertEquals(m.chains['b']['next']['c']['count'], 1)
def test_probabilities(self):
m = Markov(2)
m.scan(['bacon', 'fish', 'bacon', 'lung',
'bacon', 'lung', 'bacon', 'lung', '.'])
result = m.compute_probabilities(3)
self.assertEquals(len(result), 3)
# least likely to most
self.assertEquals(result[0]['word'], 'fish')
self.assertEquals(result[0]['chance'], 1.0/8.0 * (1.0 / 0.5))
self.assertEquals(result[1]['word'], 'lung')
self.assertEquals(result[1]['chance'], 3.0/8.0 * (1.0 / 0.5))
self.assertEquals(result[2]['word'], 'bacon')
self.assertEquals(result[2]['chance'], 0.5 * (1.0 / 0.5))