本文整理汇总了Python中dataset.Dataset.get_tokens方法的典型用法代码示例。如果您正苦于以下问题：Python Dataset.get_tokens方法的具体用法？Python Dataset.get_tokens怎么用？Python Dataset.get_tokens使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类dataset.Dataset的用法示例。
在下文中一共展示了Dataset.get_tokens方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: train
# 需要导入模块: from dataset import Dataset [as 别名]
# 或者: from dataset.Dataset import get_tokens [as 别名]
def train(self, session, dataset: Dataset):
word_vectors = []
for tokens in dataset.get_tokens():
word_vectors.append(self.encode_word_vector(tokens))
slot_vectors = []
for iob in dataset.get_iob():
slot_vectors.append(self.encode_slot_vector(iob))
cost_output = float('inf')
for _ in range(self.__step_per_checkpoints):
indexes = np.random.choice(len(word_vectors), self.__batch_size, replace=False)
x = [word_vectors[index] for index in indexes]
y = [slot_vectors[index] for index in indexes]
self.__step += 1
_, cost_output = session.run([self.__optimizer, self.__cost],
feed_dict={self.__x: x,
self.__y: y,
self.__dropout: 0.5})
checkpoint_path = os.path.join('./model', "slot_filling_model.ckpt")
self.__saver.save(session, checkpoint_path, global_step=self.__step)
return cost_output示例2: test_append
# 需要导入模块: from dataset import Dataset [as 别名]
# 或者: from dataset.Dataset import get_tokens [as 别名]
def test_append(self):
domain = 'test'
raw = 'i wanna go to (Essex Street)[to_stop]'
iob = ['o', 'o', 'o', 'o', 'b-test.to_stop', 'i-test.to_stop']
tokens = ['i', 'wanna', 'go', 'to', 'essex', 'street']
dataset = Dataset()
dataset.append(domain, raw)
self.assertEqual(dataset.get_domain(0), domain)
self.assertEqual(dataset.get_raw(0), raw)
self.assertEqual(dataset.get_iob(0), iob)
self.assertListEqual(dataset.get_tokens(0), tokens)示例3: test
# 需要导入模块: from dataset import Dataset [as 别名]
# 或者: from dataset.Dataset import get_tokens [as 别名]
def test(self, session, dataset: Dataset):
word_vectors = []
for tokens in dataset.get_tokens():
word_vectors.append(self.encode_word_vector(tokens))
slot_vectors = []
for iob in dataset.get_iob():
slot_vectors.append(self.encode_slot_vector(iob))
prediction_output, score_output, length_output = session.run(
[self.__prediction, self.__score, self.__length],
feed_dict={self.__x: word_vectors,
self.__y: slot_vectors,
self.__dropout: 1.0})
predict = []
for i in range(dataset.length()):
predict.append(self.decode_slot_vector(prediction_output[i][:length_output[i]]))
return score_output, predict