本文整理汇总了Python中whoosh.analysis.Token类的典型用法代码示例。如果您正苦于以下问题:Python Token类的具体用法?Python Token怎么用?Python Token使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Token类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: __call__
def __call__(self, text, **kargs):
token = Token()
words = set()
words_list = []
for (i, start_pos, stop_pos) in jieba.tokenize(text, mode='search'):
i = i.strip()
if not i:
continue
if i in words:
continue
if i in punct:
continue
words.add(i)
words_list.append(i)
for w in words:
if not accepted_chars.match(w):
if len(w) <= 1:
continue
token.original = token.text = w
token.pos = start_pos
token.startchar = start_pos
token.endchar = stop_pos
yield token
示例2: __call__
def __call__(self, text, **kargs):
token = Token()
start_pos = 0
for w in group_words(text):
token.original = token.text = w
token.pos = start_pos
token.startchar = start_pos
token.endchar = start_pos + len(w)
start_pos = token.endchar
yield token
示例3: __call__
def __call__(self, text, **kargs):
words = jieba.tokenize(text, mode="search")
token = Token()
for (w,start_pos,stop_pos) in words:
if not accepted_chars.match(w) and len(w)<=1:
continue
token.original = token.text = w
token.pos = start_pos
token.startchar = start_pos
token.endchar = stop_pos
yield token
示例4: __call__
def __call__(self,text,**kargs):
words = tokenize_1(text)
token = Token()
for (w,start_pos,stop_pos) in words:
if not accepted_chars.match(w):
if len(w) <= 1:
continue
token.original = token.text = w
token.pos = start_pos
token.startchar = start_pos
token.endchar = stop_pos
yield token
示例5: __call__
def __call__(self, value, positions=False, chars=False, keeporiginal=False, removestops=True, start_pos=0, start_char=0, mode='', **kwargs):
assert isinstance(value, text_type), "%r is not unicode" % value
t = Token(positions, chars, removestops=removestops, mode=mode, **kwargs)
seglist=jieba.cut_for_search(value) #使用结巴分词库进行分词
for w in seglist:
t.original = t.text = w
t.boost = 1.0
if positions:
t.pos=start_pos+value.find(w)
if chars:
t.startchar=start_char+value.find(w)
t.endchar=start_char+value.find(w)+len(w)
yield t #通过生成器返回每个分词的结果token
示例6: __call__
def __call__(self, value, positions=False, chars=False, keeporiginal=False,removestops=True,start_pos=0, start_char=0, mode='',**kwargs):
assert isinstance(value, text_type), "%r is not unicode" % value
t = Token(positions, chars, removestops=removestops, mode=mode,**kwargs)
seglist = value.split(' ')
for w in seglist:
t.original = t.text = w
t.boost = 1.0
if positions:
t.pos=start_pos+value.find(w)
if chars:
t.startchar=start_char+value.find(w)
t.endchar=start_char+value.find(w)+len(w)
yield t
示例7: __call__
def __call__(self,text,**kargs):
words = _cuttor.tokenize(text, search=True)
token = Token()
for (w,start_pos,stop_pos) in words:
if not accepted_chars.match(w):
if len(w)>1:
pass
else:
continue
token.original = token.text = w
token.pos = start_pos
token.startchar = start_pos
token.endchar = stop_pos
yield token
示例8: __call__
def __call__(self, value, positions=False, chars=False,
keeporiginal=False, removestops=True,
start_pos=0, start_char=0, mode='', **kwargs):
t = Token(positions, chars, removestops=removestops, mode=mode,
**kwargs)
seglist = jieba.cut(value, cut_all=False)
for word in seglist:
t.original = t.text = word
t.boost = 1.0
if positions:
t.pos = start_pos + value.find(word)
if chars:
t.startchar = start_char + value.find(word)
t.endchar = t.startchar + len(word)
yield t
示例9: __call__
def __call__(self, value, mode='', positions=False, **kwargs):
assert isinstance(value, unicode), "%r is not unicode" % value
token = Token(**kwargs)
tagger = MeCab.Tagger('mecabrc')
result = tagger.parse(value.encode("utf8")).decode('utf8')
cur = 0
for match in re.compile("(\S+)\s+(\S+)\n").finditer(result):
category = match.group(2).split(",")
if 0 < len(category) and \
(category[0] == u'名詞' or category[0] == u'動詞' \
or category[0] == u'形容詞' or category[0] == u'副詞'):
token.text = match.group(1)
token.pos = cur
yield token
cur += len(match.group(1))
示例10: _merge_matched_tokens
def _merge_matched_tokens(self, tokens):
token_ready = False
for t in tokens:
if not t.matched:
yield t
continue
if not token_ready:
token = Token(**t.__dict__)
token_ready = True
elif t.startchar <= token.endchar:
if t.endchar > token.endchar:
token.text += t.text[token.endchar-t.endchar:]
token.endchar = t.endchar
else:
yield token
token_ready = False
if token_ready:
yield token
示例11: __call__
def __call__(self, value, start_pos=0, positions=False, **kwargs):
"""
Tokenizer behaviour:
Input: u"text/x.moin.wiki;charset=utf-8"
Output: u"text/x.moin.wiki;charset=utf-8", u"text", u"x.moin.wiki", u"charset=utf-8"
Input: u"application/pdf"
Output: u"application/pdf", u"application", u"pdf"
:param value: String for tokenization
:param start_pos: The position number of the first token. For example,
if you set start_pos=2, the tokens will be numbered 2,3,4,...
instead of 0,1,2,...
:param positions: Whether to record token positions in the token.
"""
assert isinstance(value, unicode), "{0!r} is not unicode".format(value)
if u'/' not in value: # Add '/' if user forgot do this
value += u'/'
pos = start_pos
tk = Token()
tp = Type(value)
# we need to yield the complete contenttype in one piece,
# so we can find it with Term(CONTENTTYPE, contenttype):
if tp.type is not None and tp.subtype is not None:
# note: we do not use "value" directly, so Type.__unicode__ can normalize it:
tk.text = unicode(tp)
if positions:
tk.pos = pos
pos += 1
yield tk
# now yield the pieces:
tk.text = tp.type
if positions:
tk.pos = pos
pos += 1
yield tk
if tp.subtype is not None:
tk.text = tp.subtype
if positions:
tk.pos = pos
pos += 1
yield tk
for key, value in tp.parameters.items():
tk.text = u"{0}={1}".format(key, value)
if positions:
tk.pos = pos
pos += 1
yield tk
示例12: __call__
def __call__(self, value, positions=False, chars=False,
keeporiginal=False, removestops=True,
start_pos=0, start_char=0, mode='', **kwargs):
assert isinstance(value, text_type), "%r is not unicode" % value
t = Token(positions, chars, removestops=removestops, mode=mode,
**kwargs)
nlpir.Init(nlpir.PACKAGE_DIR, nlpir.UTF8_CODE)
pynlpir.open()
pynlpir.open(encoding='utf-8')
seglist = pynlpir.segment(value,)
for w in seglist:
t.original = t.text = w
t.boost = 1.0
if positions:
t.pos=start_pos+value.find(w)
if chars:
t.startchar=start_char+value.find(w)
t.endchar=start_char+value.find(w)+len(w)
yield t #通过生成器返回每个分词的结果token
示例13: __call__
def __call__(self, value, positions=False, chars=False,
keeporiginal=False, removestops=True,
start_pos=0, start_char=0,
tokenize=True, mode='', **kwargs):
assert isinstance(value, text_type), "%r is not unicode" % value
t = Token(positions, chars, removestops=removestops, mode=mode)
if not tokenize:
t.original = t.text = value
t.boost = 1.0
if positions:
t.pos = start_pos
if chars:
t.startchar = start_char
t.endchar = start_char + len(value)
yield t
else:
if self.strip:
strip = lambda s: s.strip()
else:
strip = lambda s: s
pos = start_pos
startchar = start_char
for s, l in \
[(strip(s), len(s)) for s in
self.segmenter.tokenize(value)]:
t.text = s
t.boost = 1.0
if keeporiginal:
t.original = t.text
t.stopped = False
if positions:
t.pos = pos
pos += 1
if chars:
t.startchar = startchar
startchar += l
t.endchar = startchar
yield t
示例14: __call__
def __call__(
self,
value,
positions=False,
chars=False,
keeporiginal=False,
removestops=True,
start_pos=0,
start_char=0,
tokenize=True,
mode="",
**kwargs
):
assert isinstance(value, text_type), "%r is not unicode" % value
t = Token(positions, chars, removestops=removestops, mode=mode)
if not tokenize:
t.original = t.text = value
t.boost = 1.0
if positions:
t.pos = start_pos
if chars:
t.startchar = start_char
t.endchar = start_char + len(value)
yield t
else:
pos = start_pos
for m in self.tagger.parse(value):
t.text = m.surface
t.feature = m.feature
# TODO: use base form.
t.boost = 1.0
if keeporiginal:
t.original = t.text
t.stopped = False
if positions:
t.pos = pos
pos += 1
if chars:
t.startchar = start_char + m.start
t.endchar = t.startchar + len(m.surface)
yield t
示例15: frozenset
# STOP_WORDS = frozenset(('a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'can',
# 'for', 'from', 'have', 'if', 'in', 'is', 'it', 'may',
# 'not', 'of', 'on', 'or', 'tbd', 'that', 'the', 'this',
# 'to', 'us', 'we', 'when', 'will', 'with', 'yet',
# 'you', 'your',u'的',u'了',u'和',u'的',u'我',u'你',u'地',u'我们',u'我的',u'你们',u'你的',u'','_'))
STOP_WORDS =frozenset(([for line.strip() in open("stopwords.dic",'r')])
print 'stopwords'
accepted_chars = re.compile(ur"[\u4E00-\u9FA5]+")
class ChineseTokenizer(Tokenizer):
def __call__(self,text,**kargs):
words = jieba.tokenize(text,mode="search")
token = Token()
for (w,start_pos,stop_pos) in words:
if not accepted_chars.match(w):
if len(w)>1:
pass
else:
continue
token.original = token.text = w
token.pos = start_pos
token.startchar = start_pos
token.endchar = stop_pos
yield token
def ChineseAnalyzer(stoplist=STOP_WORDS,minsize=1,stemfn=stem,cachesize=50000):
return ChineseTokenizer() | LowercaseFilter() | StopFilter(stoplist=stoplist,minsize=minsize)\
|StemFilter(stemfn=stemfn, ignore=None,cachesize=cachesize)