本文整理汇总了C#中ITokenizer.Next方法的典型用法代码示例。如果您正苦于以下问题：C# ITokenizer.Next方法的具体用法？C# ITokenizer.Next怎么用？C# ITokenizer.Next使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类ITokenizer的用法示例。
在下文中一共展示了ITokenizer.Next方法的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C#代码示例。
示例1: ParseListNoDelimiter
private void ParseListNoDelimiter(ITokenizer tokenizer, Dictionary<string, string> result)
{
while ( !tokenizer.AtEnd && tokenizer.Token != ":" ) {
String p1 = tokenizer.Token;
tokenizer.Next();
String p2 = tokenizer.Token;
if ( p2 == "=" ) {
tokenizer.Next();
String p3 = tokenizer.Token;
result[p1] = p3;
tokenizer.Next();
} else {
result[p1] = "";
// tokenizer.Token contains the next option we want to process
// so don't consume it
}
}
}示例2: ParseModeLine
private void ParseModeLine(ITokenizer tokenizer, Dictionary<string, string> result)
{
// we expect it to look something like this:
// vim: noai:ts=4:sw=4
// vim: set noai ts=4 sw=4:
if ( !tokenizer.Next() ) return;
String tool = tokenizer.Token;
if ( !tokenizer.Next() ) return;
if ( tokenizer.Token != ":" ) return; // not what we expect
if ( !tokenizer.Next() ) return;
if ( tokenizer.Token == "set" ) {
tokenizer.Next();
ParseListNoDelimiter(tokenizer, result);
} else {
ParseList(tokenizer, result, ":");
}
}示例3: ParseList
private void ParseList(ITokenizer tokenizer, Dictionary<string, string> result, string delimiter)
{
while ( !tokenizer.AtEnd ) {
String p1 = tokenizer.Token;
tokenizer.Next();
String p2 = tokenizer.Token;
if ( p2 == "=" ) {
tokenizer.Next();
String p3 = tokenizer.Token;
result[p1] = p3;
tokenizer.Next();
} else {
result[p1] = "";
}
if ( tokenizer.Token != delimiter ) break;
// consume the delimiter
tokenizer.Next();
}
}示例4: PennTreeReader
/// <summary>
/// Read parse trees from a Reader.
/// </summary>
/// <param name="input">The Reader</param>
/// <param name="tf">TreeFactory -- factory to create some kind of Tree</param>
/// <param name="tn">the method of normalizing trees</param>
/// <param name="st">Tokenizer that divides up Reader</param>
public PennTreeReader(TextReader input, ITreeFactory tf, TreeNormalizer tn, ITokenizer<string> st)
{
reader = input;
treeFactory = tf;
treeNormalizer = tn;
tokenizer = st;
// check for whacked out headers still present in Brown corpus in Treebank 3
string first = (st.HasNext() ? st.Peek() : null);
if (first != null && first.StartsWith("*x*x*x"))
{
int foundCount = 0;
while (foundCount < 4 && st.HasNext())
{
first = st.Next();
if (first != null && first.StartsWith("*x*x*x"))
{
foundCount++;
}
}
}
}