本文整理汇总了C#中Tokenizer.EnumerateTokens方法的典型用法代码示例。如果您正苦于以下问题：C# Tokenizer.EnumerateTokens方法的具体用法？C# Tokenizer.EnumerateTokens怎么用？C# Tokenizer.EnumerateTokens使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类Tokenizer的用法示例。
在下文中一共展示了Tokenizer.EnumerateTokens方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C#代码示例。
示例1: FirstTest
public void FirstTest()
{
var luaFile = TestUtils.GetTestPath(@"lua-5.2.1-tests\literals.lua");
var engine = Lua.CreateEngine();
var context = Lua.GetLuaContext(engine);
var unit = context.CreateFileUnit(luaFile);
var reader = TestUtils.OpenReaderOrIgnoreTest(unit.GetReader);
Console.WriteLine("Reading data from {0}", new Uri(luaFile));
var tokenizer = new Tokenizer(ErrorSink.Default, new LuaCompilerOptions() { SkipFirstLine = true });
tokenizer.Initialize(null, reader, unit, SourceLocation.MinValue);
var fname = @"C:\tmp\tokenizer.txt";
using (var fout = File.CreateText(fname))
{
foreach (var token in tokenizer.EnumerateTokens().TakeWhile(t => t.Symbol != Symbol.Eof))
{
if (token.Symbol == Symbol.Whitespace)
continue;
if (token.Symbol == Symbol.EndOfLine)
continue;
fout.Write("{0,-12}", token.Symbol);
fout.Write("{0,-10}", token.Span.Start);
fout.Write("{0,-10}", token.Span.End);
fout.Write("{0}", token.Lexeme);
fout.WriteLine();
}
}
Console.WriteLine("Written results to {0}", new Uri(fname));
}示例2: RunLexerOnLuaTestSuiteFile
public void RunLexerOnLuaTestSuiteFile(string luaFile, bool useLua52)
{
var options = new LuaCompilerOptions()
{
SkipFirstLine = true,
UseLua52Features = useLua52,
};
var engine = Lua.CreateEngine();
var context = Lua.GetLuaContext(engine);
var unit = context.CreateFileUnit(luaFile);
var reader = TestUtils.OpenReaderOrIgnoreTest(unit.GetReader);
var tokenizer = new Tokenizer(ErrorSink.Default, options);
var sw = new Stopwatch();
sw.Start();
tokenizer.Initialize(null, reader, unit, SourceLocation.MinValue);
int counter = tokenizer.EnumerateTokens()
.TakeWhile(t => t.Symbol != Symbol.Eof)
.Count();
sw.Stop();
Console.WriteLine("Tokenizer run: {0} ms, {1} tokens", sw.ElapsedMilliseconds, counter);
}示例3: LexerErrorReportTests
public void LexerErrorReportTests(string snippet, string expect)
{
bool mustfail = TestContext.CurrentContext.Test.Properties.Contains("FailureCase");
var tokenizer = new Tokenizer(ErrorSink.Default, new LuaCompilerOptions() { SkipFirstLine = true });
var sourceUnit = engine.GetLuaContext().CreateSnippet(snippet, SourceCodeKind.Expression);
tokenizer.Initialize(null, sourceUnit.GetReader(), sourceUnit, SourceLocation.MinValue);
try
{
var unused = tokenizer.EnumerateTokens(s => true) // all tokens
.TakeWhile(t => t.Symbol != Symbol.Eof)
.Last();
if (mustfail)
Assert.Fail("Expected a SyntaxErrorException");
}
catch (SyntaxErrorException ex)
{
Assert.That(ex.Message, Is.EqualTo(expect));
}
}