本文整理汇总了C#中Tokenizer.token方法的典型用法代码示例。如果您正苦于以下问题：C# Tokenizer.token方法的具体用法？C# Tokenizer.token怎么用？C# Tokenizer.token使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类Tokenizer的用法示例。
在下文中一共展示了Tokenizer.token方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C#代码示例。
示例1: tokenize_file
void tokenize_file (SourceFile sourceFile, ModuleContainer module, ParserSession session)
{
Stream input;
try {
input = File.OpenRead (sourceFile.Name);
} catch {
Report.Error (2001, "Source file `" + sourceFile.Name + "' could not be found");
return;
}
using (input){
SeekableStreamReader reader = new SeekableStreamReader (input, ctx.Settings.Encoding);
var file = new CompilationSourceFile (module, sourceFile);
Tokenizer lexer = new Tokenizer (reader, file, session, ctx.Report);
int token, tokens = 0, errors = 0;
while ((token = lexer.token ()) != Token.EOF){
tokens++;
if (token == Token.ERROR)
errors++;
}
Console.WriteLine ("Tokenized: " + tokens + " found " + errors + " errors");
}
return;
}示例2: tokenize_file
void tokenize_file (SourceFile sourceFile, ModuleContainer module, ParserSession session)
{
Stream input;
try {
input = File.OpenRead (sourceFile.Name);
} catch {
Report.Error (2001, "Source file `" + sourceFile.Name + "' could not be found");
return;
}
using (input){
SeekableStreamReader reader = new SeekableStreamReader (input, ctx.Settings.Encoding);
var file = new CompilationSourceFile (module, sourceFile);
if (sourceFile.FileType == SourceFileType.CSharp) {
Tokenizer lexer = new Tokenizer (reader, file, session);
int token, tokens = 0, errors = 0;
while ((token = lexer.token ()) != Token.EOF){
tokens++;
if (token == Token.ERROR)
errors++;
}
} else {
Mono.PlayScript.Tokenizer lexer = new Mono.PlayScript.Tokenizer (reader, file, session);
lexer.ParsingPlayScript = sourceFile.PsExtended;
int token, tokens = 0, errors = 0;
while ((token = lexer.token ()) != Mono.PlayScript.Token.EOF){
tokens++;
if (token == Mono.PlayScript.Token.ERROR)
errors++;
}
}
}
return;
}示例3: ToplevelOrStatement
//
// Deambiguates the input string to determine if we
// want to process a statement or if we want to
// process a compilation unit.
//
// This is done using a top-down predictive parser,
// since the yacc/jay parser can not deambiguage this
// without more than one lookahead token. There are very
// few ambiguities.
//
InputKind ToplevelOrStatement (SeekableStreamReader seekable)
{
Tokenizer tokenizer = new Tokenizer (seekable, source_file, new ParserSession ());
int t = tokenizer.token ();
switch (t){
case Token.EOF:
return InputKind.EOF;
// These are toplevels
case Token.EXTERN:
case Token.OPEN_BRACKET:
case Token.ABSTRACT:
case Token.CLASS:
case Token.ENUM:
case Token.INTERFACE:
case Token.INTERNAL:
case Token.NAMESPACE:
case Token.PRIVATE:
case Token.PROTECTED:
case Token.PUBLIC:
case Token.SEALED:
case Token.STATIC:
case Token.STRUCT:
return InputKind.CompilationUnit;
// Definitely expression
case Token.FIXED:
case Token.BOOL:
case Token.BYTE:
case Token.CHAR:
case Token.DECIMAL:
case Token.DOUBLE:
case Token.FLOAT:
case Token.INT:
case Token.LONG:
case Token.NEW:
case Token.OBJECT:
case Token.SBYTE:
case Token.SHORT:
case Token.STRING:
case Token.UINT:
case Token.ULONG:
return InputKind.StatementOrExpression;
// These need deambiguation help
case Token.USING:
t = tokenizer.token ();
if (t == Token.EOF)
return InputKind.EOF;
if (t == Token.IDENTIFIER)
return InputKind.CompilationUnit;
return InputKind.StatementOrExpression;
// Distinguish between:
// delegate opt_anonymous_method_signature block
// delegate type
case Token.DELEGATE:
t = tokenizer.token ();
if (t == Token.EOF)
return InputKind.EOF;
if (t == Token.OPEN_PARENS || t == Token.OPEN_BRACE)
return InputKind.StatementOrExpression;
return InputKind.CompilationUnit;
// Distinguih between:
// unsafe block
// unsafe as modifier of a type declaration
case Token.UNSAFE:
t = tokenizer.token ();
if (t == Token.EOF)
return InputKind.EOF;
if (t == Token.OPEN_PARENS)
return InputKind.StatementOrExpression;
return InputKind.CompilationUnit;
// These are errors: we list explicitly what we had
// from the grammar, ERROR and then everything else
case Token.READONLY:
case Token.OVERRIDE:
case Token.ERROR:
return InputKind.Error;
// This catches everything else allowed by
// expressions. We could add one-by-one use cases
// if needed.
default:
//.........这里部分代码省略.........