C# CSharp.Tokenizer类代码示例

		void tokenize_file (CompilationUnit file, CompilerContext ctx)
		{
			Stream input;

			try {
				input = File.OpenRead (file.Name);
			} catch {
				Report.Error (2001, "Source file `" + file.Name + "' could not be found");
				return;
			}

			using (input){
				SeekableStreamReader reader = new SeekableStreamReader (input, RootContext.Encoding);
				Tokenizer lexer = new Tokenizer (reader, file, ctx);
				int token, tokens = 0, errors = 0;

				while ((token = lexer.token ()) != Token.EOF){
					tokens++;
					if (token == Token.ERROR)
						errors++;
				}
				Console.WriteLine ("Tokenized: " + tokens + " found " + errors + " errors");
			}
			
			return;
		}

		void tokenize_file (SourceFile sourceFile, ModuleContainer module, ParserSession session)
		{
			Stream input;

			try {
				input = File.OpenRead (sourceFile.Name);
			} catch {
				Report.Error (2001, "Source file `" + sourceFile.Name + "' could not be found");
				return;
			}

			using (input) {
				SeekableStreamReader reader = new SeekableStreamReader (input, ctx.Settings.Encoding);
				var file = new CompilationSourceFile (module, sourceFile);

				if (sourceFile.FileType == SourceFileType.CSharp) {
					Tokenizer lexer = new Tokenizer (reader, file, session);
					int token, tokens = 0, errors = 0;
	
					while ((token = lexer.token ()) != Token.EOF){
						tokens++;
						if (token == Token.ERROR)
							errors++;
					}
				} else {
					Mono.PlayScript.Tokenizer lexer = new Mono.PlayScript.Tokenizer (reader, file, session);
					lexer.ParsingPlayScript = sourceFile.PsExtended;
					int token, tokens = 0, errors = 0;
	
					while ((token = lexer.token ()) != Mono.PlayScript.Token.EOF){
						tokens++;
						if (token == Mono.PlayScript.Token.ERROR)
							errors++;
					}
				}
			}
			
			return;
		}

		//
		// Deambiguates the input string to determine if we
		// want to process a statement or if we want to
		// process a compilation unit.
		//
		// This is done using a top-down predictive parser,
		// since the yacc/jay parser can not deambiguage this
		// without more than one lookahead token.   There are very
		// few ambiguities.
		//
		InputKind ToplevelOrStatement (SeekableStreamReader seekable)
		{
			Tokenizer tokenizer = new Tokenizer (seekable, source_file);
			
			int t = tokenizer.token ();
			switch (t){
			case Token.EOF:
				return InputKind.EOF;
				
			// These are toplevels
			case Token.EXTERN:
			case Token.OPEN_BRACKET:
			case Token.ABSTRACT:
			case Token.CLASS:
			case Token.ENUM:
			case Token.INTERFACE:
			case Token.INTERNAL:
			case Token.NAMESPACE:
			case Token.PRIVATE:
			case Token.PROTECTED:
			case Token.PUBLIC:
			case Token.SEALED:
			case Token.STATIC:
			case Token.STRUCT:
				return InputKind.CompilationUnit;
				
			// Definitely expression
			case Token.FIXED:
			case Token.BOOL:
			case Token.BYTE:
			case Token.CHAR:
			case Token.DECIMAL:
			case Token.DOUBLE:
			case Token.FLOAT:
			case Token.INT:
			case Token.LONG:
			case Token.NEW:
			case Token.OBJECT:
			case Token.SBYTE:
			case Token.SHORT:
			case Token.STRING:
			case Token.UINT:
			case Token.ULONG:
				return InputKind.StatementOrExpression;

			// These need deambiguation help
			case Token.USING:
				t = tokenizer.token ();
				if (t == Token.EOF)
					return InputKind.EOF;

				if (t == Token.IDENTIFIER)
					return InputKind.CompilationUnit;
				return InputKind.StatementOrExpression;


			// Distinguish between:
			//    delegate opt_anonymous_method_signature block
			//    delegate type 
			case Token.DELEGATE:
				t = tokenizer.token ();
				if (t == Token.EOF)
					return InputKind.EOF;
				if (t == Token.OPEN_PARENS || t == Token.OPEN_BRACE)
					return InputKind.StatementOrExpression;
				return InputKind.CompilationUnit;

			// Distinguih between:
			//    unsafe block
			//    unsafe as modifier of a type declaration
			case Token.UNSAFE:
				t = tokenizer.token ();
				if (t == Token.EOF)
					return InputKind.EOF;
				if (t == Token.OPEN_PARENS)
					return InputKind.StatementOrExpression;
				return InputKind.CompilationUnit;
				
		        // These are errors: we list explicitly what we had
			// from the grammar, ERROR and then everything else

			case Token.READONLY:
			case Token.OVERRIDE:
			case Token.ERROR:
				return InputKind.Error;

			// This catches everything else allowed by
			// expressions.  We could add one-by-one use cases
			// if needed.
			default:
//.........这里部分代码省略.........

public CSharpParser (SeekableStreamReader reader, CompilationSourceFile file, Report report, ParserSession session)
{
	this.file = file;
	current_container = current_namespace = file;
	
	this.module = file.Module;
	this.compiler = file.Compiler;
	this.settings = compiler.Settings;
	this.report = report;
	
	lang_version = settings.Version;
	yacc_verbose_flag = settings.VerboseParserFlag;
	doc_support = settings.DocumentationFile != null;
	lexer = new Tokenizer (reader, file, session);
	oob_stack = new Stack<object> ();
	lbag = session.LocationsBag;
	use_global_stacks = session.UseJayGlobalArrays;
	parameters_bucket = session.ParametersStack;
}

public CSharpParser (SeekableStreamReader reader, CompilationUnit file, CompilerContext ctx)
{
	if (RootContext.EvalMode)
		undo = new Undo ();

	this.file = file;
	this.compiler = ctx;
	current_namespace = new NamespaceEntry (ctx, null, file, null);
	current_class = current_namespace.SlaveDeclSpace;
	current_container = current_class.PartialContainer; // == RootContest.ToplevelTypes
	oob_stack.Clear ();
	lexer = new Tokenizer (reader, file, ctx);
	
	use_global_stacks = true;
}

			public Position (Tokenizer t)
			{
				position = t.reader.Position;
				line = t.line;
				ref_line = t.ref_line;
				col = t.col;
				hidden = t.hidden;
				putback_char = t.putback_char;
				previous_col = t.previous_col;
				if (t.ifstack != null && t.ifstack.Count != 0)
					ifstack = (Stack)t.ifstack.Clone ();
				parsing_generic_less_than = t.parsing_generic_less_than;
				current_token = t.current_token;
			}

public CSharpParser (SeekableStreamReader reader, CompilationSourceFile file)
{
	this.file = file;
	current_namespace = file.NamespaceContainer;
	
	this.module = current_namespace.Module;
	this.compiler = module.Compiler;
	this.settings = compiler.Settings;
	lang_version = settings.Version;
	doc_support = settings.Documentation != null;
	current_class = current_namespace.SlaveDeclSpace;
	current_container = current_class.PartialContainer; // == RootContest.ToplevelTypes
	oob_stack.Clear ();
	lexer = new Tokenizer (reader, file, compiler);
	
	use_global_stacks = true;
}

public CSharpParser (SeekableStreamReader reader, CompilationSourceFile file, Report report)
{
	this.file = file;
	current_container = current_namespace = file;
	
	this.module = file.Module;
	this.compiler = file.Compiler;
	this.settings = compiler.Settings;
	this.report = report;
	
	lang_version = settings.Version;
	yacc_verbose_flag = settings.VerboseParserFlag;
	doc_support = settings.DocumentationFile != null;
	oob_stack.Clear ();
	lexer = new Tokenizer (reader, file);

#if FULL_AST
	lbag = new LocationsBag ();
#else
	lbag = null;
#endif
	
	use_global_stacks = true;
}