C# Tokenizer.Accept方法代码示例

 public void Run()
 {
     using (var tr = new StreamReader(setup.BuildData))
       {
     var builder = new MsBuildProjectBuilder(new StandardDefaultValueResolver(), engine);
     var tokenizer = new Tokenizer(tr);
     tokenizer.Accept(builder);
     if (!string.IsNullOrEmpty(setup.OutputXml))
     {
       builder.Project.Save(setup.OutputXml);
       return;
     }
     if (setup.HasProperties)
       foreach (var p in setup.Properties)
     builder.Project.SetProperty(p.Key, p.Value);
     if (setup.Target != null)
       engine.BuildProject(builder.Project, setup.Target);
     else
       engine.BuildProject(builder.Project);
       }
 }

	public static TypeReference ParseTypeReference (Tokenizer tokenizer)
	{
		TypeReference tr = new TypeReference ();
		StringBuilder result = new StringBuilder ();

		if (tokenizer.Accept (Token2Type.Identifier, "const"))
			tr.IsConst = true;

		if (tokenizer.Accept (Token2Type.Identifier, "unsigned"))
			result.Append ("unsigned");

		if (tokenizer.Accept (Token2Type.Identifier, "const"))
			tr.IsConst = true;

		result.Append (tokenizer.GetIdentifier ());

		if (tokenizer.Accept (Token2Type.Punctuation, ":")) {
			tokenizer.AcceptOrThrow (Token2Type.Punctuation, ":");
			result.Append ("::");
			result.Append (tokenizer.GetIdentifier ());
		}

		if (tokenizer.Accept (Token2Type.Identifier, "const"))
			tr.IsConst = true;

		while (tokenizer.Accept (Token2Type.Punctuation, "*"))
			result.Append ("*");

		if (tokenizer.Accept (Token2Type.Identifier, "const"))
			tr.IsConst = true;

		if (tokenizer.Accept (Token2Type.Punctuation, "&"))
			result.Append ("&");

		if (tokenizer.Accept (Token2Type.Identifier, "const"))
			tr.IsConst = true;

		//Console.WriteLine ("ParseTypeReference: parsed '{0}'", result.ToString ());

		tr.Value = result.ToString ();

		return tr;
	}

	static void ParseEnum (Annotations properties, MemberInfo parent, Tokenizer tokenizer)
	{
		FieldInfo field;
		StringBuilder value = new StringBuilder ();
		TypeInfo type = new TypeInfo ();

		type.Annotations = properties;
		type.IsEnum = true;

		tokenizer.AcceptOrThrow (Token2Type.Identifier, "enum");
		if (tokenizer.CurrentToken.type == Token2Type.Identifier) {
			type.Name = tokenizer.GetIdentifier ();
		} else {
			type.Name = "<anonymous>";
		}
		parent.Children.Add (type);

		tokenizer.AcceptOrThrow (Token2Type.Punctuation, "{");

		//Console.WriteLine ("ParseEnum: {0}", name);

		while (tokenizer.CurrentToken.type == Token2Type.Identifier) {
			field = new FieldInfo ();
			field.Name = tokenizer.GetIdentifier ();
			value.Length = 0;
			if (tokenizer.Accept (Token2Type.Punctuation, "=")) {
				while (tokenizer.CurrentToken.value != "," && tokenizer.CurrentToken.value != "}") {
					value.Append (" ");
					value.Append (tokenizer.CurrentToken.value);
					tokenizer.Advance (true);
				}
			}
			field.Value = value.ToString ();
			type.Children.Add (field);
			//Console.WriteLine ("ParseEnum: {0}: {1} {2} {3}", name, field, value.Length != 0 != null ? "=" : "", value);

			if (!tokenizer.Accept (Token2Type.Punctuation, ","))
				break;
		}

		tokenizer.AcceptOrThrow (Token2Type.Punctuation, "}");
		tokenizer.AcceptOrThrow (Token2Type.Punctuation, ";");
	}

	static bool ParseMembers (MemberInfo parent, Tokenizer tokenizer)
	{
		Annotations properties = new Annotations ();
		TypeInfo parent_type = parent as TypeInfo;
		string accessibility;
		TypeReference returntype;
		bool is_dtor;
		bool is_ctor;
		bool is_virtual;
		bool is_static;
		bool is_const;
		bool is_extern;
		string name;

		//Console.WriteLine ("ParseMembers ({0})", type.Name);

		do {
			returntype = null;
			is_dtor = is_ctor = is_virtual = is_static = false;
			is_extern = is_const = false;
			name = null;
			properties = new Annotations ();

			if (parent_type != null)
				accessibility = parent_type.IsStruct ? "public" : "private";
			else
				accessibility = "public";

			try {
				if (tokenizer.Accept (Token2Type.Punctuation, ";"))
					continue;
			} catch {
				return false;
			}

			if (tokenizer.CurrentToken.value == "}")
				return true;

			while (tokenizer.CurrentToken.type == Token2Type.CommentProperty) {
				properties.Add (tokenizer.CurrentToken.value);
				tokenizer.Advance (true);
			}

			//Console.WriteLine ("ParseMembers: Current token: {0}", tokenizer.CurrentToken);

			if (tokenizer.CurrentToken.type == Token2Type.Identifier) {
				string v = tokenizer.CurrentToken.value;
				switch (v) {
				case "public":
				case "protected":
				case "private":
					accessibility = v;
					tokenizer.Advance (true);
					tokenizer.Accept (Token2Type.Punctuation, ":");
					continue;
				case "enum":
					ParseEnum (properties, parent, tokenizer);
					continue;
				case "friend":
					while (!tokenizer.Accept (Token2Type.Punctuation, ";")) {
						tokenizer.Advance (true);
					}
					continue;
				case "struct":
				case "class":
				case "union":
					if (!ParseClassOrStruct (properties, parent, tokenizer))
						return false;
					continue;
				case "typedef":
					StringBuilder requisite = new StringBuilder ();
					requisite.Append (tokenizer.CurrentToken.value);
					requisite.Append (' ');
					tokenizer.Advance (true);
					while (!tokenizer.Accept (Token2Type.Punctuation, ";")) {
						requisite.Append (tokenizer.CurrentToken.value);
						requisite.Append (' ');
						if (tokenizer.CurrentToken.value == "{") {
							tokenizer.Advance (true);
							while (!tokenizer.Accept (Token2Type.Punctuation, "}")) {
								requisite.Append (tokenizer.CurrentToken.value);
								requisite.Append (' ');
								tokenizer.Advance (true);
							}
							requisite.Append (tokenizer.CurrentToken.value);
							requisite.Append (' ');
						}
						tokenizer.Advance (true);
					}
					requisite.Append (";");
					if (properties.ContainsKey ("CBindingRequisite"))
						cbinding_requisites.AppendLine (requisite.ToString ());

					continue;
				case "EVENTHANDLER":
					while (!tokenizer.Accept (Token2Type.Punctuation, ";"))
						tokenizer.Advance (true);
					continue;
				case "template":
					tokenizer.Advance (true);
//.........这里部分代码省略.........

	// Returns false if there are no more tokens (reached end of code)
	static bool ParseClassOrStruct (Annotations annotations, MemberInfo parent, Tokenizer tokenizer)
	{
		TypeInfo type = new TypeInfo ();

		type.Annotations = annotations;
		type.Header = tokenizer.CurrentFile;
		type.Parent = parent;

		type.IsPublic = tokenizer.Accept (Token2Type.Identifier, "public");

		if (tokenizer.Accept (Token2Type.Identifier, "class")) {
			type.IsClass = true;
		} else if (tokenizer.Accept (Token2Type.Identifier, "struct")) {
			type.IsStruct = true;
			type.IsValueType = true;
		} else if (tokenizer.Accept (Token2Type.Identifier, "union")) {
			type.IsStruct = true; // Not entirely correct, but a union can be parsed as a struct
			type.IsValueType = true;
		} else {
			throw new Exception (string.Format ("Expected 'class' or 'struct', not '{0}'", tokenizer.CurrentToken.value));
		}

		if (tokenizer.CurrentToken.type == Token2Type.Identifier) {
			type.Name = tokenizer.GetIdentifier ();
		} else {
			type.Name = "<anonymous>";
		}

		if (tokenizer.Accept (Token2Type.Punctuation, ";")) {
			// A forward declaration.
			//Console.WriteLine ("ParseType: Found a forward declaration to {0}", type.Name);
			return true;
		}

		if (tokenizer.Accept (Token2Type.Punctuation, ":")) {
			if (!tokenizer.Accept (Token2Type.Identifier, "public") && type.IsClass)
				throw new Exception (string.Format ("The base class of {0} is not public.", type.Name));

			type.Base = ParseTypeReference (tokenizer);

			// accept multiple inheritance the easy way
			while (tokenizer.CurrentToken.value == ",") {
				tokenizer.Accept (Token2Type.Punctuation, ",");

				while (tokenizer.CurrentToken.value != "," &&
				       tokenizer.CurrentToken.value != "{")
					tokenizer.GetIdentifier ();
			}

			//Console.WriteLine ("ParseType: Found {0}'s base class: {1}", type.Name, type.Base);
		}

		tokenizer.AcceptOrThrow (Token2Type.Punctuation, "{");

		//Console.WriteLine ("ParseType: Found a type: {0} in {1}", type.Name, type.Header);
		parent.Children.Add (type);
		ParseMembers (type, tokenizer);

		tokenizer.AcceptOrThrow (Token2Type.Punctuation, "}");

		if (tokenizer.CurrentToken.type == Token2Type.Identifier)
			tokenizer.Advance (true);

		if (tokenizer.CurrentToken.value != ";")
			throw new Exception (string.Format ("Expected ';', not '{0}'", tokenizer.CurrentToken.value));

		return tokenizer.Advance (false);
	}

	static GlobalInfo GetTypes2 ()
	{
		string srcdir = Path.Combine (Environment.CurrentDirectory, "src");
		string plugindir = Path.Combine (Environment.CurrentDirectory, "plugin");
		string paldir = Path.Combine (srcdir, "pal");
		string palgtkdir = Path.Combine (paldir, "gtk");
		List<string> all_files = new List<string> ();

		all_files.AddRange (Directory.GetFiles (srcdir, "*.h"));
		all_files.AddRange (Directory.GetFiles (plugindir, "*.h"));
		all_files.AddRange (Directory.GetFiles (paldir, "*.h"));
		all_files.AddRange (Directory.GetFiles (palgtkdir, "*.h"));

		RemoveExcludedSrcFiles (srcdir, plugindir, paldir, all_files);

		Tokenizer tokenizer = new Tokenizer (all_files.ToArray ());
		GlobalInfo all = new GlobalInfo ();

		tokenizer.Advance (false);

		try {
			while (ParseMembers (all, tokenizer)) {
				try {
					tokenizer.Accept (Token2Type.Punctuation, "}");
					tokenizer.Accept (Token2Type.Punctuation, ";");
				} catch {}
			}
		} catch (Exception ex) {
			throw new Exception (string.Format ("{0}({1}): {2}", tokenizer.CurrentFile, tokenizer.CurrentLine, ex.Message), ex);
		}

		// Add all the manual types
		TypeInfo t;
		TypeInfo IComparableInfo;
		TypeInfo IFormattableInfo;
		TypeInfo IConvertibleInfo;
		TypeInfo IEquatableBoolInfo;
		TypeInfo IComparableBoolInfo;
		TypeInfo IEquatableDoubleInfo;
		TypeInfo IComparableDoubleInfo;
		TypeInfo IEquatableFloatInfo;
		TypeInfo IComparableFloatInfo;
		TypeInfo IEquatableCharInfo;
		TypeInfo IComparableCharInfo;
		TypeInfo IEquatableIntInfo;
		TypeInfo IComparableIntInfo;
		TypeInfo IEquatableLongInfo;
		TypeInfo IComparableLongInfo;
		TypeInfo IEquatableStringInfo;
		TypeInfo IComparableStringInfo;
		TypeInfo IEquatableTimeSpanInfo;
		TypeInfo IComparableTimeSpanInfo;
		TypeInfo IEquatableUintInfo;
		TypeInfo IComparableUintInfo;
		TypeInfo IEquatableUlongInfo;
		TypeInfo IComparableUlongInfo;

		all.Children.Add (new TypeInfo ("object", "OBJECT", "INVALID", true, true));

		all.Children.Add (IComparableInfo = new TypeInfo ("IComparable", "ICOMPARABLE", "OBJECT", true, true, false, true));
		all.Children.Add (IFormattableInfo = new TypeInfo ("IFormattable", "IFORMATTABLE", "OBJECT", true, true, false, true));
		all.Children.Add (IConvertibleInfo = new TypeInfo ("IConvertible", "ICONVERTIBLE", "OBJECT", true, true, false, true));

		all.Children.Add (IEquatableBoolInfo = new TypeInfo ("IEquatable<bool>", "IEQUATABLE_BOOL", "OBJECT", true, true, false, true));
		all.Children.Add (IComparableBoolInfo = new TypeInfo ("IComparable<bool>", "ICOMPARABLE_BOOL", "OBJECT", true, true, false, true));

		all.Children.Add (IEquatableDoubleInfo = new TypeInfo ("IEquatable<double>", "IEQUATABLE_DOUBLE", "OBJECT", true, true, false, true));
		all.Children.Add (IComparableDoubleInfo = new TypeInfo ("IComparable<double>", "ICOMPARABLE_DOUBLE", "OBJECT", true, true, false, true));

		all.Children.Add (IEquatableFloatInfo = new TypeInfo ("IEquatable<float>", "IEQUATABLE_FLOAT", "OBJECT", true, true, false, true));
		all.Children.Add (IComparableFloatInfo = new TypeInfo ("IComparable<float>", "ICOMPARABLE_FLOAT", "OBJECT", true, true, false, true));

		all.Children.Add (IEquatableCharInfo = new TypeInfo ("IEquatable<char>", "IEQUATABLE_CHAR", "OBJECT", true, true, false, true));
		all.Children.Add (IComparableCharInfo = new TypeInfo ("IComparable<char>", "ICOMPARABLE_CHAR", "OBJECT", true, true, false, true));

		all.Children.Add (IEquatableIntInfo = new TypeInfo ("IEquatable<int>", "IEQUATABLE_INT", "OBJECT", true, true, false, true));
		all.Children.Add (IComparableIntInfo = new TypeInfo ("IComparable<int>", "ICOMPARABLE_INT", "OBJECT", true, true, false, true));

		all.Children.Add (IEquatableLongInfo = new TypeInfo ("IEquatable<long>", "IEQUATABLE_LONG", "OBJECT", true, true, false, true));
		all.Children.Add (IComparableLongInfo = new TypeInfo ("IComparable<long>", "ICOMPARABLE_LONG", "OBJECT", true, true, false, true));

		all.Children.Add (IEquatableStringInfo = new TypeInfo ("IEquatable<string>", "IEQUATABLE_STRING", "OBJECT", true, true, false, true));
		all.Children.Add (IComparableStringInfo = new TypeInfo ("IComparable<string>", "ICOMPARABLE_STRING", "OBJECT", true, true, false, true));

		all.Children.Add (IEquatableTimeSpanInfo = new TypeInfo ("IEquatable<TimeSpan>", "IEQUATABLE_TIMESPAN", "OBJECT", true, true, false, true));
		all.Children.Add (IComparableTimeSpanInfo = new TypeInfo ("IComparable<TimeSpan>", "ICOMPARABLE_TIMESPAN", "OBJECT", true, true, false, true));

		all.Children.Add (IEquatableUintInfo = new TypeInfo ("IEquatable<uint>", "IEQUATABLE_UINT", "OBJECT", true, true, false, true));
		all.Children.Add (IComparableUintInfo = new TypeInfo ("IComparable<uint>", "ICOMPARABLE_UINT", "OBJECT", true, true, false, true));

		all.Children.Add (IEquatableUlongInfo = new TypeInfo ("IEquatable<ulong>", "IEQUATABLE_ULONG", "OBJECT", true, true, false, true));
		all.Children.Add (IComparableUlongInfo = new TypeInfo ("IComparable<ulong>", "ICOMPARABLE_ULONG", "OBJECT", true, true, false, true));

		all.Children.Add (t = new TypeInfo ("bool", "BOOL", "OBJECT", true, true, true, false));
		t.Interfaces.Add (IComparableInfo);
		t.Interfaces.Add (IComparableBoolInfo);
		t.Interfaces.Add (IConvertibleInfo);
		t.Interfaces.Add (IEquatableBoolInfo);

		all.Children.Add (t = new TypeInfo ("float", "FLOAT", "OBJECT", true, true, true, false));
//.........这里部分代码省略.........