C# Lexer.Tokenise方法代码示例

        public void Lexer_NoMatchingTokens_ShouldThrowException()
        {
            var lexer = new Lexer(Enumerable.Empty<TokenDefinition>());

            var result = lexer.Tokenise("   ").ToArray();

            Assert.Fail("Exception should have been thrown");
        }

        public void Lexer_EmptyString_ShouldReturnOneTokenOfEnd()
        {
            var lexer = new Lexer(Enumerable.Empty<TokenDefinition>());

            var result = lexer.Tokenise("");

            AssertTokens("Empty string should have no tokens", result);
        }

        public void Lexer_DoesNotReturnIgnoreTokens_ReturnsRightTokens()
        {
            var def = new[] { new TokenDefinition(@"[-+]?\d+", Token.Int, TokenDefinitionSettings.LexerIgnore) };
            var lexer = new Lexer(def);

            var result = lexer.Tokenise("1");

            AssertTokens("Lexer should not return token when it's set to ignore", result);
        }

        public void Lexer_CanMatchAToken_ShouldReturnAppropriateTokens()
        {
            var def = new[] { new TokenDefinition(@"[-+]?\d+", Token.Int) };
            var lexer = new Lexer(def);

            var result = lexer.Tokenise("4");

            AssertTokens("String with only one int should only have int token", result, Token.Int);
        }

        public void Lexer_CanHandleMultipleTokenDefs()
        {
            var def = new[]
            {
                new TokenDefinition(@"[-+]?\d+", Token.Int, TokenDefinitionSettings.LexerIgnore),
                new TokenDefinition(@"\(", Token.OpenBracket),
                new TokenDefinition(@"\)", Token.CloseBracket),
            };

            var lexer = new Lexer(def);

            var result = lexer.Tokenise("(3)23(())");
            var expected = new [] { Token.OpenBracket, Token.CloseBracket, Token.OpenBracket, Token.OpenBracket, Token.CloseBracket, Token.CloseBracket };

            AssertTokens("Brackets should be lexed", result, expected);
        }

		// Given a bunch of raw text, load all nodes that were inside it.
		// You can call this multiple times to append to the collection of nodes,
		// but note that new nodes will replace older ones with the same name.
		// Returns the number of nodes that were loaded.
		public Program Load(string text, Library library, string fileName, Program includeProgram, bool showTokens, bool showParseTree, string onlyConsiderNode) {

			// The final parsed nodes that were in the file we were given
			Dictionary<string, Yarn.Parser.Node> nodes = new Dictionary<string, Parser.Node>();

			// Load the raw data and get the array of node title-text pairs
			var nodeInfos = ParseInput (text);

			int nodesLoaded = 0;

			foreach (NodeInfo nodeInfo in nodeInfos) {

				if (onlyConsiderNode != null && nodeInfo.title != onlyConsiderNode)
					continue;

				// Attempt to parse every node; log if we encounter any errors
				#if CATCH_EXCEPTIONS
				try {
				#endif 
					
					if (nodes.ContainsKey(nodeInfo.title)) {
						throw new InvalidOperationException("Attempted to load a node called "+
							nodeInfo.title+", but a node with that name has already been loaded!");
					}

					var lexer = new Lexer ();
					var tokens = lexer.Tokenise (nodeInfo.title, nodeInfo.text);

					if (showTokens)
						PrintTokenList (tokens);

					var node = new Parser (tokens, library).Parse();

					// If this node is tagged "rawText", then preserve its source
					if (string.IsNullOrEmpty(nodeInfo.tags) == false && 
						nodeInfo.tags.Contains("rawText")) {
						node.source = nodeInfo.text;
					}

					node.name = nodeInfo.title;

					if (showParseTree)
						PrintParseTree(node);

					nodes[nodeInfo.title] = node;

					nodesLoaded++;

				#if CATCH_EXCEPTIONS
				} catch (Yarn.TokeniserException t) {
					// Add file information
					var message = string.Format ("In file {0}: Error reading node {1}: {2}", fileName, nodeInfo.title, t.Message);
					throw new Yarn.TokeniserException (message);
				} catch (Yarn.ParseException p) {
					var message = string.Format ("In file {0}: Error parsing node {1}: {2}", fileName, nodeInfo.title, p.Message);
					throw new Yarn.ParseException (message);
				} catch (InvalidOperationException e) {
					var message = string.Format ("In file {0}: Error reading node {1}: {2}", fileName, nodeInfo.title, e.Message);
					throw new InvalidOperationException (message);
				}
				#endif 


			}

			var compiler = new Yarn.Compiler(fileName);

			foreach (var node in nodes) {
				compiler.CompileNode (node.Value);
			}

			if (includeProgram != null) {
				compiler.program.Include (includeProgram);
			}

			return compiler.program;

		}