本文整理汇总了C#中Tokenizer.process方法的典型用法代码示例。如果您正苦于以下问题：C# Tokenizer.process方法的具体用法？C# Tokenizer.process怎么用？C# Tokenizer.process使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类Tokenizer的用法示例。
在下文中一共展示了Tokenizer.process方法的13个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C#代码示例。
示例1: TokenizeCode3
public void TokenizeCode3()
{
TextReader reader = File.OpenText("code41.txt");
Assert.IsNotNull(reader);
Tokenizer t = new Tokenizer(s_errorHandler, true);
List<Token> tokens = t.process(reader);
Assert.AreEqual(tokens[0].getTokenType(), Token.TokenType.BUILT_IN_TYPE_NAME);
Assert.AreEqual(tokens[1].getTokenType(), Token.TokenType.NAME);
Assert.AreEqual(tokens[2].getTokenType(), Token.TokenType.ASSIGNMENT);
Assert.AreEqual(tokens[3].getTokenType(), Token.TokenType.NUMBER);
Assert.AreEqual(tokens[4].getTokenType(), Token.TokenType.NEW_LINE);
Assert.AreEqual(tokens[5].getTokenType(), Token.TokenType.BUILT_IN_TYPE_NAME);
Assert.AreEqual(tokens[6].getTokenType(), Token.TokenType.NAME);
Assert.AreEqual(tokens[7].getTokenType(), Token.TokenType.ASSIGNMENT);
Assert.AreEqual(tokens[8].getTokenType(), Token.TokenType.QUOTED_STRING);
Assert.AreEqual(tokens[9].getTokenType(), Token.TokenType.NEW_LINE);
Assert.AreEqual(tokens[10].getTokenType(), Token.TokenType.BUILT_IN_TYPE_NAME);
Assert.AreEqual(tokens[11].getTokenType(), Token.TokenType.NAME);
Assert.AreEqual(tokens[12].getTokenType(), Token.TokenType.PARANTHESIS_LEFT);
Assert.AreEqual(tokens[13].getTokenType(), Token.TokenType.PARANTHESIS_RIGHT);
Assert.AreEqual(tokens[14].getTokenType(), Token.TokenType.NEW_LINE);
Assert.AreEqual(tokens[15].getTokenType(), Token.TokenType.NAME);
Assert.AreEqual(tokens[16].getTokenType(), Token.TokenType.ASSIGNMENT);
Assert.AreEqual(tokens[17].getTokenType(), Token.TokenType.NUMBER);
Assert.AreEqual(tokens[18].getTokenType(), Token.TokenType.EOF);
}示例2: CreatingSimpleTree
public void CreatingSimpleTree()
{
Tokenizer tokenizer = new Tokenizer(s_errorHandler, true);
List<Token> tokens = tokenizer.process(File.OpenText("code4.txt"));
Parser parser = new Parser(tokens, s_errorHandler);
parser.process();
//Console.WriteLine("Tree: " + parser.getAST().toStringTree());
ASTPainter p = new ASTPainter();
p.PaintAST(parser.getAST());
AST root = parser.getAST();
Assert.AreEqual(Token.TokenType.PROGRAM_ROOT, root.getTokenType());
AST statementList = root.getChild(0);
Assert.AreEqual(Token.TokenType.STATEMENT_LIST, statementList.getTokenType());
AST multiplicationTree = statementList.getChild(1);
Assert.AreEqual(Token.TokenType.OPERATOR, multiplicationTree.getTokenType());
AST operand1 = multiplicationTree.getChild(0);
AST operand2 = multiplicationTree.getChild(1);
Assert.AreEqual("a", operand1.getTokenString());
Assert.AreEqual("b", operand2.getTokenString());
}示例3: DeclareVariableInSubscopes
public void DeclareVariableInSubscopes()
{
Tokenizer t = new Tokenizer(s_errorHandler, true);
Parser p = new Parser(t.process(File.OpenText("code16.txt")), s_errorHandler);
p.process();
ScopeBuilder scopeBuilder = new ScopeBuilder(p.getAST(), s_errorHandler);
scopeBuilder.process();
}示例4: DeclareAndReferenceFunctionsAndVariables
public void DeclareAndReferenceFunctionsAndVariables()
{
Tokenizer t = new Tokenizer(s_errorHandler, true);
Parser p = new Parser(t.process(File.OpenText("code13.txt")), s_errorHandler);
p.process();
ScopeBuilder scopeBuilder = new ScopeBuilder(p.getAST(), s_errorHandler);
scopeBuilder.process();
}示例5: ComplexExpressions
public void ComplexExpressions()
{
Tokenizer tokenizer = new Tokenizer(s_errorHandler, true);
List<Token> tokens = tokenizer.process(File.OpenText("code8.txt"));
Parser parser = new Parser(tokens, s_errorHandler);
parser.process();
//Console.WriteLine("Tree: " + parser.getAST().getTreeAsString());
}示例6: DefineVariableFromCode
public void DefineVariableFromCode()
{
Tokenizer t = new Tokenizer(s_errorHandler, true);
Parser p = new Parser(t.process(File.OpenText("code11.txt")), s_errorHandler);
p.process();
ScopeBuilder scopeBuilder = new ScopeBuilder(p.getAST(), s_errorHandler);
scopeBuilder.process();
Scope globalScope = scopeBuilder.getGlobalScope();
Assert.IsNotNull(globalScope.resolve("x"));
}示例7: ForgettingEndStatement
public void ForgettingEndStatement()
{
Tokenizer t = new Tokenizer(s_errorHandler, true);
ErrorHandler e = new ErrorHandler();
Parser p = new Parser(t.process(File.OpenText("code68.txt")), e);
p.process();
ScopeBuilder scopeBuilder = new ScopeBuilder(p.getAST(), e);
scopeBuilder.process();
e.printErrorsToConsole();
Assert.AreEqual(1, e.getErrors().Count);
}示例8: TokenizeCode1
public void TokenizeCode1()
{
TextReader reader = File.OpenText("code1.txt");
Tokenizer t = new Tokenizer(s_errorHandler, true);
List<Token> tokens = t.process(reader);
Assert.AreEqual(12, tokens.Count);
Assert.AreEqual(tokens[0].getTokenType(), Token.TokenType.NAME);
Assert.AreEqual(tokens[1].getTokenType(), Token.TokenType.ASSIGNMENT);
Assert.AreEqual(tokens[2].getTokenType(), Token.TokenType.NAME);
Assert.AreEqual(tokens[3].getTokenType(), Token.TokenType.OPERATOR);
Assert.AreEqual(tokens[4].getTokenType(), Token.TokenType.NUMBER);
Assert.AreEqual(tokens[5].getTokenType(), Token.TokenType.NEW_LINE);
Assert.AreEqual(tokens[6].getTokenType(), Token.TokenType.NAME);
Assert.AreEqual(tokens[7].getTokenType(), Token.TokenType.PARANTHESIS_LEFT);
Assert.AreEqual(tokens[8].getTokenType(), Token.TokenType.NAME);
Assert.AreEqual(tokens[9].getTokenType(), Token.TokenType.PARANTHESIS_RIGHT);
Assert.AreEqual(tokens[10].getTokenType(), Token.TokenType.NEW_LINE);
Assert.AreEqual(tokens[11].getTokenType(), Token.TokenType.EOF);
}示例9: TokenizeCode2
public void TokenizeCode2()
{
TextReader reader = File.OpenText("code2.txt");
Assert.IsNotNull(reader);
Tokenizer t = new Tokenizer(s_errorHandler, true);
List<Token> tokens = t.process(reader);
Assert.AreEqual(tokens[0].getTokenType(), Token.TokenType.IF);
Assert.AreEqual(tokens[1].getTokenType(), Token.TokenType.PARANTHESIS_LEFT);
Assert.AreEqual(tokens[2].getTokenType(), Token.TokenType.NAME);
Assert.AreEqual(tokens[3].getTokenType(), Token.TokenType.OPERATOR);
Assert.AreEqual(tokens[4].getTokenType(), Token.TokenType.NUMBER);
Assert.AreEqual(tokens[5].getTokenType(), Token.TokenType.PARANTHESIS_RIGHT);
Assert.AreEqual(tokens[6].getTokenType(), Token.TokenType.NEW_LINE);
Assert.AreEqual(tokens[7].getTokenType(), Token.TokenType.NAME);
Assert.AreEqual(tokens[8].getTokenType(), Token.TokenType.PARANTHESIS_LEFT);
Assert.AreEqual(tokens[9].getTokenType(), Token.TokenType.QUOTED_STRING);
Assert.AreEqual(tokens[10].getTokenType(), Token.TokenType.PARANTHESIS_RIGHT);
Assert.AreEqual(tokens[11].getTokenType(), Token.TokenType.NEW_LINE);
Assert.AreEqual(tokens[12].getTokenType(), Token.TokenType.BLOCK_END);
Assert.AreEqual(tokens[13].getTokenType(), Token.TokenType.EOF);
}示例10: TokenizeDoubleQuotedCodeWithSingleQuoteInText
public void TokenizeDoubleQuotedCodeWithSingleQuoteInText()
{
StringReader programString = new StringReader(
"\"can't\""
);
Tokenizer tokenizer = new Tokenizer(s_errorHandler, true);
List<Token> tokens = tokenizer.process(programString);
tokens.ForEach (t => Console.WriteLine(t.getTokenType().ToString() + ", " + t.getTokenString()));
Assert.AreEqual(2, tokens.Count);
Assert.AreEqual(Token.TokenType.QUOTED_STRING, tokens[0].getTokenType());
Assert.AreEqual(Token.TokenType.EOF, tokens[1].getTokenType());
Assert.AreEqual("can't", tokens[0].getTokenString());
}示例11: OperationOrder
public void OperationOrder()
{
Tokenizer tokenizer = new Tokenizer(s_errorHandler, true);
List<Token> tokens = tokenizer.process(File.OpenText("code6.txt"));
Parser parser = new Parser(tokens, s_errorHandler);
parser.process();
//Console.WriteLine("Tree: " + parser.getAST().toStringTree());
Assert.AreEqual("(<STATEMENT_LIST> <GLOBAL_VARIABLE_DEFINITIONS_LIST> (+ (* a b) (+ (* c d) e)))",
parser.getAST().getChild(0).getTreeAsString());
}示例12: ParseSomeExpressionsWithFunctions
public void ParseSomeExpressionsWithFunctions()
{
Tokenizer tokenizer = new Tokenizer(s_errorHandler, true);
List<Token> tokens = tokenizer.process(File.OpenText("code15.txt"));
ErrorHandler errorHandler = new ErrorHandler();
Parser parser = new Parser(tokens, errorHandler);
parser.process();
//Console.WriteLine("Tree: " + parser.getAST().getTreeAsString());
Assert.AreEqual(0, errorHandler.getErrors().Count);
}示例13: ParseCode3
public void ParseCode3()
{
Tokenizer tokenizer = new Tokenizer(s_errorHandler, true);
List<Token> tokens = tokenizer.process(File.OpenText("code3.txt"));
Parser parser = new Parser(tokens, s_errorHandler);
parser.process();
}