Java TokenList类代码示例

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
private static DatatypeTree parseType(TokenList tl) {
	List<String> name = typenameChain(tl);
	List<DatatypeTree> subtypes = new ArrayList<>();
        if (!name.equals("function")
	    && !name.equals("string")
	    && !name.equals("boolean")
	    && !name.equals("number")
	    && acceptIfNext(tl, "<<")) {
		do {
			DatatypeTree t = parseType(tl);
			subtypes.add(t);
		} while (acceptIfNext(tl, ","));
		accept(tl, ">>");
	}
	return new DatatypeTree(name, subtypes);
}
 

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
private static List<AnnotationTree> parseAnnotations(TokenList tl) {
	List<AnnotationTree> annotations = new ArrayList<>();
	while (acceptIfNext(tl, "@")) {
		String file = seek(tl).getFile();
		int line = seek(tl).getLine();
		List<String> namespace = new ArrayList<>();
		while (tl.seekString(1).equals(".")) {
			namespace.add(identifier(tl));
			accept(tl, ".");
			skipNewlines(tl);
		}
		String name = "@" + identifier(tl);
		ArgumentsTree arguments;
		if (acceptIfNext(tl, "(")) {
			arguments = parseArguments(tl, true);
			accept(tl, ")");
		} else arguments = parseArguments(tl, false, true);
		annotations.add(new AnnotationTree(file, line, name, namespace, arguments));
	}
	return annotations;
}
 

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
static ParameterTree parseParameter(TokenList tl, boolean allowTypes) {
	boolean reference = false;
	if (acceptIfNext(tl, "&")) {
		reference = true;
	}
	String name = varname(tl);
	DatatypeTree type = null;
	if (!reference && allowTypes && isNext(tl, ":")) {
		accept(tl, ":");
		type = parseType(tl);
	}
	ExpressionTree expr = null;
	if (!reference && acceptIfNext(tl, "=")) {
		expr = parseExpression(tl);
	}
	return new ParameterTree(name, reference, type, expr);
}
 

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
static Command parseCommand(TokenList tl, boolean allowSugarVars) {
	if (allowSugarVars) sugarVars.push(new ArrayList<>());
	
	Command cmd = parsePrefixCommand(tl);
	cmd = parseSuffix(tl, cmd);
	
	if (allowSugarVars) {
		List<String> sfvs = sugarVars.pop();
		
		if (!sfvs.isEmpty()) {
			cmd = _makeForCommand(
					cmd.file,
					cmd.line,
					sfvs,
					null, null,
					Arrays.asList(new StatementTree(Arrays.asList(cmd))));
		}
	}
	return cmd;
}
 

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
private static ArgumentsTree parseArguments(TokenList tl, boolean allowNewlines, boolean allowPrefixes) {
	ArgumentsTree arguments = new ArgumentsTree();
	arguments.arguments = new ArrayList<>();
	arguments.kwarguments = new ArrayList<>();
	boolean kwargMode = false;
	while ((allowNewlines || !tl.isNext(";", "\n"))
			&& (allowPrefixes || !tl.isNext("for", "while", "until", "if", "unless"))
			&& !isNext(tl, "|", ")", "]", "}", "in", "do", "else", "done", "<>", "<EOF>")) {
		if (validIdentifier(tl.seekString()) && tl.seekString(1).equals("=") || kwargMode) { // TODO: ei salli rivinvaihtoa nimen ja =-merkin väliin
			kwargMode = true;
			String name = identifier(tl);
			accept(tl, "=");
			arguments.kwarguments.add(_makeKwArgument(name, parseExpression(tl)));
		}
		else {
			boolean flattened = acceptIfNext(tl, "*");
			arguments.arguments.add(_makeArgument(flattened,
						    parseExpression(tl)));
		}
		if (!acceptIfNext(tl, ",")) break;
		else skipNewlines(tl);
	}
	return arguments;
}
 

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
private static JSONMap parseMap(TokenList tl, Stack<JSONKey> path) {
	Map<JSONKeyString, JSONElement> map = new HashMap<>();
	tl.accept("{");
	int i = 0;
	while (!tl.isNext("}")) {
		if (i != 0) tl.accept(",");
		tl.accept("\"");
		JSONKeyString key = new JSONKeyString(tl.nextString());
		tl.accept("\"");
		tl.accept(":");
		path.push(key);
		map.put(key, parse(tl, path));
		path.pop();
		i++;
	}
	tl.accept("}");
	return new JSONMap(new ArrayList<>(path), map);
}
 

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
@Override
public void proceed(@NonNull TokenList tl, @NonNull FortobEnvironment env) {
	env.putLocal("?", tokenListObj(tl, env));
	
	if (tl.isNext("\\")) {
		tl.accept("\\");
		return;
	}
	
	while (tl.hasNext() ? commandMap.containsKey(tl.seekString()) : false) {
		proceedOnce(tl, env);
		if (tl.isNext("\\")) {
			tl.accept("\\");
			break;
		}
	}
}
 

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
/**
 * The constructor
 * 
 * @param tl TokenList
 */
public Calculator(TokenList tl) {
	this.tokenlist = tl;
	
	/* Declares the operator library – all operators use parsePrimary() as their RHS parser */
	library = new OperatorLibrary<>(() -> parsePrimary(tl));
	
	/* Declares the operators*/
	library.add("+", (a, b) -> a + b);
	library.add("-", (a, b) -> a - b);
	library.increaseLevel();
	library.add("*", (a, b) -> a * b);
	library.add("/", (a, b) -> a / b);
	
	/* Declares the OPP*/
	opparser = OperatorPrecedenceParser.fromLibrary(library);
}
 

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
private static Token seek(TokenList tl) {
	int i = 1;
	while (tl.has(i)) {
		if (!tl.seekString(i-1).equals("\n") && !tl.seekString(i-1).equals(";")) return tl.seek(i-1);
		i++;
	}
	throw new ParsingException("Unexpected EOF", tl.seek(i-2));
}
 

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
private static boolean acceptIfNext(TokenList tl, String... keywords) {
	if (isNext(tl, keywords)) {
		skipNewlines(tl);
		tl.accept(keywords);
		return true;
	}
	return false;
}
 

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
private static String varname(TokenList tl) {
        Token token = next(tl);
	if (!validIdentifier(token.getToken()) && !token.getToken().equals("_"))
		throw new ParsingException(TokenList.expected("identifier or _"),
					   token);
	return token.getToken();
}
 

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
private static String identifier(TokenList tl) {
        Token token = next(tl);
	if (!validIdentifier(token.getToken()))
		throw new ParsingException(TokenList.expected("identifier"),
					   token);
	return token.getToken();
}
 

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
private static String typename(TokenList tl) {
        Token token = next(tl);
	if (!validTypename(token.getToken()))
		throw new ParsingException(TokenList.expected("typename"),
					   token);
	return token.getToken();
}
 

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
static ProgramTree parse(TokenList tl) {
	//System.err.println(tl);
	if (acceptIfNext(tl, "#")) { // purkkaa, tee tämä lekseriin
		while (!nextString(tl).equals("\n"));
	}

	List<FunctionTree> functions = new ArrayList<>();
	List<List<StatementTree>> preBlocks = new ArrayList<>(), postBlocks = new ArrayList<>();
	List<RecordTree> records = new ArrayList<>();
	while (!isNext(tl, "<EOF>")) {
		skipNewlines(tl);
		List<AnnotationTree> annotations = parseAnnotations(tl);
		if (isNext(tl, "record")) {
			records.add(parseRecord(tl, annotations));
		}
		else if (isNext(tl, "{")) {
			preBlocks.add(parseBody(tl));
		}
		/* TODO: ei salli uusiarivejä väliin */
		else if (validIdentifier(tl.seekString()) && tl.seekString(1).equals(":")) {
			String eventName = nextString(tl);
			accept(tl, ":");
			List<StatementTree> block = parseBody(tl);
			if (eventName.equals("pre_load")) {
				preBlocks.add(block);
			}
			else if (eventName.equals("post_load")) {
				postBlocks.add(block);
			}
		}
		else {
			functions.add(parseFunction(tl, true));
		}
	}
	return new ProgramTree(functions, records, preBlocks, postBlocks);
}
 

import org.kaivos.nept.parser.TokenList; //导入依赖的package包/类
static FunctionTree parseFunction(TokenList tl, boolean mayBeAnnotation) {
	acceptIfNext(tl, "function");

	boolean isAnnotation = mayBeAnnotation ? acceptIfNext(tl, "@") : false;
	String name = (isAnnotation ? "@" : "") + identifier(tl);
	List<String> typeparams = parseTypeparameters(tl);

	List<ParameterTree> parameters = new ArrayList<>();
	List<ParameterTree> kwparameters = new ArrayList<>();
	boolean kwargsMode = false;
	boolean isVarargs = false;
	boolean parentheses = acceptIfNext(tl, "(");
	while (!isNext(tl, ":") && !isNext(tl, "{") && !isNext(tl, ")")) {
		ParameterTree p = parseParameter(tl, parentheses);
		if (p.defaultValue != null || kwargsMode) {
			kwargsMode = true;
			kwparameters.add(p);
		}
		else if (!kwargsMode) parameters.add(p);
		else accept(tl, "="); // kw-parametrin jälkeen ei voi tulla tavallisia parametreja
		if (!isVarargs && acceptIfNext(tl, "...")) {
			isVarargs = true;
		}
		if (!isNext(tl, ":") && !isNext(tl, "{") && !isNext(tl, ")")) {
			accept(tl, ",");
		}
	}
	if (parentheses) accept(tl, ")");

	List<StatementTree> body = parseBody(tl);

	return new FunctionTree(name, typeparams, parameters, isVarargs, kwparameters, body);
}