Java StreamTokenizer.nextToken方法代码示例

import java.io.StreamTokenizer; //导入方法依赖的package包/类
String[] parseBenchArgs(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Vector vec = new Vector();
    for (;;) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                return (String[]) vec.toArray(new String[vec.size()]);

            case StreamTokenizer.TT_WORD:
            case '"':
                vec.add(tokens.sval);
                tokens.nextToken();
                break;

            default:
                throw new ConfigFormatException("unrecognized arg token " +
                        "on line " + tokens.lineno());
        }
    }
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
String parseBenchName(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    String name;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            name = tokens.sval;
            tokens.nextToken();
            return name;

        default:
            throw new ConfigFormatException("missing benchmark name on " +
                    "line " + tokens.lineno());
    }
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
private static void loadCmdFile(String name, List<String> args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        args.add(st.sval);
    }
    r.close();
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
float parseBenchWeight(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                weight = Float.parseFloat(tokens.sval);
            } catch (NumberFormatException e) {
                throw new ConfigFormatException("illegal weight value \"" +
                        tokens.sval + "\" on line " + tokens.lineno());
            }
            tokens.nextToken();
            return weight;

        default:
            throw new ConfigFormatException("missing weight value on line "
                    + tokens.lineno());
    }
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
private AppConfigurationEntry parseAppConfigurationEntry(StreamTokenizer tokenizer) throws IOException {
    String loginModule = tokenizer.sval;
    if (tokenizer.nextToken() == StreamTokenizer.TT_EOF)
        throw new IllegalArgumentException("Login module control flag not specified in JAAS config");
    LoginModuleControlFlag controlFlag = loginModuleControlFlag(tokenizer.sval);
    Map<String, String> options = new HashMap<>();
    while (tokenizer.nextToken() != StreamTokenizer.TT_EOF && tokenizer.ttype != ';') {
        String key = tokenizer.sval;
        if (tokenizer.nextToken() != '=' || tokenizer.nextToken() == StreamTokenizer.TT_EOF || tokenizer.sval == null)
            throw new IllegalArgumentException("Value not specified for key '" + key + "' in JAAS config");
        String value = tokenizer.sval;
        options.put(key, value);
    }
    if (tokenizer.ttype != ';')
        throw new IllegalArgumentException("JAAS config entry not terminated by semi-colon");
    return new AppConfigurationEntry(loginModule, controlFlag, options);
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
public JaasConfig(String loginContextName, String jaasConfigParams) {
    StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(jaasConfigParams));
    tokenizer.slashSlashComments(true);
    tokenizer.slashStarComments(true);
    tokenizer.wordChars('-', '-');
    tokenizer.wordChars('_', '_');
    tokenizer.wordChars('$', '$');

    try {
        configEntries = new ArrayList<>();
        while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
            configEntries.add(parseAppConfigurationEntry(tokenizer));
        }
        if (configEntries.isEmpty())
            throw new IllegalArgumentException("Login module not specified in JAAS config");

        this.loginContextName = loginContextName;

    } catch (IOException e) {
        throw new KafkaException("Unexpected exception while parsing JAAS config");
    }
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
private static void loadCmdFile(String name, List args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != st.TT_EOF) {
        args.add(st.sval);
    }
    r.close();
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
Benchmark parseBenchClass(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Benchmark bench;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                Class cls = Class.forName(tokens.sval);
                bench = (Benchmark) cls.newInstance();
            } catch (Exception e) {
                throw new ConfigFormatException("unable to instantiate " +
                        "benchmark \"" + tokens.sval + "\" on line " +
                        tokens.lineno());
            }
            tokens.nextToken();
            return bench;

        default:
            throw new ConfigFormatException("missing benchmark class " +
                    "name on line " + tokens.lineno());
    }
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
/**
 * Skips the back slash in the next token if it's followed by a new line.  
 */
private static void skipBackSlash(StreamTokenizer tokenizer) throws IOException
{
	tokenizer.ordinaryChar('\\');
	if (tokenizer.nextToken() == '\\')
	{
		if (tokenizer.nextToken() != StreamTokenizer.TT_EOL)
		{
			throw new IncorrectFormatException("Expected new line after \\ character");
		}
	}
	else
	{
		tokenizer.pushBack();
	}
	tokenizer.wordChars('\\', '\\');
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
public static String unquote2(String str, boolean escapeHtml) {
	if (str == null || str.equals("null")) {
		return null;
	}
	StreamTokenizer parser = new StreamTokenizer(new StringReader(str));
	String result;
	try {
		parser.nextToken();
		if (parser.ttype == '"') {
			result = parser.sval;
		} else {
			result = unquote(str, escapeHtml);
		}
	} catch (IOException e) {
		result = unquote(str, escapeHtml);
	}
	
	return result;
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
private static void loadCmdFile(String name, ListBuffer<String> args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        args.append(st.sval);
    }
    r.close();
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
private static HollowSetSchema parseSetSchema(String typeName, StreamTokenizer tokenizer) throws IOException {
    int tok = tokenizer.nextToken();

    if(tokenizer.ttype != '<')
        throw new IOException("Invalid Syntax: Expected '<' after 'Set' for type " + typeName);

    tok = tokenizer.nextToken();
    if(tok != StreamTokenizer.TT_WORD) {
        log.warning("Invalid Syntax: Expected element type declaration: " + typeName);
    }

    String elementType = tokenizer.sval;

    tok = tokenizer.nextToken();
    if(tokenizer.ttype != '>')
        throw new IOException("Invalid Syntax: Expected '>' element type declaration: " + typeName);

    tok = tokenizer.nextToken();
    String hashKeyPaths[] = parseHashKey(tokenizer);

    if(tokenizer.ttype != ';')
        throw new IOException("Invalid Syntax: Expected semicolon after Set schema declaration: " + typeName);

    return new HollowSetSchema(typeName, elementType, hashKeyPaths);
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
@Override
public ParserState parse(StreamTokenizer st) throws IOException, ParseException
{
    int token = st.ttype;
    if (token != StreamTokenizer.TT_WORD || !st.sval.equals(ExpectedResultsParser.SECTION_IDENTIFIER))
    {
        throw new ParseException("expected line " + st.lineno() + " to begin with Section", st.lineno());
    }
    token = st.nextToken();
    if (token != StreamTokenizer.TT_WORD && token != '"')
    {
        throw new ParseException("expected a section name on line " + st.lineno(), st.lineno());
    }
    String testName = st.sval;

    token = st.nextToken();
    String tableName = null;
    if (token == StreamTokenizer.TT_WORD || token == '"')
    {
        tableName = st.sval;
        token = st.nextToken();
    }

    this.getParser().startNewSection(testName, tableName);

    if (token != StreamTokenizer.TT_EOL)
    {
        throw new ParseException("invalid data after the class name on line " + st.lineno(), st.lineno());
    }

    return this.getParser().getHeaderState();
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
/** Delivers the next token and checks for an unexpected end of line or file. */
public static void getNextToken(StreamTokenizer tokenizer) throws IOException {
	if (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
		throw new IOException("unexpected end of line " + tokenizer.lineno());
	}

	if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
		throw new IOException("unexpected end of file in line " + tokenizer.lineno());
	} else if (tokenizer.ttype == '\'' || tokenizer.ttype == '"') {
		tokenizer.ttype = StreamTokenizer.TT_WORD;
	} else if (tokenizer.ttype == StreamTokenizer.TT_WORD && tokenizer.sval.equals("?")) {
		tokenizer.ttype = '?';
	}
}
 

import java.io.StreamTokenizer; //导入方法依赖的package包/类
/** Skips all tokens before next end of line (EOL). */
public static void waitForEOL(StreamTokenizer tokenizer) throws IOException {
	// skip everything until EOL
	while (tokenizer.nextToken() != StreamTokenizer.TT_EOL) {
	}
	;
	tokenizer.pushBack();
}