Java StreamTokenizer.TT_EOF屬性代碼示例

private AppConfigurationEntry parseAppConfigurationEntry(StreamTokenizer tokenizer) throws IOException {
    String loginModule = tokenizer.sval;
    if (tokenizer.nextToken() == StreamTokenizer.TT_EOF)
        throw new IllegalArgumentException("Login module control flag not specified in JAAS config");
    LoginModuleControlFlag controlFlag = loginModuleControlFlag(tokenizer.sval);
    Map<String, String> options = new HashMap<>();
    while (tokenizer.nextToken() != StreamTokenizer.TT_EOF && tokenizer.ttype != ';') {
        String key = tokenizer.sval;
        if (tokenizer.nextToken() != '=' || tokenizer.nextToken() == StreamTokenizer.TT_EOF || tokenizer.sval == null)
            throw new IllegalArgumentException("Value not specified for key '" + key + "' in JAAS config");
        String value = tokenizer.sval;
        options.put(key, value);
    }
    if (tokenizer.ttype != ';')
        throw new IllegalArgumentException("JAAS config entry not terminated by semi-colon");
    return new AppConfigurationEntry(loginModule, controlFlag, options);
}
 

public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
 

private static void loadCmdFile(String name, ListBuffer<String> args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        args.append(st.sval);
    }
    r.close();
}
 

/**
 * Parses a option line likes
 *    -genkaypair -dname "CN=Me"
 * and add the results into a list
 * @param list the list to fill into
 * @param s the line
 */
private static void parseArgsLine(List<String> list, String s)
        throws IOException, PropertyExpander.ExpandException {
    StreamTokenizer st = new StreamTokenizer(new StringReader(s));

    st.resetSyntax();
    st.whitespaceChars(0x00, 0x20);
    st.wordChars(0x21, 0xFF);
    // Everything is a word char except for quotation and apostrophe
    st.quoteChar('"');
    st.quoteChar('\'');

    while (true) {
        if (st.nextToken() == StreamTokenizer.TT_EOF) {
            break;
        }
        list.add(PropertyExpander.expand(st.sval));
    }
}
 

public JaasConfig(String loginContextName, String jaasConfigParams) {
    StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(jaasConfigParams));
    tokenizer.slashSlashComments(true);
    tokenizer.slashStarComments(true);
    tokenizer.wordChars('-', '-');
    tokenizer.wordChars('_', '_');
    tokenizer.wordChars('$', '$');

    try {
        configEntries = new ArrayList<>();
        while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
            configEntries.add(parseAppConfigurationEntry(tokenizer));
        }
        if (configEntries.isEmpty())
            throw new IllegalArgumentException("Login module not specified in JAAS config");

        this.loginContextName = loginContextName;

    } catch (IOException e) {
        throw new KafkaException("Unexpected exception while parsing JAAS config");
    }
}
 

private static void loadCmdFile(String name, List<String> args)
    throws IOException
{
    Reader r = new BufferedReader(new FileReader(name));
    StreamTokenizer st = new StreamTokenizer(r);
    st.resetSyntax();
    st.wordChars(' ', 255);
    st.whitespaceChars(0, ' ');
    st.commentChar('#');
    st.quoteChar('"');
    st.quoteChar('\'');
    while (st.nextToken() != StreamTokenizer.TT_EOF) {
        args.add(st.sval);
    }
    r.close();
}
 

Token next() throws IOException {
  int type = tok.nextToken();
  switch (type) {
    case StreamTokenizer.TT_EOF:
    case StreamTokenizer.TT_EOL:
      return null;
    case StreamTokenizer.TT_NUMBER:
      return new NumToken(tok.nval);
    case StreamTokenizer.TT_WORD:
      return new StrToken(TType.IDENT, tok.sval);
    case '"':
      return new StrToken(TType.QUOT, tok.sval);
    default:
      switch (type) {
        case ',':
          return new Token(TType.COMMA);
        case '(':
          return new Token(TType.LPAREN);
        case ')':
          return new Token(TType.RPAREN);
        default:
          throw new IOException("Unexpected: " + type);
      }
  }
}
 

String[] parseBenchArgs(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Vector vec = new Vector();
    for (;;) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                return (String[]) vec.toArray(new String[vec.size()]);

            case StreamTokenizer.TT_WORD:
            case '"':
                vec.add(tokens.sval);
                tokens.nextToken();
                break;

            default:
                throw new ConfigFormatException("unrecognized arg token " +
                        "on line " + tokens.lineno());
        }
    }
}
 

public ParserState parse(StreamTokenizer st) throws IOException
{
    ParserState nextState = null;
    while(nextState == null && st.ttype != StreamTokenizer.TT_EOF)
    {
        int nextToken = st.nextToken();
        if (nextToken != StreamTokenizer.TT_EOL && nextToken != StreamTokenizer.TT_EOF)
        {
            if (nextToken == StreamTokenizer.TT_WORD && st.sval.equals(AbstractMithraDataFileParser.CLASS_IDENTIFIER))
            {
                nextState = this.getParser().getClassReaderState();
            }
            else
            {
                nextState = this.getParser().getDataReaderState();
            }
        }
    }
    return nextState;
}
 

private void checkEndOfFile() throws SgfError, IOException
{
    while (true)
    {
        m_tokenizer.nextToken();
        int t = m_tokenizer.ttype;
        if (t == '(')
            throw getError("Multiple SGF trees not supported");
        else if (t == StreamTokenizer.TT_EOF)
            return;
        else if (t != ' ' && t != '\t' && t != '\n' && t != '\r')
        {
            setWarning("Extra text after SGF tree");
            return;
        }
    }
}
 

public static List<String> tokenizeString(final String str) {
    final StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(str));
    tokenizer.resetSyntax();
    tokenizer.wordChars(0, 255);
    tokenizer.whitespaceChars(0, ' ');
    tokenizer.commentChar('#');
    tokenizer.quoteChar('"');
    tokenizer.quoteChar('\'');
    final List<String> tokenList = new ArrayList<>();
    final StringBuilder toAppend = new StringBuilder();
    while (nextToken(tokenizer) != StreamTokenizer.TT_EOF) {
        final String s = tokenizer.sval;
        // The tokenizer understands about honoring quoted strings and recognizes
        // them as one token that possibly contains multiple space-separated words.
        // It does not recognize quoted spaces, though, and will split after the
        // escaping \ character. This is handled here.
        if (s.endsWith("\\")) {
            // omit trailing \, append space instead
            toAppend.append(s.substring(0, s.length() - 1)).append(' ');
        } else {
            tokenList.add(toAppend.append(s).toString());
            toAppend.setLength(0);
        }
    }
    if (toAppend.length() != 0) {
        tokenList.add(toAppend.toString());
    }
    return tokenList;
}
 

public String toString() {
    StringBuilder sb = new StringBuilder();
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        sb.append("ttype=TT_EOL");
        break;
    case StreamTokenizer.TT_EOF:
        sb.append("ttype=TT_EOF");
        break;
    case StreamTokenizer.TT_NUMBER:
        sb.append("ttype=TT_NUM,").append("nval="+nval);
        break;
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            sb.append("ttype=TT_WORD:IDENTIFIER");
        } else {
            sb.append("ttype=TT_WORD:").append("sval="+sval);
        }
        break;
    default:
        if (ttype == (int)'"') {
            sb.append("ttype=TT_STRING:").append("sval="+sval);
        } else {
            sb.append("ttype=TT_CHAR:").append((char)ttype);
        }
        break;
    }
    return sb.toString();
}
 

/** Delivers the next token and checks for an unexpected end of line or file. */
public static void getNextToken(StreamTokenizer tokenizer) throws IOException {
	if (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
		throw new IOException("unexpected end of line " + tokenizer.lineno());
	}

	if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
		throw new IOException("unexpected end of file in line " + tokenizer.lineno());
	} else if (tokenizer.ttype == '\'' || tokenizer.ttype == '"') {
		tokenizer.ttype = StreamTokenizer.TT_WORD;
	} else if (tokenizer.ttype == StreamTokenizer.TT_WORD && tokenizer.sval.equals("?")) {
		tokenizer.ttype = '?';
	}
}
 

private static List<String> tokenize(String sentence) {
    List<String> tokens = new ArrayList<>();
    StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(sentence));

    try {
        while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
            if (tokenizer.ttype == StreamTokenizer.TT_WORD) {
                tokens.add(tokenizer.sval);
            }
        }
        return tokens;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public ParserState parse(StreamTokenizer st) throws IOException, ParseException
{
    if (this.sectionName == null)
    {
        throw new ParseException("no section name found before line " + st.lineno(), st.lineno());
    }

    // parse the data
    int currentAttribute = 0;
    int token = st.ttype;

    boolean wantData = true;
    List<Object> rowValue = FastList.newList();
    while (token != StreamTokenizer.TT_EOL && token != StreamTokenizer.TT_EOF)
    {
        if (wantData)
        {
            this.getParser().getExpectedTable().parseData(st, currentAttribute, rowValue);
            currentAttribute++;
        }
        else
        {
            if (token != ',')
            {
                throw new ParseException("Expected a comma on line " + st.lineno(), st.lineno());
            }
        }
        wantData = !wantData;
        token = st.nextToken();
    }
    if (Iterate.notEmpty(rowValue))
    {
        this.getParser().getExpectedTable().addRowToList(rowValue);
    }
    return this.getParser().getBeginningOfLineState();
}