Java StreamTokenizer.TT_EOL属性代码示例

String[] parseBenchArgs(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Vector vec = new Vector();
    for (;;) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                return (String[]) vec.toArray(new String[vec.size()]);

            case StreamTokenizer.TT_WORD:
            case '"':
                vec.add(tokens.sval);
                tokens.nextToken();
                break;

            default:
                throw new ConfigFormatException("unrecognized arg token " +
                        "on line " + tokens.lineno());
        }
    }
}
 

Token next() throws IOException {
  int type = tok.nextToken();
  switch (type) {
    case StreamTokenizer.TT_EOF:
    case StreamTokenizer.TT_EOL:
      return null;
    case StreamTokenizer.TT_NUMBER:
      return new NumToken(tok.nval);
    case StreamTokenizer.TT_WORD:
      return new StrToken(TType.IDENT, tok.sval);
    case '"':
      return new StrToken(TType.QUOT, tok.sval);
    default:
      switch (type) {
        case ',':
          return new Token(TType.COMMA);
        case '(':
          return new Token(TType.LPAREN);
        case ')':
          return new Token(TType.RPAREN);
        default:
          throw new IOException("Unexpected: " + type);
      }
  }
}
 

public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
 

@Override
public ParserState parse(StreamTokenizer st) throws IOException, ParseException
{
    ParserState nextState = null;
    while (nextState == null && st.ttype != StreamTokenizer.TT_EOF)
    {
        int nextToken = st.nextToken();
        if (nextToken != StreamTokenizer.TT_EOL && nextToken != StreamTokenizer.TT_EOF)
        {
            if (nextValueIs(ExpectedResultsParser.SECTION_IDENTIFIER, st, nextToken))
            {
                nextState = this.getParser().getSectionReaderState();
            }
            else if (nextValueIs(ExpectedResultsParser.METADATA_IDENTIFIER, st, nextToken))
            {
                nextState = this.getParser().getMetadataReaderState();
            }
            else
            {
                nextState = this.getParser().getDataReaderState();
            }
        }
    }
    return nextState;
}
 

public ParserState parse(StreamTokenizer st) throws IOException
{
    ParserState nextState = null;
    while(nextState == null && st.ttype != StreamTokenizer.TT_EOF)
    {
        int nextToken = st.nextToken();
        if (nextToken != StreamTokenizer.TT_EOL && nextToken != StreamTokenizer.TT_EOF)
        {
            if (nextToken == StreamTokenizer.TT_WORD && st.sval.equals(AbstractMithraDataFileParser.CLASS_IDENTIFIER))
            {
                nextState = this.getParser().getClassReaderState();
            }
            else
            {
                nextState = this.getParser().getDataReaderState();
            }
        }
    }
    return nextState;
}
 

private static void test(StreamTokenizer st) throws Exception {
    st.eolIsSignificant(true);
    int tt = st.nextToken();
    if (tt != StreamTokenizer.TT_WORD) fail("expected TT_WORD");
    if (!st.sval.equals("foo")) fail("expected word token \"foo\"");
    tt = st.nextToken();
    if (tt != StreamTokenizer.TT_EOL) fail("expected TT_EOL");
}
 

/** Delivers the next token and checks for an unexpected end of line or file. */
public static void getNextToken(StreamTokenizer tokenizer) throws IOException {
	if (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
		throw new IOException("unexpected end of line " + tokenizer.lineno());
	}

	if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
		throw new IOException("unexpected end of file in line " + tokenizer.lineno());
	} else if (tokenizer.ttype == '\'' || tokenizer.ttype == '"') {
		tokenizer.ttype = StreamTokenizer.TT_WORD;
	} else if (tokenizer.ttype == StreamTokenizer.TT_WORD && tokenizer.sval.equals("?")) {
		tokenizer.ttype = '?';
	}
}
 

/** Skips all tokens before next end of line (EOL). */
public static void waitForEOL(StreamTokenizer tokenizer) throws IOException {
	// skip everything until EOL
	while (tokenizer.nextToken() != StreamTokenizer.TT_EOL) {
	}
	;
	tokenizer.pushBack();
}
 

/**
 * Gets next token, skipping empty lines.
 *
 * @throws IOException if reading the next token fails
 */
private void getFirstToken(StreamTokenizer tokenizer) throws IOException {
    while (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
    }

    if ((tokenizer.ttype == '\'') || (tokenizer.ttype == '"')) {
        tokenizer.ttype = StreamTokenizer.TT_WORD;
    } else if ((tokenizer.ttype == StreamTokenizer.TT_WORD) && (tokenizer.sval.equals("?"))) {
        tokenizer.ttype = '?';
    }
}
 

/**
 * Gets next token, checking for a premature and of line.
 *
 * @throws IllegalStateException if it finds a premature end of line
 */
private void getNextToken(StreamTokenizer tokenizer) throws IOException, ParseException {
    if (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
        throw new ParseException("premature end of line", tokenizer.lineno());
    }
    if (tokenizer.ttype == StreamTokenizer.TT_EOF) {
        throw new ParseException(PREMATURE_END_OF_FILE, tokenizer.lineno());
    } else if ((tokenizer.ttype == '\'') || (tokenizer.ttype == '"')) {
        tokenizer.ttype = StreamTokenizer.TT_WORD;
    } else if ((tokenizer.ttype == StreamTokenizer.TT_WORD) && (tokenizer.sval.equals("?"))) {
        tokenizer.ttype = '?';
    }
}
 

BenchInfo parseBenchInfo(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight = parseBenchWeight(tokens);
    String name = parseBenchName(tokens);
    Benchmark bench = parseBenchClass(tokens);
    String[] args = parseBenchArgs(tokens);
    if (tokens.ttype == StreamTokenizer.TT_EOL)
        tokens.nextToken();
    return new BenchInfo(bench, name, weight, args);
}
 

@Override
public ParserState parse(StreamTokenizer st) throws IOException, ParseException
{
    int token = st.ttype;
    if (token != StreamTokenizer.TT_WORD || !st.sval.equals(ExpectedResultsParser.SECTION_IDENTIFIER))
    {
        throw new ParseException("expected line " + st.lineno() + " to begin with Section", st.lineno());
    }
    token = st.nextToken();
    if (token != StreamTokenizer.TT_WORD && token != '"')
    {
        throw new ParseException("expected a section name on line " + st.lineno(), st.lineno());
    }
    String testName = st.sval;

    token = st.nextToken();
    String tableName = null;
    if (token == StreamTokenizer.TT_WORD || token == '"')
    {
        tableName = st.sval;
        token = st.nextToken();
    }

    this.getParser().startNewSection(testName, tableName);

    if (token != StreamTokenizer.TT_EOL)
    {
        throw new ParseException("invalid data after the class name on line " + st.lineno(), st.lineno());
    }

    return this.getParser().getHeaderState();
}
 

public String toString() {
    StringBuilder sb = new StringBuilder();
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        sb.append("ttype=TT_EOL");
        break;
    case StreamTokenizer.TT_EOF:
        sb.append("ttype=TT_EOF");
        break;
    case StreamTokenizer.TT_NUMBER:
        sb.append("ttype=TT_NUM,").append("nval="+nval);
        break;
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            sb.append("ttype=TT_WORD:IDENTIFIER");
        } else {
            sb.append("ttype=TT_WORD:").append("sval="+sval);
        }
        break;
    default:
        if (ttype == (int)'"') {
            sb.append("ttype=TT_STRING:").append("sval="+sval);
        } else {
            sb.append("ttype=TT_CHAR:").append((char)ttype);
        }
        break;
    }
    return sb.toString();
}
 

private void parseAttributes(StreamTokenizer st) throws IOException, ParseException
{
    int token = st.nextToken();
    boolean wantAttribute = true;
    while (token != StreamTokenizer.TT_EOL)
    {
        if (wantAttribute)
        {
            if (token != StreamTokenizer.TT_WORD && token != QUOTE_TOKEN)
            {
                throw new ParseException("expected an column name on line " + st.lineno(), st.lineno());
            }
            this.getParser().getExpectedTable().addColumnHeader(st.sval);
        }
        else
        {
            if (token != ',')
            {
                throw new ParseException("Expected a comma on line " + st.lineno(), st.lineno());
            }
        }
        wantAttribute = !wantAttribute;
        token = st.nextToken();
    }
    if (wantAttribute)
    {
        throw new ParseException("extra comma at the end of line " + st.lineno(), st.lineno());
    }
}
 

/**
 * Gets token.
 * 
 * @param tokenizer the stream tokenizer
 * @throws IOException if reading the next token fails
 */
public static void getToken(StreamTokenizer tokenizer) throws IOException {

  tokenizer.nextToken();
  if (tokenizer.ttype == StreamTokenizer.TT_EOL) {
    return;
  }

  if ((tokenizer.ttype == '\'') || (tokenizer.ttype == '"')) {
    tokenizer.ttype = StreamTokenizer.TT_WORD;
  } else if ((tokenizer.ttype == StreamTokenizer.TT_WORD)
      && (tokenizer.sval.equals("?"))) {
    tokenizer.ttype = '?';
  }
}