Java StreamTokenizer.TT_WORD属性代码示例

String[] parseBenchArgs(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    Vector vec = new Vector();
    for (;;) {
        switch (tokens.ttype) {
            case StreamTokenizer.TT_EOF:
            case StreamTokenizer.TT_EOL:
                return (String[]) vec.toArray(new String[vec.size()]);

            case StreamTokenizer.TT_WORD:
            case '"':
                vec.add(tokens.sval);
                tokens.nextToken();
                break;

            default:
                throw new ConfigFormatException("unrecognized arg token " +
                        "on line " + tokens.lineno());
        }
    }
}
 

String parseBenchName(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    String name;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            name = tokens.sval;
            tokens.nextToken();
            return name;

        default:
            throw new ConfigFormatException("missing benchmark name on " +
                    "line " + tokens.lineno());
    }
}
 

float parseBenchWeight(StreamTokenizer tokens)
    throws IOException, ConfigFormatException
{
    float weight;
    switch (tokens.ttype) {
        case StreamTokenizer.TT_WORD:
        case '"':
            try {
                weight = Float.parseFloat(tokens.sval);
            } catch (NumberFormatException e) {
                throw new ConfigFormatException("illegal weight value \"" +
                        tokens.sval + "\" on line " + tokens.lineno());
            }
            tokens.nextToken();
            return weight;

        default:
            throw new ConfigFormatException("missing weight value on line "
                    + tokens.lineno());
    }
}
 

public String toMessage() {
    switch(ttype) {
    case StreamTokenizer.TT_EOL:
        return "\"EOL\"";
    case StreamTokenizer.TT_EOF:
        return "\"EOF\"";
    case StreamTokenizer.TT_NUMBER:
        return "NUMBER";
    case StreamTokenizer.TT_WORD:
        if (sval == null) {
            return "IDENTIFIER";
        } else {
            return "IDENTIFIER " + sval;
        }
    default:
        if (ttype == (int)'"') {
            String msg = "QUOTED STRING";
            if (sval != null)
                msg = msg + " \"" + sval + "\"";
            return msg;
        } else {
            return "CHARACTER \'" + (char)ttype + "\'";
        }
    }
}
 

Token next() throws IOException {
  int type = tok.nextToken();
  switch (type) {
    case StreamTokenizer.TT_EOF:
    case StreamTokenizer.TT_EOL:
      return null;
    case StreamTokenizer.TT_NUMBER:
      return new NumToken(tok.nval);
    case StreamTokenizer.TT_WORD:
      return new StrToken(TType.IDENT, tok.sval);
    case '"':
      return new StrToken(TType.QUOT, tok.sval);
    default:
      switch (type) {
        case ',':
          return new Token(TType.COMMA);
        case '(':
          return new Token(TType.LPAREN);
        case ')':
          return new Token(TType.RPAREN);
        default:
          throw new IOException("Unexpected: " + type);
      }
  }
}
 

/**
        * Is the current token an indicator of a type specification? E.g., see TypeSpec.isaModeSpec for the full list.
        *
        * @return Whether the current token is a type specification.
        * @throws IOException
        */
private boolean atTypeSpec() throws IOException {
	int tokenRead = tokenizer.ttype();
	if (tokenRead == '+' || tokenRead == '-') {
		if (tokenizer.prevToken() == '\\') { return false; } // Currently at the ISO standard, but semi-weirdly named, predicate '\+'.
		// If '+' or '-' need to see if the next item is a string of digits.
		int nextToken  = getNextToken();
		if (nextToken == StreamTokenizer.TT_WORD && isAllDigits(tokenizer.sval())) {  // This '+' or '-' is a sign in front of some digits.
			tokenizer.pushBack();
			return false;
		}
		if (nextToken == StreamTokenizer.TT_WORD && atInfinity()) { // Have +inf or -inf, which is not a type spec for type 'inf'.
			tokenizer.pushBack();
			return false;
		}
		tokenizer.pushBack();
		return true;
	}
	return TypeSpec.isaModeSpec((char) tokenRead);
}
 

/**
 * Parses the next number in the stream. Numbers with exponents are handled.
 * 
 * @param tokenizer tokenizer over a stream of text in Well-known Text
 *        format. The next token must be a number.
 * @return the next number in the stream
 * @throws ParseException if the next token is not a valid number
 * @throws IOException if an I/O error occurs
 */
private double getNextNumber() throws IOException, ParseException {
    int type = tokenizer.nextToken();
    switch (type) {
    case StreamTokenizer.TT_WORD: {
        try {
            return Double.parseDouble(tokenizer.sval);
        }
        catch (NumberFormatException ex) {
            throw new ParseException("Invalid number: " + tokenizer.sval);
        }
    }
    }
    parseError("number");
    return 0.0;
}
 

private static HollowListSchema parseListSchema(String typeName, StreamTokenizer tokenizer) throws IOException {
    int tok = tokenizer.nextToken();

    if(tokenizer.ttype != '<')
        throw new IOException("Invalid Syntax: Expected '<' after 'List' for type " + typeName);

    tok = tokenizer.nextToken();
    if(tok != StreamTokenizer.TT_WORD) {
        log.warning("Invalid Syntax: Expected element type declaration: " + typeName);
    }

    String elementType = tokenizer.sval;

    tok = tokenizer.nextToken();
    if(tokenizer.ttype != '>')
        throw new IOException("Invalid Syntax: Expected '>' element type declaration: " + typeName);

    tok = tokenizer.nextToken();
    if(tokenizer.ttype != ';')
        throw new IOException("Invalid Syntax: Expected semicolon after List schema declaration: " + typeName);

    return new HollowListSchema(typeName, elementType);
}
 

/**
 * Parses the next number in the stream.
 * Numbers with exponents are handled.
 * <tt>NaN</tt> values are handled correctly, and
 * the case of the "NaN" symbol is not significant.
 *
 * @param tokenizer tokenizer over a stream of text in Well-known Text
 * format. The next token must be a number.
 * @return the next number in the stream
 * @throws ParseException if the next token is not a valid number
 * @throws IOException if an I/O error occurs
 */
private double getNextNumber() throws IOException,
        ParseException {
    int type = this.tokenizer.nextToken();
    switch (type) {
        case StreamTokenizer.TT_WORD: {
            if (this.tokenizer.sval.equalsIgnoreCase(NAN_SYMBOL)) {
                return Double.NaN;
            } else {
                try {
                    return Double.parseDouble(this.tokenizer.sval);
                } catch (NumberFormatException ex) {
                    this.parseErrorWithLine("Invalid number: " + this.tokenizer.sval);
                }
            }
        }
    }
    this.parseErrorExpected("number");
    return 0.0;
}
 

/**
 * Read an int or codepoint - codepoint is given as a string
 */
private int getIntOrCP() throws IOException
{
	int val = tkn.nextToken();
	if (val == StreamTokenizer.TT_NUMBER)
	{
		return new Double(tkn.nval).intValue();
	}
	else if (val == StreamTokenizer.TT_WORD)
	{
		return decodeCP(tkn.sval);
	}
	else
	{
		fail("Expecting number, got " + tkn.sval + " on line "
						   + tkn.lineno());
		System.exit(1);
	}
	return 0;
}
 

/**
 * Process something like:  cost: p/2, 1.5;
 * Such costs can be used when scoring clauses (default cost is 1.0).
 * @throws ParsingException
 */
private void processCost(boolean isFinal) throws ParsingException, IOException {
	// Have already read the 'cost:"
	int tokenRead = checkForPredicateNamesThatAreCharacters(getNextToken());

	if (tokenRead == StreamTokenizer.TT_WORD) {
		String currentWord = tokenizer.sval();
		PredicateName pName = stringHandler.getPredicateName(currentWord);
		tokenRead = getNextToken();
		if (tokenRead != '/') { throw new ParsingException("Expecting a '/' (slash) in an 'cost' specification, but got: '" + reportLastItemRead() + "'."); }
		int arity = readInteger();

		checkAndConsume(','); // OK if there is a commas separating the items.

		tokenRead = getNextToken();
		double cost = processNumber(tokenRead);

		if (debugLevel > 1) { Utils.println("cost: " + pName + "/" + arity + ", " + cost); }
		pName.setCost(arity, cost, isFinal);
		return;
	}
	throw new ParsingException("Expecting the name of a predicate in a 'cost' but read: '" + reportLastItemRead() + "'.");
}
 

/**
 * Print failure message and skip to the next test
	 *
 * @exception 	IOException 	error reading file
 */
private void fail(String msg) throws IOException
{
	System.out.println("FAILED - " + msg + " in line " + tkn.lineno());
	// skip remainder of the test look for endtest or end of file
	int val = tkn.nextToken();
	while (val != StreamTokenizer.TT_EOF)
	{
		if (val == StreamTokenizer.TT_WORD && tkn.sval.toLowerCase(Locale.ENGLISH).equals("endtest"))
			break;

		val = tkn.nextToken();
	}
	failed = true;
	// get ready for next test
	reset();
	// print out stack trace so we know where the failure occurred
	Exception e = new Exception();
	e.printStackTrace();
}
 

private static List<String> tokenize(String sentence) {
    List<String> tokens = new ArrayList<>();
    StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(sentence));

    try {
        while (tokenizer.nextToken() != StreamTokenizer.TT_EOF) {
            if (tokenizer.ttype == StreamTokenizer.TT_WORD) {
                tokens.add(tokenizer.sval);
            }
        }
        return tokens;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

/**
 * Gets next token, skipping empty lines.
 *
 * @throws IOException if reading the next token fails
 */
private void getFirstToken(StreamTokenizer tokenizer) throws IOException {
    while (tokenizer.nextToken() == StreamTokenizer.TT_EOL) {
    }

    if ((tokenizer.ttype == '\'') || (tokenizer.ttype == '"')) {
        tokenizer.ttype = StreamTokenizer.TT_WORD;
    } else if ((tokenizer.ttype == StreamTokenizer.TT_WORD) && (tokenizer.sval.equals("?"))) {
        tokenizer.ttype = '?';
    }
}
 

@Override
public ParserState parse(StreamTokenizer st) throws IOException, ParseException
{
    int token = st.ttype;
    if (token != StreamTokenizer.TT_WORD || !st.sval.equals(ExpectedResultsParser.SECTION_IDENTIFIER))
    {
        throw new ParseException("expected line " + st.lineno() + " to begin with Section", st.lineno());
    }
    token = st.nextToken();
    if (token != StreamTokenizer.TT_WORD && token != '"')
    {
        throw new ParseException("expected a section name on line " + st.lineno(), st.lineno());
    }
    String testName = st.sval;

    token = st.nextToken();
    String tableName = null;
    if (token == StreamTokenizer.TT_WORD || token == '"')
    {
        tableName = st.sval;
        token = st.nextToken();
    }

    this.getParser().startNewSection(testName, tableName);

    if (token != StreamTokenizer.TT_EOL)
    {
        throw new ParseException("invalid data after the class name on line " + st.lineno(), st.lineno());
    }

    return this.getParser().getHeaderState();
}