Java LexerInput.EOF属性代码示例

/**
 * Check that there are no more characters to be read from the given
 * lexer input operation.
 */
private void checkLexerInputFinished() {
    if (read() != LexerInput.EOF || readLength() > 0) {
        StringBuilder sb = new StringBuilder(100);
        int readLen = readLength();
        sb.append("Lexer ").append(lexer); // NOI18N
        sb.append("\n  returned null token but lexerInput.readLength()="); // NOI18N
        sb.append(readLen);
        sb.append("\n  lexer-state: ").append(lexer.state());
        sb.append("\n  ").append(this); // NOI18N
        sb.append("\n  Chars: \"");
        for (int i = 0; i < readLen; i++) {
            sb.append(CharSequenceUtilities.debugChar(readExistingAtIndex(i)));
        }
        sb.append("\" - these characters need to be tokenized."); // NOI18N
        sb.append("\nFix the lexer to not return null token in this state."); // NOI18N
        throw new IllegalStateException(sb.toString());
    }
}
 

public Token<ATokenId> nextToken () {
    LexerInput lexerInput = info.input ();
    int i = lexerInput.read ();
    if (i == ' ') {
        do {
            i = lexerInput.read ();
        } while (i == ' ');
        if (i != LexerInput.EOF) {
            lexerInput.backup (1);
        }
        return info.tokenFactory ().createToken (tokenIds.get (0));
    } else {
        if (i == LexerInput.EOF) {
            return null;
        } else {
            do {
                i = lexerInput.read ();
            } while (i != ' ' && i != LexerInput.EOF);
            if (i != LexerInput.EOF) {
                lexerInput.backup (1);
            }
            return info.tokenFactory ().createToken (tokenIds.get (1));
        }
    }
}
 

public int read(int index) { // index >= 0 is guaranteed by contract
    index += tokenStartIndex;
    if (index < readEndIndex()) {
        // Check whether the char is preprocessed
        int rls = preprocessedText.rawLengthShift(index);
        if (rls != lastRawLengthShift) { // Preprocessed
            lastRawLengthShift = rls;
            if (prepStartIndex >= index) { // prepStartIndex already inited
                prepStartIndex = index;
            }
            prepEndIndex = index + 1;
        }
        return preprocessedText.charAt(index);
    } else { // must read next or return EOF
        return LexerInput.EOF;
    }
}
 

private Token<DTDTokenId> processParsedEntity() {
    int ch;
    boolean first = true;
    while ((ch = input.read()) != LexerInput.EOF) {
        if (ch == ';') {
            return first ? null : tokenFactory.createToken(DTDTokenId.REFERENCE);
        }
        if (!((first && Character.isLetter(ch)) || 
                (!first && (
                    ch == '-' || ch == '.' || Character.isLetterOrDigit(ch))))) {
            // not an entity reference, fall back to normal text processing:
            break;
        } 
        first = false;
    }
    return null;
}
 

/**
 * Skips whitespaces within declaration, produces DTDToken if whitespace
 * is found. Does not change state / substate.
 * @return 
 */
private Token<DTDTokenId> skipWhitespace() {
    int ch;
    int start = input.readLength();
    while ((ch = input.read()) != LexerInput.EOF) {
        if (!Character.isWhitespace(ch)) {
            input.backup(1);
            if ((input.readLength() - start) > 0) {
                return tokenFactory.createToken(DTDTokenId.WS, input.readLength() - start);
            }
            break;
        }
    }
    if ((input.readLength() - start) > 0) {
        return tokenFactory.createToken(DTDTokenId.WS, input.readLength() - start);
    } else {
        return null;
    }
}
 

@SuppressWarnings("fallthrough")
private final void readRemainingLine() {
    
    int character = input.read();
    
    while(character != LexerInput.EOF) {
        switch (character) {
            case '\r':
                input.consumeNewline();
            case '\n':
            case LexerInput.EOF:
                return;
        }
        character = input.read();
    }
    
}
 

private int read() {
    int result = input.read();
    if (result == LexerInput.EOF) {
        result = CharStream.EOF;
    }

    return result;
}
 

private Token<DTDTokenId> processDeclarationName() {
    int ch;
    boolean first = true;
    while ((ch = input.read()) != LexerInput.EOF) {
        if (first && (ch == '%') && (state == ISI_ENTITY)) {
            int n = input.read();
            if (Character.isWhitespace(n)) {
                // ENTITY % whatever. Emit OPERATOR
                return tokenFactory.createToken(DTDTokenId.OPERATOR);
            }
            input.backup(1);
        }
        if (ch == '%' || ch == '&') {
            Token<DTDTokenId> t = processEntityOrCharacterRef();
            substate = SUB_DECLARATION_DEF;
            if (t != null) {
                return t;
            } else {
                return error();
            }
        }
        if (!isNametokenChar(ch, first)) {
            if (first) {
                substate = SUB_DECLARATION_DEF;
                return error();
            }
            input.backup(1);
            break;
        }
        first = false;
    }
    substate = SUB_DECLARATION_DEF;
    return tokenFactory.createToken(DTDTokenId.NAME);
}
 

@Override
public char readChar() throws IOException {
    int read = input.read();

    if (read == LexerInput.EOF) {
        throw new IOException("EOF reached.");
    }
    return (char) read;
}
 

private Token<DTDTokenId> nextProcessingContent() {
    int ch;
    boolean white = false;
    int whiteStart = -1;
    
    while ((ch = input.read()) != LexerInput.EOF) {
        if (ch == '?') {
            if (whiteStart > 0) {
                return tokenFactory.createToken(DTDTokenId.PI_CONTENT, whiteStart);
            }
            if (input.readLength() > 1) {
                input.backup(1);
                return tokenFactory.createToken(DTDTokenId.PI_CONTENT);
            }
            ch = input.read();
            if (ch == '>') {
                return tokenFactory.createToken(DTDTokenId.SYMBOL);
            }
            // exit the processing content
            setState(ISI_INIT);
            return tokenFactory.createToken(DTDTokenId.ERROR);
        }
        if (Character.isWhitespace(ch)) {
            if (whiteStart == -1) {
                whiteStart = input.readLength();
            }
            if (input.readLength() == 1) {
                white = true;
            }
        } else if (white) {
            // report whitespace & continue lexing in the same state:
            return tokenFactory.createToken(DTDTokenId.WS);
        }
    }
    return tokenFactory.createToken(DTDTokenId.PI_CONTENT);
}
 

@Override
public Token<CPTokenId> nextToken() {
    //just read whole input
    while(input.read() != LexerInput.EOF) {};
    //and create one big token
    return input.readLength() > 0 
            ? tokenFactory.createToken(CPTokenId.CSS) 
            : null;
}
 

@Override
public char BeginToken() throws IOException {
    int chr = input.read();
    if (chr == LexerInput.EOF) {
        throw new IOException();
    }
    return (char) chr;
}
 

private Token<JShellToken>  commandLine() {
    int c = input.read();
    if (c == LexerInput.EOF) {
        return null;
    }
    if (Character.isWhitespace(c)) {
        boolean endsNonWhite = eatWhitespace();
        if (!endsNonWhite) {
            input.backup(1);
        } else {
            setState(S.INITIAL);
        }
        return blockToken(JShellToken.WHITESPACE);
    }
    int quote = 0;
    if (c == '\'' || c == '"') {
        quote = c;
        c = input.read();
    }
    
    if (c == '-') {
        // dash-option
        return readParameter(JShellToken.COMMAND_OPTION, quote);
    } else {
        return readParameter(quote == 0  ? JShellToken.COMMAND_PARAM : JShellToken.COMMAND_STRING, quote);
    }
}
 

private Token<JShellToken> readParameter(JShellToken tokenId, int quote) {
    int c = input.read();
    boolean verbatim = false;
    
    while (true) {
        if (c == LexerInput.EOF) {
            // only partial token
            state = S.INITIAL;
            return tokenFactory.createToken(tokenId, input.readLength(), PartType.START);
        }
        if (c == '\\') { // NOI18N
            // next character verbatim
            verbatim = true;
            c = input.read();
            continue;
        }
        if (!verbatim && c == quote) {
            break;
        }
        if (!verbatim && quote == 0) {
            if (Character.isWhitespace(c)) {
                input.backup(1);
                break;
            }
        } else if (c == '\n') { // NOI18N
            // unterminated quote
            input.backup(1);
            state = S.INITIAL;
            return tokenFactory.createToken(tokenId, input.readLength(), PartType.START);
        }
        verbatim = false;
        c = input.read();
    }
    return tokenFactory.createToken(tokenId, input.readLength());
}
 

private Token<DTDTokenId> nextProcessingTarget() {
    int ch;
    boolean first = true;
    
    while ((ch = input.read()) != LexerInput.EOF) {
        if (ch == '?') { // NOI18N
            if (first) {
                return nextProcessingContent();
            }
            substate = SUB_PROCESSING_CONTENT;
            input.backup(1);
            break;
        } else if (Character.isWhitespace(ch)) {
            if (first) {
                return tokenFactory.createToken(DTDTokenId.ERROR);
            }
            substate = SUB_PROCESSING_CONTENT;
            input.backup(1);
            break;
        }
        if (!((first & Character.isLetter(ch)) ||
            (!first && Character.isLetterOrDigit(ch)))) {
            // apparently an error:
            return tokenFactory.createToken(DTDTokenId.ERROR);
        }
        first = false;
    }
    if ("xml".equalsIgnoreCase(input.readText().toString())) { // NOI18N
        substate = SUB_PROCESSING_XML;
    }
    return tokenFactory.createToken(DTDTokenId.TARGET);
}