Java LexerInput类代码示例

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
public Token<TestTokenId> nextToken() {
    if (input.read() == LexerInput.EOF) {
        return null;
    }
    
    input.read();

    if (input.readText().toString().startsWith("%%")) {
        readUntil("\n");
        
        return factory.createToken(TestTokenId.METADATA);
    }

    if (readUntil("\n%%")) {
        input.backup(2);
    }

    return factory.createToken(TestTokenId.JAVA_CODE);
}
 

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
private Token<DTDTokenId> processCharacterReference() {
    int ch = input.read();
    boolean hex = ch == 'x';
    if (hex) {
        ch = input.read();
    }
    boolean first = true;
    do {
        if (ch == ';') {
            break;
        }
        if (!((ch >= '0' && ch <= '9') ||
             hex && ((ch >= 'A' && ch <= 'F') || (ch >= 'a' && ch <= 'f')))) {
            return null;
        }
        first = false;
    } while ((ch = input.read()) != LexerInput.EOF);
    return createReferenceToken(first ? DTDTokenId.ERROR : DTDTokenId.CHARACTER);
}
 

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
private Token<DTDTokenId> processParsedEntity() {
    int ch;
    boolean first = true;
    while ((ch = input.read()) != LexerInput.EOF) {
        if (ch == ';') {
            return first ? null : tokenFactory.createToken(DTDTokenId.REFERENCE);
        }
        if (!((first && Character.isLetter(ch)) || 
                (!first && (
                    ch == '-' || ch == '.' || Character.isLetterOrDigit(ch))))) {
            // not an entity reference, fall back to normal text processing:
            break;
        } 
        first = false;
    }
    return null;
}
 

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
@Override
public Token<DTDTokenId> nextToken() {
    switch (state) {
        case ISI_INIT:
            return nextTokenInit();
        case ISI_COMMENT:
            return skipComment();
        case ISI_PROCESSING_INSTR:
            return nextProcessingInstr();
        case ISI_DECLARATION:
            return nextDeclaration();
        case ISI_ELEMENT:
        case ISI_ATTLIST:
            return processElementOrAttlist();
        case ISI_ENTITY:
        case ISI_NOTATION:
            break;
    }
    int ch = input.read();
    if (ch != LexerInput.EOF) {
        return error();
    } else {
        return null;
    }
}
 

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
private Token<DTDTokenId> stringvalue(int delimiter) {
    int ch;

    while ((ch = input.read()) != LexerInput.EOF) {
        if (ch == delimiter) {
            restoreState();
            return tokenFactory.createToken(DTDTokenId.STRING);
        }
        // entity reference in attribute !
        if (ch == '&') {    // NOI18N
            intrSubstate = saveState();
            substate = delimiter == '"' ? SUB_VALUE_DOUBLE : SUB_VALUE_QUOTE;
            return valueEntityReference();
        } else if (ch == '<') { // NOI18N
            // ouch ! unterminated value - go to the basic state.
            input.backup(1);
            break;
        }
    }
    if (input.readLength() > 0) {
        return tokenFactory.createToken(DTDTokenId.STRING);
    } 
    setState(ISI_INIT);
    return error();
}
 

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
/**
 * Skips whitespaces within declaration, produces DTDToken if whitespace
 * is found. Does not change state / substate.
 * @return 
 */
private Token<DTDTokenId> skipWhitespace() {
    int ch;
    int start = input.readLength();
    while ((ch = input.read()) != LexerInput.EOF) {
        if (!Character.isWhitespace(ch)) {
            input.backup(1);
            if ((input.readLength() - start) > 0) {
                return tokenFactory.createToken(DTDTokenId.WS, input.readLength() - start);
            }
            break;
        }
    }
    if ((input.readLength() - start) > 0) {
        return tokenFactory.createToken(DTDTokenId.WS, input.readLength() - start);
    } else {
        return null;
    }
}
 

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
private Token<DTDTokenId> processName() {
    int ch;
    boolean first = true;
    while ((ch = input.read()) != LexerInput.EOF) {
        if (isNametokenChar(ch, first)) {
            first = false;
        } else {
            input.backup(1);
            break;
        }
    }
    if (input.readLength() > 0) {
        // output a NAME token
        return tokenFactory.createToken(DTDTokenId.NAME);
    } else if (ch == '?') {
        return endProcessingInstruction();
    } else {
        return error();
    }
}
 

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
public Token<ATokenId> nextToken () {
    LexerInput lexerInput = info.input ();
    int i = lexerInput.read ();
    if (i == ' ') {
        do {
            i = lexerInput.read ();
        } while (i == ' ');
        if (i != LexerInput.EOF) {
            lexerInput.backup (1);
        }
        return info.tokenFactory ().createToken (tokenIds.get (0));
    } else {
        if (i == LexerInput.EOF) {
            return null;
        } else {
            do {
                i = lexerInput.read ();
            } while (i != ' ' && i != LexerInput.EOF);
            if (i != LexerInput.EOF) {
                lexerInput.backup (1);
            }
            return info.tokenFactory ().createToken (tokenIds.get (1));
        }
    }
}
 

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
public int read(int index) { // index >= 0 is guaranteed by contract
    index += tokenStartIndex;
    if (index < readEndIndex()) {
        // Check whether the char is preprocessed
        int rls = preprocessedText.rawLengthShift(index);
        if (rls != lastRawLengthShift) { // Preprocessed
            lastRawLengthShift = rls;
            if (prepStartIndex >= index) { // prepStartIndex already inited
                prepStartIndex = index;
            }
            prepEndIndex = index + 1;
        }
        return preprocessedText.charAt(index);
    } else { // must read next or return EOF
        return LexerInput.EOF;
    }
}
 

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
/**
 * Read next char or return EOF.
 */
int read(int offset) {
    offset += readOffsetShift;
    if (offset < tokenListEndOffset) {
        if (offset >= tokenListStartOffset) {
            return inputSourceText.charAt(offset);
        } else { // Goto previous
            while (true) { // Char should exist
                offset -= movePreviousTokenList();
                if (offset >= tokenListStartOffset) { // ETL might be empty
                    return inputSourceText.charAt(offset);
                }
            }
        }
    } else { // offset >= tokenListEndOffset
        while (tokenListIndex + 1 < tokenListCount()) {
            offset += moveNextTokenList();
            if (offset < tokenListEndOffset) { // ETL might be empty
                return inputSourceText.charAt(offset);
            }
        }
        return LexerInput.EOF;
    }
}
 

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
public int read() {
        // Check whether any characters need to be preprocessed first
        if (readIndex == lookaheadIndex) {
            // Most typical situation - preprocess char
//            LexerSpiPackageAccessor.get().preprocessChar(preprocessor);
            readIndex++;
            // Expect only a single char to be put into lastOutputChar
            if (readIndex == lookaheadIndex) {
                return lastOutputChar;
            } else { // possibly more chars processed or EOF found etc.
                readIndex--;
                // Check whether EOF was processed (returned)
                if (readIndex == lookaheadIndex && lastOutputChar == LexerInput.EOF) {
                    return LexerInput.EOF;
                }
            }
            
        }
        
        return readExisting(readIndex++);
    }
 

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
/**
 * Check that there are no more characters to be read from the given
 * lexer input operation.
 */
private void checkLexerInputFinished() {
    if (read() != LexerInput.EOF || readLength() > 0) {
        StringBuilder sb = new StringBuilder(100);
        int readLen = readLength();
        sb.append("Lexer ").append(lexer); // NOI18N
        sb.append("\n  returned null token but lexerInput.readLength()="); // NOI18N
        sb.append(readLen);
        sb.append("\n  lexer-state: ").append(lexer.state());
        sb.append("\n  ").append(this); // NOI18N
        sb.append("\n  Chars: \"");
        for (int i = 0; i < readLen; i++) {
            sb.append(CharSequenceUtilities.debugChar(readExistingAtIndex(i)));
        }
        sb.append("\" - these characters need to be tokenized."); // NOI18N
        sb.append("\nFix the lexer to not return null token in this state."); // NOI18N
        throw new IllegalStateException(sb.toString());
    }
}
 

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
public FTLLexer(LexerRestartInfo<FTLTokenId> info) {
    this.info = info;
    final LexerInput input = info.input();
    InputStream istream = new InputStream() {

        @Override
        public int read() throws IOException {
            int result = input.read();
            //debug("read " + result);
            //if (result == LexerInput.EOF) {
            //    throw new IOException("LexerInput EOF");
            //}
            return result;
        }
    };
    SimpleCharStream stream = new SimpleCharStream(istream);
    fmParserTokenManager = new FMParserWSTokenManager(stream);
}
 

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
@SuppressWarnings("fallthrough")
private final void readRemainingLine() {
    
    int character = input.read();
    
    while(character != LexerInput.EOF) {
        switch (character) {
            case '\r':
                input.consumeNewline();
            case '\n':
            case LexerInput.EOF:
                return;
        }
        character = input.read();
    }
    
}
 

import org.netbeans.spi.lexer.LexerInput; //导入依赖的package包/类
private boolean readUntil(String condition) {
    int read;

    while ((read = input.read()) != LexerInput.EOF && !input.readText().toString().endsWith(condition))
        ;

    return read != LexerInput.EOF;
}