Java CommonToken類代碼示例

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
@Override
protected void announce(Token start, Token stop, AbstractElement element) {
	if (start != null && start != Token.EOF_TOKEN) {
		if (start == stop) {
			announce(start, element);
		} else {
			CommonToken castedStart = (CommonToken) start;
			if (stop == null) { // possible error condition
				if (start.getTokenIndex() == state.lastErrorIndex) {
					return;
				}
			}
			CommonToken castedEnd = (CommonToken) stop;
			Integer newType = rewriter.rewrite(castedStart, element);
			if (newType != null && castedEnd != null && castedEnd != Token.EOF_TOKEN) {
				LazyTokenStream castedInput = (LazyTokenStream) this.input;
				for (int i = castedStart.getTokenIndex() + 1; i < castedEnd.getTokenIndex(); i++) {
					Token token = castedInput.get(i);
					if (token.getChannel() != Token.HIDDEN_CHANNEL)
						token.setType(newType);
				}
				castedEnd.setType(newType);
			}
		}
	}
}
 

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
public void test_getTrimmedNodeRange() {
    String source = " hello! ";
    //               012345678
    CommonToken token = new CommonToken(Css3Lexer.IDENT);
    token.setText(source);
    token.setStartIndex(0);
    token.setStopIndex(7); //len - 1 -> points to last char not the end!
    
    Node node = new TokenNode(source, token);
    
    assertEquals(" hello! ", node.image().toString());
    int[] result = NodeUtil.getTrimmedNodeRange(node);

    assertEquals(1, result[0]);
    assertEquals(7, result[1]);
    
}
 

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
/** Creates a qualified name tree by extending an existing one
 * with an additional fragment in front.
 * @param init token holding the additional level text
 * @param subTree the existing tree; may be {@code null}, in which case
 * the result is calculated using {@link #toQualName(Token,Token)} with first
 * argument {@code null}
 */
CommonTree toQualName(Token init, CtrlTree subTree) {
    CtrlTree result;
    if (subTree == null || this.namespace.hasErrors()) {
        result = toQualName(null, init);
    } else {
        Token subTop = subTree.getToken();
        QualName qualName = subTree.getQualName()
            .nest(getText(init));
        CommonToken top = new CommonToken(subTop.getType(), qualName.toString());
        top.setLine(init.getLine());
        top.setTokenIndex(init.getTokenIndex());
        result = new CtrlTree(top);
        result.setQualName(qualName);
        result.addChild(subTree.getChild(0));
    }
    return result;
}
 

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
/**
 * Tests if the rule name is qualified;
 * if not, first tries to look it up in the import map, and if that fails,
 * prefixes it with the package name.
 */
CtrlTree qualify(CtrlTree ruleNameToken) {
    CtrlTree result = ruleNameToken;
    QualName qualName = ruleNameToken.getQualName();
    if (!this.namespace.hasErrors() && qualName.parent()
        .isTop()) {
        String simpleName = qualName.last();
        Map<String,QualName> importMap = getNamespace().getImportMap();
        if (importMap.containsKey(simpleName)) {
            qualName = importMap.get(simpleName);
        } else if (!isAnyOther(simpleName)) {
            qualName = this.packageName.extend(simpleName);
        }
        CommonToken token = new CommonToken(ruleNameToken.getType(), qualName.toString());
        token.setLine(ruleNameToken.getLine());
        token.setTokenIndex(ruleNameToken.getToken()
            .getTokenIndex());
        result = new CtrlTree(token);
        result.setQualName(qualName);
        result.addChild(ruleNameToken.getChild(0));
    }
    return result;
}
 

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
/** for testing */

    public CompiledST defineTemplate(String templateName, String template) {
        if ( templateName.charAt(0) !='/' ) templateName = "/"+templateName;
        try {
            CompiledST impl = defineTemplate(templateName,
                                             new CommonToken(GroupParser.ID, templateName),
                                             null,
                                             template,
                                             null);
            return impl;
        }
        catch (STException se) {
            // we have reported the error; the exception just blasts us
            // out of parsing this template
        }
        return null;
    }
 

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
/** for testing */

    public CompiledST defineTemplate(String templateName, String template) {
        if ( templateName.charAt(0)!='/' ) templateName = "/"+templateName;
        try {
            CompiledST impl = defineTemplate(templateName,
                                             new CommonToken(GroupParser.ID, templateName),
                                             null,
                                             template,
                                             null);
            return impl;
        }
        catch (STException se) {
            // we have reported the error; the exception just blasts us
            // out of parsing this template
        }
        return null;
    }
 

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
@Test 
	public void testTokenRegionContainsRegion() throws Exception {
		String modelAsString = "a1 / /* comment */ b2";
		List<CommonToken> tokens = getTokens(modelAsString);
		for(int length=0; length < modelAsString.length(); ++length) {
			for(int offset = 0; offset + length < modelAsString.length(); ++offset) {
				ITextRegion tokenRegion = tokenRegionProvider.getTokenRegion(modelAsString, new TextRegion(offset, length));
//				System.out.println(offset + ":" + length + " -> " + tokenRegion);
				CommonToken firstToken = findTokenStartingAt(tokenRegion.getOffset(), tokens);
				assertTrue(firstToken.getStartIndex() <= offset);
				if(tokenRegion.getLength() != 0) {
					CommonToken lastToken = findTokenStopingAt(tokenRegion.getOffset() + tokenRegion.getLength()-1, tokens);
					assertTrue(lastToken.getStopIndex() >= offset + length -1);
				}
			}
		}
	}
 

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
/**
 * Create a new token from the given prototype. Any argument besides the prototype is optional and
 * will be ignored if its value is <code>null</code>.
 */
protected CommonToken createToken(CommonToken prototype, String text, 
		Integer charPosInLine, Integer channel, Integer start, Integer stop, Integer type) {
	if (prototype == null)
		throw new IllegalArgumentException("Prototype may not be null.");
	CommonToken result = new CommonToken(prototype);
	if (text != null)
		result.setText(text);
	if (charPosInLine != null)
		result.setCharPositionInLine(charPosInLine.intValue());
	if (channel != null)
		result.setChannel(channel.intValue());
	if (start != null)
		result.setStartIndex(start.intValue());
	if (stop != null)
		result.setStopIndex(stop.intValue());
	if (type != null)
		result.setType(type.intValue());
	return result;
}
 

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
/** for testing */
public CompiledST defineTemplate(String templateName, String template) {
	if ( templateName.charAt(0)!='/' ) templateName = "/"+templateName;
	try {
		CompiledST impl =
			defineTemplate(templateName,
						   new CommonToken(GroupParser.ID, templateName),
						   null, template, null);
		return impl;
	}
	catch (STException se) {
		// we have reported the error; the exception just blasts us
		// out of parsing this template
	}
	return null;
}
 

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
private void splitIntoBeginToken(Token token, int indentation, ITokenAcceptor result) {
	result.accept(token);
	if (shouldEmitPendingEndTokens()) {
		Token nextToken = getDelegate().nextToken();
		if (shouldSplitToken(nextToken)) {
			nextOffset = ((CommonToken) token).getStopIndex() + 1;
			doSplitToken(nextToken, result);
		} else {
			indentationStack.push(indentation);
			currentIndentation = indentation;
			result.accept(createBeginToken(((CommonToken) token).getStopIndex() + 1));
			result.accept(nextToken);
		}	
	} else {
		indentationStack.push(indentation);
		currentIndentation = indentation;
		result.accept(createBeginToken(((CommonToken) token).getStopIndex() + 1));
	}
}
 

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
public Token nextToken()
{
    super.nextToken();
    if (tokens.size() == 0)
        return new CommonToken(Token.EOF);
    return tokens.remove(0);
}
 

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
/**
 * The second pass over the given input. If the input ends with an ASI, the semicolon is removed and it is parsed
 * again since the production may have skipped the ASI if more input was present. Same applies for the opposite
 * direction, e.g if it does not end with an ASI but the prev token suggests that there may have been a semicolon,
 * it is inserted.
 */
private void adjustASIAndCollectFollowElements(CustomInternalN4JSParser previousParser, boolean strict,
		Set<FollowElement> result) {
	ObservableXtextTokenStream tokens = (ObservableXtextTokenStream) previousParser.getTokenStream();
	int lastTokenIndex = tokens.size() - 1;
	if (lastTokenIndex >= 0) {
		CommonToken lastToken = (CommonToken) tokens.LT(-1);
		@SuppressWarnings("unchecked")
		List<Token> tokenList = tokens.getTokens();
		if (lastToken == null) {
			return; // TODO ask Sebastian why and how this can happen... it happens!
		}

		if (shouldSkipASI(lastToken)) {
			// Some statements may not span multiple lines, e.g between the return keyword
			// and the returned expression, there may not be an ASI. Filter these locations.
			if (maySkipASI(lastToken, tokens)) {
				tokenList.remove(lastTokenIndex);
				result.addAll(resetAndGetFollowElements(tokens, strict));
				// If a postfix operator sneaked into the result, remove it since
				// we'd have produce an ASI before that
				removePostfixOperator(result);
			}
		} else if (shouldAddSyntheticSemicolon(previousParser, lastTokenIndex, lastToken)) {
			CommonToken token = new CommonToken(semi);
			tokenList.add(token);
			result.addAll(resetAndGetFollowElements(tokens, strict));
			// Same here, if we had added an ASI, the postfix operator would be rendered
			// invalid, remove it.
			removePostfixOperator(result);
		}
	}
}
 

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
/**
 * Prevent ASIs to be skipped at certain locations, e.g. after a return keyword.
 */
private boolean maySkipASI(CommonToken lastToken, ObservableXtextTokenStream tokens) {
	int countDownFrom = lastToken.getTokenIndex();
	for (int i = countDownFrom - 1; i >= 0; i--) {
		Token prevToken = tokens.get(i);
		if (prevToken.getChannel() == Token.DEFAULT_CHANNEL) {
			if (mandatoryASI.get(prevToken.getType())) {
				return false;
			}
			return true;
		}
	}
	return true;
}
 

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
/**
 * Returns true if the ASI at the end should be skipped for a second pass of the CA parser. Otherwise false.
 */
private boolean shouldSkipASI(CommonToken lastToken) {
	if (lastToken.getType() == eol) {
		return true;
	}
	if (lastToken.getType() == semi && lastToken.getText() != null && !";".equals(lastToken.getText())) {
		return true;
	}
	return false;
}
 

import org.antlr.runtime.CommonToken; //導入依賴的package包/類
/**
 * Implementation of the {@link TokenSource} interface. Return new tokens as long as there are some, afterwards
 * return {@link Token#EOF_TOKEN}.
 */
@Override
public Token nextToken() {
	if (next != null) {
		Token result = next;
		next = null;
		return result;
	}
	if (!leafNodes.hasNext()) {
		return Token.EOF_TOKEN;
	}
	ILeafNode leaf = leafNodes.next();
	if (leaf.getTotalOffset() >= endOffset) {
		leafNodes = Iterators.emptyIterator();
		return Token.EOF_TOKEN;
	}
	if (leaf.getTotalEndOffset() <= startOffset) {
		return nextToken();
	}
	if (leaf.getTotalEndOffset() > endOffset) {
		return toPrefixToken(leaf);
	}
	SyntaxErrorMessage syntaxErrorMessage = leaf.getSyntaxErrorMessage();
	if (syntaxErrorMessage != null && SEMICOLON_INSERTED.equals(syntaxErrorMessage.getIssueCode())) {
		return toASIToken(leaf);
	}
	if (leaf.isHidden()) {
		return processHiddenToken(leaf);
	}
	int tokenType = tokenTypeMapper.getInternalTokenType(leaf);
	return new CommonToken(tokenType, leaf.getText());
}