Java ImploderAttachment.getTokenizer方法代码示例

import org.spoofax.jsglr.client.imploder.ImploderAttachment; //导入方法依赖的package包/类
private ICompletion createCompletionReplaceTerm(String name, String text, String additionalInfo,
    StrategoAppl change, boolean blankLineCompletion, String prefix, String suffix) {

    final StrategoTerm oldNode = (StrategoTerm) change.getSubterm(0);
    final StrategoTerm newNode = (StrategoTerm) change.getSubterm(1);

    if(change.getSubtermCount() != 2 || !(newNode instanceof IStrategoAppl)
        || !(oldNode instanceof IStrategoAppl)) {
        return null;
    }

    final String sort = ImploderAttachment.getSort(oldNode);

    int insertionPoint, suffixPoint;

    final ImploderAttachment oldNodeIA = oldNode.getAttachment(ImploderAttachment.TYPE);
    ITokens tokenizer = ImploderAttachment.getTokenizer(oldNode);

    // check if it's an empty node
    if(oldNodeIA.getLeftToken().getStartOffset() > oldNodeIA.getRightToken().getEndOffset()) {
        // get the last non-layout token before the new node
        int tokenPosition =
            oldNodeIA.getLeftToken().getIndex() - 1 > 0 ? oldNodeIA.getLeftToken().getIndex() - 1 : 0;
        while(tokenPosition > 0 && (tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_LAYOUT
            || tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_ERROR))
            tokenPosition--;
        insertionPoint = tokenizer.getTokenAt(tokenPosition).getEndOffset();

        // if completion does not spam multiple lines preserve everything starting at the first non-layout char
        if(!additionalInfo.contains("\n")) {
            tokenPosition = oldNodeIA.getLeftToken().getIndex() + 1 < tokenizer.getTokenCount()
                ? oldNodeIA.getLeftToken().getIndex() + 1 : tokenizer.getTokenCount() - 1;
            while(tokenPosition < tokenizer.getTokenCount()
                && (tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_LAYOUT
                    || tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_ERROR))
                tokenPosition++;
            suffixPoint = tokenizer.getTokenAt(tokenPosition).getStartOffset();
        } else { // if completion spams multiple lines keep the lines
            suffixPoint = insertionPoint + 1;
        }
        // if completion is triggered in an empty line, consume that line
        IToken checkToken;
        boolean blankLine = false;
        if(blankLineCompletion) {
            for(; tokenPosition < tokenizer.getTokenCount(); tokenPosition++) {
                checkToken = tokenizer.getTokenAt(tokenPosition);
                if(tokenizer.toString(checkToken, checkToken).contains("\n")) {
                    suffixPoint = checkToken.getEndOffset();
                    if(!blankLine) {
                        blankLine = true;
                    } else {
                        break;
                    }
                }
            }
        }


    } else { // if not, do a regular replacement
        insertionPoint = oldNodeIA.getLeftToken().getStartOffset() - 1;
        suffixPoint = oldNodeIA.getRightToken().getEndOffset() + 1;
    }

    CompletionKind kind;

    if(prefix.equals("") && suffix.equals("")) {
        kind = CompletionKind.expansion;
    } else {
        kind = CompletionKind.expansionEditing;
    }

    return new Completion(name, sort, text, additionalInfo, insertionPoint + 1, suffixPoint, kind, prefix, suffix);
}
 

import org.spoofax.jsglr.client.imploder.ImploderAttachment; //导入方法依赖的package包/类
private ICompletion createCompletionInsertBefore(String name, String text, String additionalInfo,
    StrategoAppl change) {

    final StrategoTerm oldNode = (StrategoTerm) change.getSubterm(0);
    final StrategoTerm newNode = (StrategoTerm) change.getSubterm(1);


    // expect two terms and 1st should be an element of a list
    final StrategoTerm oldList = (StrategoTerm) ParentAttachment.getParent(oldNode);

    if(change.getSubtermCount() != 2 || !(oldNode instanceof IStrategoAppl) || !(newNode instanceof IStrategoList)
        || !(oldList instanceof IStrategoList)) {
        return null;
    }

    final String sort = ImploderAttachment.getSort(oldNode);

    int insertionPoint, suffixPoint;

    ITokens tokenizer = ImploderAttachment.getTokenizer(oldNode);

    IStrategoTerm[] subterms = oldList.getAllSubterms();
    int indexOfElement;
    for(indexOfElement = 0; indexOfElement < subterms.length; indexOfElement++) {
        if(subterms[indexOfElement] == oldNode)
            break;
    }

    // if inserted element is first (only two elements)
    if(indexOfElement == 0) {
        // insert after the first non-layout token before the leftmost token of the
        // completion node
        final ImploderAttachment oldNodeIA = oldNode.getAttachment(ImploderAttachment.TYPE);
        int tokenPosition =
            oldNodeIA.getLeftToken().getIndex() - 1 > 0 ? oldNodeIA.getLeftToken().getIndex() - 1 : 0;
        while((tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_LAYOUT
            || tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_ERROR) && tokenPosition > 0)
            tokenPosition--;

        insertionPoint = tokenizer.getTokenAt(tokenPosition).getEndOffset();
    } else {
        // if inserted element is not first
        // insert after at end offset of the rightmost token of the element before the
        // completion
        StrategoTerm elementBefore = (StrategoTerm) oldList.getSubterm(indexOfElement - 1);
        insertionPoint = elementBefore.getAttachment(ImploderAttachment.TYPE).getRightToken().getEndOffset();
    }

    // if completion is separated by a newline, preserve indentation of the subsequent node
    // else separation follows from the grammar
    String separator = "";
    for(int i = text.length() - 1; i >= 0; i--) {
        if(text.charAt(i) == additionalInfo.charAt(additionalInfo.length() - 1)) {
            break;
        }
        separator = text.charAt(i) + separator;
    }

    IToken checkToken = oldNode.getAttachment(ImploderAttachment.TYPE).getLeftToken();
    int checkTokenIdx = oldNode.getAttachment(ImploderAttachment.TYPE).getLeftToken().getIndex();
    suffixPoint = insertionPoint;
    if(separator.contains("\n")) {
        for(; checkTokenIdx >= 0; checkTokenIdx--) {
            checkToken = tokenizer.getTokenAt(checkTokenIdx);
            if(tokenizer.toString(checkToken, checkToken).contains("\n")) {
                break;
            }
            suffixPoint = checkToken.getStartOffset();
        }
    } else {
        suffixPoint = checkToken.getStartOffset();
    }

    return new Completion(name, sort, text, additionalInfo, insertionPoint + 1, suffixPoint,
        CompletionKind.expansion);
}
 

import org.spoofax.jsglr.client.imploder.ImploderAttachment; //导入方法依赖的package包/类
private ICompletion createCompletionInsertAtEnd(String name, String text, String additionalInfo,
    StrategoAppl change, boolean blankLineCompletion) {

    final StrategoTerm oldNode = (StrategoTerm) change.getSubterm(0);
    final StrategoTerm newNode = (StrategoTerm) change.getSubterm(1);

    // expected two lists
    if(change.getSubtermCount() != 2 || !(oldNode instanceof IStrategoList)
        || !(newNode instanceof IStrategoList)) {
        return null;
    }

    final String sort = ImploderAttachment.getElementSort(oldNode);

    int insertionPoint, suffixPoint;

    ITokens tokenizer = ImploderAttachment.getTokenizer(oldNode);
    final ImploderAttachment oldListIA = oldNode.getAttachment(ImploderAttachment.TYPE);
    int tokenPosition;
    // if list is empty
    // insert after the first non-layout token before the leftmost token of the completion
    // node
    if(oldNode.getSubtermCount() == 0) {
        tokenPosition = oldListIA.getLeftToken().getIndex() - 1 > 0 ? oldListIA.getLeftToken().getIndex() - 1 : 0;
        while((tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_LAYOUT
            || tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_ERROR) && tokenPosition > 0)
            tokenPosition--;
        insertionPoint = tokenizer.getTokenAt(tokenPosition).getEndOffset();
    } else {
        // if list is not empty
        // insert after at end offset of the rightmost token of the element before the
        // completion
        StrategoTerm elementBefore = (StrategoTerm) oldNode.getSubterm(oldNode.getAllSubterms().length - 1);
        int leftIdx = elementBefore.getAttachment(ImploderAttachment.TYPE).getLeftToken().getIndex();
        int rightIdx = elementBefore.getAttachment(ImploderAttachment.TYPE).getRightToken().getIndex();
        while((tokenizer.getTokenAt(rightIdx).getKind() == IToken.TK_LAYOUT
            || tokenizer.getTokenAt(rightIdx).getLength() == 0) && rightIdx > leftIdx) {
            rightIdx--;
        }
        insertionPoint = tokenizer.getTokenAt(rightIdx).getEndOffset();
        tokenPosition = rightIdx;
    }
    suffixPoint = insertionPoint + 1;

    // if completion is triggered in an empty line, consume that line
    IToken checkToken;
    boolean blankLine = false;
    if(blankLineCompletion) {
        for(; tokenPosition < tokenizer.getTokenCount(); tokenPosition++) {
            checkToken = tokenizer.getTokenAt(tokenPosition);
            if(tokenizer.toString(checkToken, checkToken).contains("\n")) {
                suffixPoint = checkToken.getEndOffset();
                if(!blankLine) {
                    blankLine = true;
                } else {
                    break;
                }
            }
        }
    }

    return new Completion(name, sort, text, additionalInfo, insertionPoint + 1, suffixPoint,
        CompletionKind.expansion);
}
 

import org.spoofax.jsglr.client.imploder.ImploderAttachment; //导入方法依赖的package包/类
private ICompletion createCompletionInsertionTermFixing(String name, String text, String additionalInfo,
    String prefix, String suffix, StrategoAppl change, String completionKind) {
    final StrategoTerm newNode = (StrategoTerm) change.getSubterm(0);


    if(change.getSubtermCount() != 1 || !(newNode instanceof IStrategoAppl)) {
        return null;
    }

    final String sort = ImploderAttachment.getSort(newNode);

    int insertionPoint, suffixPoint;

    ITokens tokenizer = ImploderAttachment.getTokenizer(newNode);

    final StrategoTerm topMostAmb = findTopMostAmbNode(newNode);
    final ImploderAttachment topMostAmbIA = topMostAmb.getAttachment(ImploderAttachment.TYPE);

    // get the last non-layout token before the topmost ambiguity
    int tokenPosition =
        topMostAmbIA.getLeftToken().getIndex() - 1 > 0 ? topMostAmbIA.getLeftToken().getIndex() - 1 : 0;
    while((tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_LAYOUT
        || tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_ERROR) && tokenPosition > 0)
        tokenPosition--;

    insertionPoint = tokenizer.getTokenAt(tokenPosition).getEndOffset();

    if(topMostAmbIA.getRightToken().getEndOffset() < topMostAmbIA.getRightToken().getStartOffset()) {
        // keep all the characters after the last non-layout token if completion ends with a
        // placeholder
        tokenPosition = topMostAmbIA.getRightToken().getIndex();
        while(tokenPosition > 0
            && (tokenizer.getTokenAt(tokenPosition).getEndOffset() < tokenizer.getTokenAt(tokenPosition)
                .getStartOffset() || tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_LAYOUT))
            tokenPosition--;
        suffixPoint = tokenizer.getTokenAt(tokenPosition).getEndOffset() + 1;

    } else {
        // skip all the (erroneous) characters that were in the text already
        suffixPoint = topMostAmbIA.getRightToken().getEndOffset() + 1;
    }

    CompletionKind kind;
    if(completionKind.equals("recovery")) {
        kind = CompletionKind.recovery;
    } else if(completionKind.equals("expansionEditing")) {
        kind = CompletionKind.expansionEditing;
    } else {
        kind = CompletionKind.expansion;
    }

    return new Completion(name, sort, text, additionalInfo, insertionPoint + 1, suffixPoint, kind, prefix, suffix);
}
 

import org.spoofax.jsglr.client.imploder.ImploderAttachment; //导入方法依赖的package包/类
private ICompletion createCompletionInsertBeforeFixing(String name, String text, String additionalInfo,
    String prefix, String suffix, StrategoAppl change, String completionKind) {

    // expect two terms and 1st should be an element of a list
    final StrategoTerm oldNode = (StrategoTerm) change.getSubterm(0);
    final StrategoTerm newNode = (StrategoTerm) change.getSubterm(1);
    final StrategoTerm oldList = (StrategoTerm) ParentAttachment.getParent(oldNode);

    if(change.getSubtermCount() != 2 || !(oldNode instanceof IStrategoAppl) || !(newNode instanceof IStrategoAppl)
        || !(oldList instanceof IStrategoList)) {
        return null;
    }

    final String sort = ImploderAttachment.getSort(oldNode);

    int insertionPoint, suffixPoint;

    IStrategoTerm[] subterms = ((IStrategoList) oldList).getAllSubterms();
    int indexOfCompletion;
    for(indexOfCompletion = 0; indexOfCompletion < subterms.length; indexOfCompletion++) {
        if(subterms[indexOfCompletion] == oldNode)
            break;
    }
    // if inserted element is first (only two elements)
    if(indexOfCompletion == 1) {
        // insert after the first non-layout token before the leftmost token of the list
        ITokens tokenizer = ImploderAttachment.getTokenizer(oldList);

        // to avoid keeping duplicate tokens due to ambiguity
        IStrategoTerm topMostAmbOldList = findTopMostAmbNode(oldList);
        final ImploderAttachment oldListIA = topMostAmbOldList.getAttachment(ImploderAttachment.TYPE);

        int tokenPosition =
            oldListIA.getLeftToken().getIndex() - 1 > 0 ? oldListIA.getLeftToken().getIndex() - 1 : 0;
        while((checkEmptyOffset(tokenizer.getTokenAt(tokenPosition))
            || tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_LAYOUT
            || tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_ERROR) && tokenPosition > 0)
            tokenPosition--;

        insertionPoint = tokenizer.getTokenAt(tokenPosition).getEndOffset();
    } else {
        // if inserted element is not first
        // insert after at end offset of the rightmost token of the element before the completion
        StrategoTerm elementBefore = (StrategoTerm) oldList.getSubterm(indexOfCompletion - 2);
        insertionPoint = elementBefore.getAttachment(ImploderAttachment.TYPE).getRightToken().getEndOffset();

    }
    // suffix point should be the first token of the next element
    suffixPoint = oldNode.getAttachment(ImploderAttachment.TYPE).getLeftToken().getStartOffset();

    CompletionKind kind;
    if(completionKind.equals("recovery")) {
        kind = CompletionKind.recovery;
    } else if(completionKind.equals("expansionEditing")) {
        kind = CompletionKind.expansionEditing;
    } else {
        kind = CompletionKind.expansion;
    }

    return new Completion(name, sort, text, additionalInfo, insertionPoint + 1, suffixPoint, kind, prefix, suffix);
}
 

import org.spoofax.jsglr.client.imploder.ImploderAttachment; //导入方法依赖的package包/类
private ICompletion createCompletionReplaceTermFixing(String name, String text, String additionalInfo,
    String prefix, String suffix, StrategoAppl change, String completionKind) {
    final StrategoTerm oldNode = (StrategoTerm) change.getSubterm(0);
    final StrategoTerm newNode = (StrategoTerm) change.getSubterm(1);


    if(change.getSubtermCount() != 2 || !(newNode instanceof IStrategoAppl)
        || !(oldNode instanceof IStrategoAppl)) {
        return null;
    }

    final String sort = ImploderAttachment.getSort(oldNode);

    int insertionPoint, suffixPoint;

    final ImploderAttachment oldNodeIA = oldNode.getAttachment(ImploderAttachment.TYPE);
    ITokens tokenizer = ImploderAttachment.getTokenizer(oldNode);

    // check if it's an empty node
    if(oldNodeIA.getLeftToken().getStartOffset() > oldNodeIA.getRightToken().getEndOffset()) {
        // get the last non-layout token before the new node
        int tokenPosition =
            oldNodeIA.getLeftToken().getIndex() - 1 > 0 ? oldNodeIA.getLeftToken().getIndex() - 1 : 0;
        while((tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_LAYOUT
            || tokenizer.getTokenAt(tokenPosition).getKind() == IToken.TK_ERROR) && tokenPosition > 0)
            tokenPosition--;
        insertionPoint = tokenizer.getTokenAt(tokenPosition).getEndOffset();
    } else { // if not, do a regular replacement
        insertionPoint = oldNodeIA.getLeftToken().getStartOffset() - 1;
    }


    // insert after the first non-layout token
    int tokenPositionEnd = oldNodeIA.getRightToken().getIndex();

    while((tokenizer.getTokenAt(tokenPositionEnd).getEndOffset() < tokenizer.getTokenAt(tokenPositionEnd)
        .getStartOffset() || tokenizer.getTokenAt(tokenPositionEnd).getKind() == IToken.TK_LAYOUT
        || tokenizer.getTokenAt(tokenPositionEnd).getKind() == IToken.TK_ERROR) && tokenPositionEnd > 0)
        tokenPositionEnd--;

    suffixPoint = tokenizer.getTokenAt(tokenPositionEnd).getEndOffset() + 1;

    CompletionKind kind;
    if(completionKind.equals("recovery")) {
        kind = CompletionKind.recovery;
    } else {
        kind = CompletionKind.expansion;
    }

    return new Completion(name, sort, text, additionalInfo, insertionPoint + 1, suffixPoint, kind, prefix, suffix);
}
 

import org.spoofax.jsglr.client.imploder.ImploderAttachment; //导入方法依赖的package包/类
private void detectEditRegions() {
	ITokens tokens = ImploderAttachment.getTokenizer(correctAST);

	long time = System.currentTimeMillis();
	
	// calculates offsets deleted characters (correct input), and offsets inserted characters (erroneous input)
	constructCharacterMatching();
	ArrayList<Integer> offsetsDeletedChars = lcs.getUnMatchedIndices1();
	System.out.println("LCS: " + (System.currentTimeMillis()-time));
	time = System.currentTimeMillis();
	//TODO: LCS on lines (instead of characters) in case LCS optimized does not work to reduce the size enough
	
	DamagedTokenAnalyzer tokenEdits = new DamagedTokenAnalyzer((AbstractTokenizer) tokens, lcs);
	System.out.println("token edits: " + (System.currentTimeMillis()-time));
	time = System.currentTimeMillis();
	//TODO: Improve performance!!

	//removes from deletion offsets, all offsets of layout characters that are irrelevant for the parse result.
	//detects all edited comment regions, since these may affect the parse result if they are broken.
	LayoutEditsAnalyzer loAnalyzer = new LayoutEditsAnalyzer(tokenEdits);
	this.discardableCommentRegions = loAnalyzer.getDamagedCommentRegions();
	loAnalyzer.filterNonLayoutOffsets(offsetsDeletedChars);
	System.out.println("layout edits: " + (System.currentTimeMillis()-time));
	time = System.currentTimeMillis();

	//extends deletion offsets so that all possible damaged tokens are covered
	TerminalEditsAnalyzer terminalAnalyzer = new TerminalEditsAnalyzer(tokenEdits);
	terminalAnalyzer.addDamagedTokensStartOffsets(offsetsDeletedChars);
	System.out.println("terminal edits: " + (System.currentTimeMillis()-time));
	time = System.currentTimeMillis();

	//detects discardable regions that correspond to edited terms.
	//NonTerminalEditsAnalyzer brokenConstructDetector = new NonTerminalEditsAnalyzer(correctAST, offsetsDeletedChars);
	//this.discardableRegions = brokenConstructDetector.getDamagedTermRegions();
	
	//detects discardable regions that correspond to edited terms.
	TermEditsAnalyzer brokenConstructDetector = new TermEditsAnalyzer(offsetsDeletedChars, correctAST);
	this.discardRecovery = brokenConstructDetector.getDiscardRecovery();		
	System.out.println("Term edits: " + (System.currentTimeMillis()-time));
	time = System.currentTimeMillis();
}