Java Token.addToIndexes方法代码示例

import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; //导入方法依赖的package包/类
private static void copyParagraphAndTokenAnnotations(JCas source, JCas target)
{
    if (!source.getDocumentText().equals(target.getDocumentText())) {
        throw new IllegalArgumentException("Source and target have different content");
    }

    for (Paragraph p : JCasUtil.select(source, Paragraph.class)) {
        Paragraph paragraph = new Paragraph(target);
        paragraph.setBegin(p.getBegin());
        paragraph.setEnd(p.getEnd());
        paragraph.addToIndexes();
    }

    for (Token t : JCasUtil.select(source, Token.class)) {
        Token token = new Token(target);
        token.setBegin(t.getBegin());
        token.setEnd(t.getEnd());
        token.addToIndexes();
    }
}
 

import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; //导入方法依赖的package包/类
@Before
public void setUp()
        throws Exception
{
    jCas = JCasFactory.createJCas();
    jCas.setDocumentLanguage("en");
    jCas.setDocumentText("This is a test.");

    // Add some annotations
    tokenThis = new Token(jCas, 0, 4);
    tokenThis.addToIndexes();
    tokenIs = new Token(jCas, 5, 7);
    tokenIs.addToIndexes();
    tokenA = new Token(jCas, 8, 9);
    tokenA.addToIndexes();
    tokenTest = new Token(jCas, 10, 14);
    tokenTest.addToIndexes();
    tokenDot = new Token(jCas, 14, 15);
    tokenDot.addToIndexes();
}
 

import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; //导入方法依赖的package包/类
public static void tokenize(JCas aJCas)
{
    BreakIterator bi = BreakIterator.getWordInstance(Locale.US);
    for (Sentence s : select(aJCas, Sentence.class)) {
        bi.setText(s.getCoveredText());
        int last = bi.first();
        int cur = bi.next();
        while (cur != BreakIterator.DONE) {
            int[] span = new int[] { last, cur };
            trim(s.getCoveredText(), span);
            if (!isEmpty(span[0], span[1])) {
                Token seg = new Token(aJCas, span[0] + s.getBegin(), span[1] + s.getBegin());
                seg.addToIndexes(aJCas);
            }
            last = cur;
            cur = bi.next();
        }
    }
}
 

import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; //导入方法依赖的package包/类
public List<String> posTag(List<String> sentence)
    throws UIMAException
{

    JCas jCas = JCasFactory.createJCas();
    StringBuilder documentText = new StringBuilder();

    int start = 0;
    for (int i = 0; i < sentence.size(); i++) {
        String token = sentence.get(i);
        documentText.append(token);

        Token t = new Token(jCas, start, documentText.length());
        t.addToIndexes();

        if (i + 1 < sentence.size()) {
            documentText.append(" ");
        }
        start = documentText.length();
    }
    jCas.setDocumentText(documentText.toString());
    
    Sentence s = new Sentence(jCas, 0, jCas.getDocumentText().length());
    s.addToIndexes();

    callFlexTag(jCas);

    List<String> posTags = new ArrayList<String>();
    JCasUtil.select(jCas, POS.class).forEach(x -> posTags.add(x.getPosValue()));

    return posTags;
}
 

import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; //导入方法依赖的package包/类
public List<List<String>> tagSentences(List<List<String>> sentences)
    throws Exception
{

    JCas jCas = JCasFactory.createJCas();

    StringBuilder sb = new StringBuilder();
    for (List<String> sentence : sentences) {
        int sentStart = sb.length();
        for (String token : sentence) {
            int start = sb.length();
            int end = sb.length() + token.length();

            Token t = new Token(jCas, start, end);
            t.addToIndexes();

            sb.append(token);
        }
        int sentEnd = sb.length();
        Sentence s = new Sentence(jCas, sentStart, sentEnd);
        s.addToIndexes();
    }
    jCas.setDocumentText(sb.toString().trim());

    flexTagEngine.process(jCas);

    return extractTags(jCas);
}
 

import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; //导入方法依赖的package包/类
/**
 * Create {@link Token} in the {@link CAS}. If the lemma and pos columns are not empty it will
 * create {@link Lemma} and {@link POS} annotations
 */
private void createToken(JCas aJCas, StringBuilder text, Map<Integer, String> tokens,
        Map<Integer, String> pos, Map<Integer, String> lemma, Map<String, Token> tokensStored)
{
    int tokenBeginPosition = 0;
    int tokenEndPosition = 0;

    for (int i = 1; i <= tokens.size(); i++) {
        tokenBeginPosition = text.indexOf(tokens.get(i), tokenBeginPosition);
        Token outToken = new Token(aJCas, tokenBeginPosition, text.indexOf(tokens.get(i),
                tokenBeginPosition) + tokens.get(i).length());
        tokenEndPosition = text.indexOf(tokens.get(i), tokenBeginPosition)
                + tokens.get(i).length();
        tokenBeginPosition = tokenEndPosition;
        outToken.addToIndexes();

        // Add pos to CAS if exist
        if (!pos.get(i).equals("_")) {
            POS outPos = new POS(aJCas, outToken.getBegin(), outToken.getEnd());
            outPos.setPosValue(pos.get(i));
            outPos.addToIndexes();
            outToken.setPos(outPos);
        }

        // Add lemma if exist
        if (!lemma.get(i).equals("_")) {
            Lemma outLemma = new Lemma(aJCas, outToken.getBegin(), outToken.getEnd());
            outLemma.setValue(lemma.get(i));
            outLemma.addToIndexes();
            outToken.setLemma(outLemma);
        }
        tokensStored.put("t_" + i, outToken);
    }
}
 

import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; //导入方法依赖的package包/类
@Test
public void test()
    throws Exception
{
    JCas jcas = JCasFactory.createJCas();

    jcas.setDocumentText("This is a test.");
    
    Token span1 = new Token(jcas, 0, 4);
    span1.addToIndexes();
    
    Token span2 = new Token(jcas, 6, 8);
    
    Dependency dep = new Dependency(jcas, 0, 8);
    dep.setGovernor(span1);
    dep.setDependent(span2);
    dep.addToIndexes();
    
    List<LogMessage> messages = new ArrayList<>();
    CasDoctor cd = new CasDoctor(RemoveDanglingRelationsRepair.class,
            AllFeatureStructuresIndexedCheck.class);
    // A project is not required for this check
    boolean result = cd.analyze(null, jcas.getCas(), messages);
    // A project is not required for this repair
    cd.repair(null, jcas.getCas(), messages);
    
    assertFalse(result);
    
    messages.forEach(System.out::println);
}
 

import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; //导入方法依赖的package包/类
public static FeatureStructure makeLinkFS(JCas aJCas, String aSlotLabel, int aTargetBegin,
        int aTargetEnd)
{
    Token token1 = new Token(aJCas, aTargetBegin, aTargetEnd);
    token1.addToIndexes();

    Type linkType = aJCas.getTypeSystem().getType(LINK_TYPE);
    FeatureStructure linkA1 = aJCas.getCas().createFS(linkType);
    linkA1.setStringValue(linkType.getFeatureByBaseName("role"), aSlotLabel);
    linkA1.setFeatureValue(linkType.getFeatureByBaseName("target"), token1);
    aJCas.getCas().addFsToIndexes(linkA1);

    return linkA1;
}
 

import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; //导入方法依赖的package包/类
public void getNext(JCas aJCas)
    throws IOException, CollectionException
{

    DocumentMetaData md = new DocumentMetaData(aJCas);
    md.setDocumentTitle("");
    md.setDocumentId("" + (instanceId++));
    md.setLanguage(language);
    md.addToIndexes();

    try {
        posMappingProvider.configure(aJCas.getCas());
    }
    catch (AnalysisEngineProcessException e1) {
        throw new CollectionException(e1);
    }

    StringBuilder documentText = new StringBuilder();

    int seqStart = 0;
    for (int k = 0; k < sequences.size(); k++) {

        List<String> sequence = sequences.get(k);

        for (int i = 0; i < sequence.size(); i++) {
            String pairs = sequence.get(i).replaceAll(" +", " ");

            int idxLastSpace = pairs.lastIndexOf(" ");
            String token = pairs.substring(0, idxLastSpace);
            String tag = pairs.substring(idxLastSpace+1);

            int tokenLen = token.length();
            if(lowerCase){
                token = token.toLowerCase();
            }
            documentText.append(token);

            int tokStart = documentText.length() - tokenLen;
            int tokEnd = documentText.length();
            Token t = new Token(aJCas, tokStart, tokEnd);
            t.addToIndexes();

            if (i + 1 < sequence.size()) {
                documentText.append(" ");
            }

            Type posTag = posMappingProvider.getTagType(tag);
            POS pos = (POS) aJCas.getCas().createAnnotation(posTag, t.getBegin(),
                    t.getEnd());
            pos.setPosValue(tag);
            pos.addToIndexes();
            t.setPos(pos);
        }
        Sentence sentence = new Sentence(aJCas, seqStart, documentText.length());
        sentence.addToIndexes();

        if (k + 1 < sequences.size()) {
            documentText.append(" ");
        }
        seqStart = documentText.length();

    }
    aJCas.setDocumentText(documentText.toString());
}
 

import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; //导入方法依赖的package包/类
@Test
public void testSimplePath()
    throws Exception
{
    ConstraintsGrammar parser = new ConstraintsGrammar(new FileInputStream(
            "src/test/resources/rules/10.rules"));
    Parse p = parser.Parse();

    ParsedConstraints constraints = p.accept(new ParserVisitor());

    JCas jcas = JCasFactory.createJCas();
    jcas.setDocumentText("The sun.");

    // Add token annotations
    Token t_the = new Token(jcas, 0, 3);
    t_the.addToIndexes();
    Token t_sun = new Token(jcas, 0, 3);
    t_sun.addToIndexes();

    // Add POS annotations and link them to the tokens
    POS p_the = new POS(jcas, t_the.getBegin(), t_the.getEnd());
    p_the.setPosValue("DET");
    p_the.addToIndexes();
    t_the.setPos(p_the);
    POS p_sun = new POS(jcas, t_sun.getBegin(), t_sun.getEnd());
    p_sun.setPosValue("NN");
    p_sun.addToIndexes();
    t_sun.setPos(p_sun);

    // Add dependency annotations
    Dependency dep_the_sun = new Dependency(jcas);
    dep_the_sun.setGovernor(t_sun);
    dep_the_sun.setDependent(t_the);
    dep_the_sun.setDependencyType("det");
    dep_the_sun.setBegin(dep_the_sun.getGovernor().getBegin());
    dep_the_sun.setEnd(dep_the_sun.getGovernor().getEnd());
    dep_the_sun.addToIndexes();

    Evaluator constraintsEvaluator = new ValuesGenerator();

    List<PossibleValue> possibleValues = constraintsEvaluator.generatePossibleValues(
            dep_the_sun, "DependencyType", constraints);

    List<PossibleValue> expectedOutput = new LinkedList<>();
    expectedOutput.add(new PossibleValue("det", false));

    assertEquals(expectedOutput, possibleValues);
}