Java InputReader类代码示例

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
public static void main(final String[] args) {
	final POSTagger postagger = POSTagger.getStanfordTagger(Util.getFile(args[0]));
	final InputReader reader = InputReader.make(InputFormat.TOKENIZED);

	final Scanner inputLines = new Scanner(System.in, "UTF-8");

	while (inputLines.hasNext()) {
		final String line = inputLines.nextLine();
		final List<InputWord> words = postagger.tag(reader.readInput(line).getInputWords());
		for (int i = 0; i < words.size(); i++) {
			if (i > 0) {
				System.out.print(" ");
			}

			System.out.print(words.get(i).word + "|" + words.get(i).pos);
		}
		System.out.println();
	}

	inputLines.close();
}
 

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
private static int parseFile(final Parser astar, final InputReader reader, final Stopwatch parsingTime,
		final File file, final ByteBuffer wrBuf) throws IOException {
	int sentences = 0;
	for (final InputToParser input : reader.readFile(file)) {
		parsingTime.start();
		final List<Scored<SyntaxTreeNode>> parses = astar.doParsing(input);
		parsingTime.stop();

		if (parses != null) {
			wrBuf.put(parses.get(0).getObject().toString().getBytes());
		}

		sentences++;
	}
	return sentences;
}
 

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
@Override
public void buildAgenda(final Agenda agenda, final List<InputReader.InputWord> words) {
    Preconditions.checkArgument(goldCategories.size() == words.size());
    for (int i = 0; i < words.size(); i++) {
        if (categories.contains(goldCategories.get(i))) {
            final InputReader.InputWord word = words.get(i);
            final SyntaxTreeNode node = new SyntaxTreeNodeLeaf(word.word, word.pos, word.ner,
                    goldCategories.get(i), i, true);
            agenda.add(new AgendaItem(node, 0, 0, i, 1, true));
        }
    }
}
 

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
public static Optional<NBestList> getNBestList(final BaseCcgParser parser, int sentenceId,
                                     final List<InputReader.InputWord> inputSentence) {
    return Optional
        .ofNullable(parser.parseNBest(sentenceId, inputSentence))
        .map(ImmutableList::copyOf)
        .map(NBestList::new);
}
 

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
public static ImmutableMap<Integer, NBestList> getAllNBestLists(
        final BaseCcgParser parser,
        final ImmutableList<ImmutableList<InputReader.InputWord>> inputSentences) {
    Map<Integer, NBestList> allParses = new HashMap<>();
    IntStream
        .range(0, inputSentences.size()).boxed()
        .forEach(sentenceId -> getNBestList(parser, sentenceId, inputSentences.get(sentenceId))
                 .ifPresent(nBestList -> allParses.put(sentenceId, nBestList)));
    return ImmutableMap.copyOf(allParses);
}
 

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
private BioinferCCGCorpus(final ImmutableList<ImmutableList<InputReader.InputWord>> inputSentences,
                          final ImmutableList<ImmutableList<String>> sentences,
                          final ImmutableList<ImmutableList<String>> postags,
                          final ImmutableList<ImmutableList<Category>> goldCategories) {
    this.inputSentences = inputSentences;
    this.sentences = sentences;
    this.postags = postags;
    this.goldCategories = goldCategories;
}
 

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
public static Optional<BioinferCCGCorpus> readTest() {
    POSTagger postagger = POSTagger.getStanfordTagger(Util.getFile(BaseCcgParser.modelFolder + "/posTagger"));
    List<ImmutableList<InputReader.InputWord>> inputSentences = new ArrayList<>();
    List<ImmutableList<String>> sentences = new ArrayList<>();
    try {
        BufferedReader reader = new BufferedReader(new FileReader(new File(BioinferTestFile)));
        String line;
        while ((line = reader.readLine()) != null) {
            if (line.trim().isEmpty()) {
                continue;
            }
            String[] tokens = line.split("\\s+");
            List<InputReader.InputWord> inputs = new ArrayList<>();
            List<String> words = new ArrayList<>();
            for (String tok : tokens) {
                words.add(tok);
                inputs.add(new InputReader.InputWord(tok, "", ""));
            }
            if (words.size() > 0) {
                List<InputReader.InputWord> taggedInputs = postagger.tag(inputs);
                inputSentences.add(ImmutableList.copyOf(taggedInputs));
                sentences.add(ImmutableList.copyOf(words));
            }
        }

    } catch (IOException e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        return Optional.empty();
    }
    System.out.println(String.format("Read %d sentences from %s.", sentences.size(), BioinferTestFile));
    return Optional.of(new BioinferCCGCorpus(ImmutableList.copyOf(inputSentences), ImmutableList.copyOf(sentences),
            ImmutableList.of(), ImmutableList.of()));
}
 

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
private static ParseData makeParseData(List<List<InputReader.InputWord>> sentenceInputWords,
                                       List<Parse> goldParses) {
    ImmutableList<ImmutableList<InputReader.InputWord>> thisSentences = sentenceInputWords
            .stream()
            .map(ImmutableList::copyOf)
            .collect(toImmutableList());
    ImmutableList<Parse> thisGoldParses = goldParses.stream().allMatch(p -> p != null) ?
            goldParses.stream().collect(toImmutableList()) :
            ImmutableList.of();
    return new ParseData(thisSentences, thisGoldParses);
}
 

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
public static Optional<ParseData> loadFromTestPool(boolean includeGold) {
    if (includeGold) {
        System.err.println("### Waring ### Reading test data with gold parses.");
    }
    POSTagger postagger = POSTagger.getStanfordTagger(Util.getFile(BaseCcgParser.modelFolder + "/posTagger"));
    List<List<InputReader.InputWord>> sentenceInputWords = new ArrayList<>();
    List<Parse> goldParses = new ArrayList<>();
    Iterator<ParallelCorpusReader.Sentence> sentenceIterator;
    try {
        sentenceIterator = ParallelCorpusReader.READER.readCcgTestSet();
    } catch (IOException e) {
        System.out.println(String.format("Failed to read %d sentences.", sentenceInputWords.size()));
        return Optional.empty();
    }
    while (sentenceIterator.hasNext()) {
        ParallelCorpusReader.Sentence sentence = sentenceIterator.next();
        List<InputReader.InputWord> taggedInput = postagger.tag(sentence.getInputWords());
        sentenceInputWords.add(taggedInput);
        Set<ResolvedDependency> goldDependencies = CCGBankEvaluation
                .asResolvedDependencies(sentence.getCCGBankDependencyParse().getDependencies());
        if (includeGold) {
            goldParses.add(new Parse(sentence.getCcgbankParse(), sentence.getLexicalCategories(), goldDependencies));
        } else {
            goldParses.add(null);
        }
    }
    System.out.println(String.format("Read %d sentences.", sentenceInputWords.size()));
    return Optional.of(makeParseData(sentenceInputWords, goldParses));
}
 

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
private static Optional<ParseData> loadFromPropBank(final boolean readDev) {
    if(readDev && devData != null) {
        return devData;
    }
    POSTagger postagger = POSTagger.getStanfordTagger(Util.getFile(BaseCcgParser.modelFolder + "/posTagger"));
    List<List<InputReader.InputWord>> sentenceInputWords = new ArrayList<>();
    List<Parse> goldParses = new ArrayList<>();
    Iterator<ParallelCorpusReader.Sentence> sentenceIterator;
    try {
        sentenceIterator = ParallelCorpusReader.READER.readCcgCorpus(readDev);
    } catch (IOException e) {
        System.out.println(String.format("Failed to read %d sentences.", sentenceInputWords.size()));
        devData = Optional.empty();
        return devData;
    }
    while (sentenceIterator.hasNext()) {
        ParallelCorpusReader.Sentence sentence = sentenceIterator.next();
        List<InputReader.InputWord> taggedInput = postagger.tag(sentence.getInputWords());
        sentenceInputWords.add(taggedInput);
        Set<ResolvedDependency> goldDependencies = CCGBankEvaluation
                .asResolvedDependencies(sentence.getCCGBankDependencyParse().getDependencies());
        goldParses.add(new Parse(sentence.getCcgbankParse(), sentence.getLexicalCategories(), goldDependencies));
    }
    System.out.println(String.format("Read %d sentences.", sentenceInputWords.size()));
    Optional<ParseData> data = Optional.of(makeParseData(sentenceInputWords, goldParses));
    if(readDev) {
        devData = data;
    }
    return data;
}
 

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
public static Optional<ParseData> loadFromBioinferDev() {
    POSTagger postagger = POSTagger.getStanfordTagger(Util.getFile(BaseCcgParser.modelFolder + "/posTagger"));
    List<List<InputReader.InputWord>> sentenceInputWords = new ArrayList<>();
    List<Parse> goldParses = new ArrayList<>();
    try {
        BufferedReader reader = new BufferedReader(new FileReader(new File(BioinferCCGCorpus.BioinferDevFile)));
        String line;
        while ((line = reader.readLine()) != null) {
            if (line.trim().isEmpty()) {
                continue;
            }
            String[] segments = line.split("\\s+");
            List<InputReader.InputWord> inputs = new ArrayList<>();
            List<String> words = new ArrayList<>(); //, pos = new ArrayList<>();
            List<Category> categories = new ArrayList<>();
            for (String seg : segments) {
                String[] info = seg.split("\\|");
                words.add(info[0]);
                // pos.add(info[1]);
                categories.add(Category.valueOf(info[2]));
                inputs.add(new InputReader.InputWord(info[0], "", ""));

            }
            if (words.size() > 0) {
                sentenceInputWords.add(ImmutableList.copyOf(postagger.tag(inputs)));
                goldParses.add(new Parse(words, categories));
            }
        }

    } catch (IOException e) {
        return Optional.empty();
    }

    System.out.println(String.format("Read %d sentences from %s.", sentenceInputWords.size(),
            BioinferCCGCorpus.BioinferDevFile));
    return Optional.of(makeParseData(sentenceInputWords, goldParses));
}
 

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
public ParseData(ImmutableList<ImmutableList<InputReader.InputWord>> sentenceInputWords,
                 ImmutableList<Parse> goldParses) {
    this.sentenceInputWords = sentenceInputWords;
    this.goldParses = goldParses;
    this.sentences = sentenceInputWords
        .stream()
        .map(sentenceIWs -> sentenceIWs
             .stream()
             .map(iw -> iw.word)
             .collect(toImmutableList()))
        .collect(toImmutableList());
}
 

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
protected Parse getParse(final List<InputReader.InputWord> sentence, final Scored<SyntaxTreeNode> scoredParse,
                         DependencyGenerator dependencyGenerator) {
    SyntaxTreeNode ccgParse = scoredParse.getObject();
    List<Category> categories = ccgParse.getLeaves().stream().map(SyntaxTreeNode::getCategory)
            .collect(Collectors.toList());
    Set<UnlabelledDependency> unlabelledDeps = new HashSet<>();
    dependencyGenerator.generateDependencies(ccgParse, unlabelledDeps);
    Set<ResolvedDependency> dependencies = CCGBankEvaluation.convertDeps(sentence, unlabelledDeps)
                    .stream()
                    .filter(x -> x.getHead() != x.getArgument())
                    .filter(x -> frequentDependenciesSet.contains(x.getCategory() + "." +  x.getArgNumber()))
                    .collect(Collectors.toSet());
    return new Parse(scoredParse.getObject(), categories, dependencies, scoredParse.getScore());
}
 

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
public void cacheSupertags(ImmutableList<ImmutableList<InputReader.InputWord>> inputSentences) {
    if (batchTagger != null) {
        System.err.println("Batch tagging " + inputSentences.size() + " sentences ...");
        taggedSentences = batchTagger.tagBatch(inputSentences.parallelStream()
                .map(s -> s.stream().collect(Collectors.toList())))
                .collect(GuavaCollectors.toImmutableList());
    }
}
 

import edu.uw.easysrl.main.InputReader; //导入依赖的package包/类
@Override
public Parse parse(List<InputReader.InputWord> sentence) {
    List<Scored<SyntaxTreeNode>> parses = parser.doParsing(
            new InputReader.InputToParser(sentence, null, null, false));
    if (parses == null || parses.size() == 0) {
        return null;
    }
    return getParse(sentence, parses.get(0), dependencyGenerator);
}