Java LabelAwareIterator类代码示例

import org.deeplearning4j.text.documentiterator.LabelAwareIterator; //导入依赖的package包/类
/**
 * transforms paragraph vectors into hierarchical vectors
 * @param iterator iterator over docs
 * @param lookupTable the paragraph vector table
 * @param labels the labels
 * @param k the no. of centroids
 * @return a map doc->hierarchical vector
 */
static Map<String, INDArray> getPar2Hier(LabelAwareIterator iterator,
                                         WeightLookupTable<VocabWord> lookupTable,
                                         List<String> labels, int k, Method method) {
  Collections.sort(labels);
  LabelsSource labelsSource = iterator.getLabelsSource();
  PatriciaTrie<String> trie = new PatriciaTrie<>();
  for (String label : labels) {
    trie.put(label, label);
  }

  Map<String, INDArray> hvs = new TreeMap<>();
  // for each doc
  for (String node : labelsSource.getLabels()) {
    Par2HierUtils.getPar2HierVector(lookupTable, trie, node, k, hvs, method);
  }
  return hvs;
}
 

import org.deeplearning4j.text.documentiterator.LabelAwareIterator; //导入依赖的package包/类
public BasicTransformerIterator(@NonNull LabelAwareIterator iterator, @NonNull SentenceTransformer transformer) {
    this.iterator = iterator;
    this.allowMultithreading = false;
    this.sentenceTransformer = transformer;

    this.iterator.reset();
}
 

import org.deeplearning4j.text.documentiterator.LabelAwareIterator; //导入依赖的package包/类
public ParallelTransformerIterator(@NonNull LabelAwareIterator iterator, @NonNull SentenceTransformer transformer,
                boolean allowMultithreading) {
    super(new AsyncLabelAwareIterator(iterator, 512), transformer);
    this.allowMultithreading = allowMultithreading;
    this.stringBuffer = new LinkedBlockingQueue<>(512);

    threads = new TokenizerThread[allowMultithreading ? Math.max(Runtime.getRuntime().availableProcessors(), 2) : 1];

    try {
        int cnt = 0;
        while (cnt < 256) {
            boolean before = underlyingHas;

            if (before)
                underlyingHas = this.iterator.hasNextDocument();

            if (underlyingHas)
                stringBuffer.put(this.iterator.nextDocument());
            else
                cnt += 257;

            cnt++;
        }
    } catch (Exception e) {
        //
    }

    for (int x = 0; x < threads.length; x++) {
        threads[x] = new TokenizerThread(x, transformer, stringBuffer, buffer, processing);
        threads[x].setDaemon(true);
        threads[x].setName("ParallelTransformer thread " + x);
        threads[x].start();
    }
}
 

import org.deeplearning4j.text.documentiterator.LabelAwareIterator; //导入依赖的package包/类
/**
 * This method used to feed LabelAwareIterator, that contains training corpus, into Par2Hier
 *
 */
public Builder iterate(@NonNull LabelAwareIterator iterator) {
  this.labelAwareIterator = iterator;
  return this;
}
 

import org.deeplearning4j.text.documentiterator.LabelAwareIterator; //导入依赖的package包/类
/** Specify how the (labelled) sentences / documents should be provided */
public CnnSentenceDataSetIterator.Builder sentenceProvider(
    LabelAwareIterator iterator, @NonNull List<String> labels) {
  LabelAwareConverter converter = new LabelAwareConverter(iterator, labels);
  return sentenceProvider(converter);
}
 

import org.deeplearning4j.text.documentiterator.LabelAwareIterator; //导入依赖的package包/类
private SentenceTransformer(@NonNull LabelAwareIterator iterator) {
    this.iterator = iterator;
}
 

import org.deeplearning4j.text.documentiterator.LabelAwareIterator; //导入依赖的package包/类
public Builder setIterator(@NonNull LabelAwareIterator iterator) {
    this.iterator = iterator;
    return this;
}
 

import org.deeplearning4j.text.documentiterator.LabelAwareIterator; //导入依赖的package包/类
public LabelAwareConverter(@NonNull LabelAwareIterator iterator, @NonNull List<String> labels) {
    this.backingIterator = iterator;
    this.labels = labels;
}
 

import org.deeplearning4j.text.documentiterator.LabelAwareIterator; //导入依赖的package包/类
/**
 * Specify how the (labelled) sentences / documents should be provided
 */
public Builder sentenceProvider(LabelAwareIterator iterator, @NonNull List<String> labels) {
    LabelAwareConverter converter = new LabelAwareConverter(iterator, labels);
    return sentenceProvider(converter);
}
 

import org.deeplearning4j.text.documentiterator.LabelAwareIterator; //导入依赖的package包/类
private void checkUnlabelledData(Word2Vec paragraphVectors, LabelAwareIterator iterator, TokenizerFactory tokenizerFactory) throws FileNotFoundException {
  ClassPathResource unClassifiedResource = new ClassPathResource("papers/unlabeled");
  FileLabelAwareIterator unClassifiedIterator = new FileLabelAwareIterator.Builder()
      .addSourceFolder(unClassifiedResource.getFile())
      .build();

  MeansBuilder meansBuilder = new MeansBuilder(
      (InMemoryLookupTable<VocabWord>) paragraphVectors.getLookupTable(),
      tokenizerFactory);
  LabelSeeker seeker = new LabelSeeker(iterator.getLabelsSource().getLabels(),
      (InMemoryLookupTable<VocabWord>) paragraphVectors.getLookupTable());

  System.out.println(paragraphVectors + " classification results");
  double cc = 0;
  double size = 0;
  while (unClassifiedIterator.hasNextDocument()) {
    LabelledDocument document = unClassifiedIterator.nextDocument();
    INDArray documentAsCentroid = meansBuilder.documentAsVector(document);
    List<Pair<String, Double>> scores = seeker.getScores(documentAsCentroid);

    double max = -Integer.MAX_VALUE;
    String cat = null;
    for (Pair<String, Double> p : scores) {
      if (p.getSecond() > max) {
        max = p.getSecond();
        cat = p.getFirst();
      }
    }
    if (document.getLabels().contains(cat)) {
      cc++;
    }
    size++;

  }
  System.out.println("acc:" + (cc / size));

}
 

import org.deeplearning4j.text.documentiterator.LabelAwareIterator; //导入依赖的package包/类
/**
 * This method used to feed LabelAwareIterator, that is usually used
 *
 * @param iterator
 * @return
 */
public Builder iterate(@NonNull LabelAwareIterator iterator) {
    this.labelAwareIterator = iterator;
    return this;
}
 

import org.deeplearning4j.text.documentiterator.LabelAwareIterator; //导入依赖的package包/类
/**
 * This test is not indicative.
 * there's no need in this test within travis, use it manually only for problems detection
 *
 * @throws Exception
 */
@Test
@Ignore
public void testParagraphVectorsReducedLabels1() throws Exception {
    ClassPathResource resource = new ClassPathResource("/labeled");
    File file = resource.getFile();

    LabelAwareIterator iter = new FileLabelAwareIterator.Builder().addSourceFolder(file).build();

    TokenizerFactory t = new DefaultTokenizerFactory();

    /**
     * Please note: text corpus is REALLY small, and some kind of "results" could be received with HIGH epochs number, like 30.
     * But there's no reason to keep at that high
     */

    ParagraphVectors vec = new ParagraphVectors.Builder().minWordFrequency(1).epochs(3).layerSize(100)
                    .stopWords(new ArrayList<String>()).windowSize(5).iterate(iter).tokenizerFactory(t).build();

    vec.fit();

    //WordVectorSerializer.writeWordVectors(vec, "vectors.txt");

    INDArray w1 = vec.lookupTable().vector("I");
    INDArray w2 = vec.lookupTable().vector("am");
    INDArray w3 = vec.lookupTable().vector("sad.");

    INDArray words = Nd4j.create(3, vec.lookupTable().layerSize());

    words.putRow(0, w1);
    words.putRow(1, w2);
    words.putRow(2, w3);


    INDArray mean = words.isMatrix() ? words.mean(0) : words;

    log.info("Mean" + Arrays.toString(mean.dup().data().asDouble()));
    log.info("Array" + Arrays.toString(vec.lookupTable().vector("negative").dup().data().asDouble()));

    double simN = Transforms.cosineSim(mean, vec.lookupTable().vector("negative"));
    log.info("Similarity negative: " + simN);


    double simP = Transforms.cosineSim(mean, vec.lookupTable().vector("neutral"));
    log.info("Similarity neutral: " + simP);

    double simV = Transforms.cosineSim(mean, vec.lookupTable().vector("positive"));
    log.info("Similarity positive: " + simV);
}