Java VocabCache.numWords方法代码示例

import org.deeplearning4j.models.word2vec.wordstore.VocabCache; //导入方法依赖的package包/类
/**
 * This method saves specified SequenceVectors model to target  OutputStream
 *
 * @param vectors SequenceVectors model
 * @param factory SequenceElementFactory implementation for your objects
 * @param stream Target output stream
 * @param <T>
 */
public static <T extends SequenceElement> void writeSequenceVectors(@NonNull SequenceVectors<T> vectors,
                @NonNull SequenceElementFactory<T> factory, @NonNull OutputStream stream) throws IOException {
    WeightLookupTable<T> lookupTable = vectors.getLookupTable();
    VocabCache<T> vocabCache = vectors.getVocab();

    PrintWriter writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(stream, "UTF-8")));

    // at first line we save VectorsConfiguration
    writer.write(vectors.getConfiguration().toEncodedJson());

    // now we have elements one by one
    for (int x = 0; x < vocabCache.numWords(); x++) {
        T element = vocabCache.elementAtIndex(x);
        String json = factory.serialize(element);
        INDArray d = Nd4j.create(1);
        double[] vector = lookupTable.vector(element.getLabel()).dup().data().asDouble();
        ElementPair pair = new ElementPair(json, vector);
        writer.println(pair.toEncodedJson());
        writer.flush();
    }
    writer.flush();
    writer.close();
}
 

import org.deeplearning4j.models.word2vec.wordstore.VocabCache; //导入方法依赖的package包/类
/**
 * This mehod writes word vectors to the given OutputStream.
 * Please note: this method doesn't load whole vocab/lookupTable into memory, so it's able to process large vocabularies served over network.
 *
 * @param lookupTable
 * @param stream
 * @param <T>
 * @throws IOException
 */
public static <T extends SequenceElement> void writeWordVectors(WeightLookupTable<T> lookupTable,
                OutputStream stream) throws IOException {
    VocabCache<T> vocabCache = lookupTable.getVocabCache();

    PrintWriter writer = new PrintWriter(new OutputStreamWriter(stream, "UTF-8"));
    // saving header as "NUM_WORDS VECTOR_SIZE NUM_DOCS"
    String str = vocabCache.numWords() + " " + lookupTable.layerSize() + " " + vocabCache.totalNumberOfDocs();
    log.debug("Saving header: {}", str);
    writer.println(str);

    // saving vocab content
    for (int x = 0; x < vocabCache.numWords(); x++) {
        T element = vocabCache.elementAtIndex(x);

        StringBuilder builder = new StringBuilder();

        builder.append(encodeB64(element.getLabel())).append(" ");
        INDArray vec = lookupTable.vector(element.getLabel());
        for (int i = 0; i < vec.length(); i++) {
            builder.append(vec.getDouble(i));
            if (i < vec.length() - 1)
                builder.append(" ");
        }
        writer.println(builder.toString());
    }
    writer.flush();
    writer.close();
}
 

import org.deeplearning4j.models.word2vec.wordstore.VocabCache; //导入方法依赖的package包/类
/**
 * This method saves vocab cache to provided OutputStream.
 * Please note: it saves only vocab content, so it's suitable mostly for BagOfWords/TF-IDF vectorizers
 *
 * @param vocabCache
 * @param stream
 * @throws UnsupportedEncodingException
 */
public static void writeVocabCache(@NonNull VocabCache<VocabWord> vocabCache, @NonNull OutputStream stream)
                throws IOException {
    PrintWriter writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(stream, "UTF-8")));

    for (int x = 0; x < vocabCache.numWords(); x++) {
        VocabWord word = vocabCache.elementAtIndex(x);
        writer.println(word.toJSON());
    }

    writer.flush();
    writer.close();
}