Java RecordMetaData类代码示例

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
@Test
public void testCodecReaderMeta() throws Exception {
    File file = new ClassPathResource("fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new CodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    assertEquals(500, record.size()); //500 frames

    reader.reset();
    SequenceRecord seqR = reader.nextSequence();
    assertEquals(record, seqR.getSequenceRecord());
    RecordMetaData meta = seqR.getMetaData();
    //        System.out.println(meta);
    assertTrue(meta.getURI().toString().endsWith("fire_lowres.mp4"));

    SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta);
    assertEquals(seqR, fromMeta);
}
 

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
/**
 * @see #loadFromMetaData(RecordMetaData)
 */
@Override
public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
    List<Record> ret = new ArrayList<>();

    for (RecordMetaData rmd : recordMetaDatas) {
        if (!(rmd instanceof RecordMetaDataJdbc)) {
            throw new IllegalArgumentException(
                "Invalid metadata; expected RecordMetaDataJdbc instance; got: " + rmd);
        }
        QueryRunner runner = new QueryRunner();
        String request = ((RecordMetaDataJdbc) rmd).getRequest();

        try {
            Object[] item = runner
                .query(this.conn, request, new ArrayHandler(), ((RecordMetaDataJdbc) rmd).getParams().toArray());
            ret.add(new org.datavec.api.records.impl.Record(toWritable(item), rmd));
        } catch (SQLException e) {
            throw new IllegalArgumentException("Could not execute statement \"" + request + "\"", e);
        }
    }
    return ret;
}
 

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
@Override
public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {

    List<Record> out = new ArrayList<>();
    for (RecordMetaData metaData : recordMetaDatas) {
        URI uri = metaData.getURI();

        String fileAsString;
        try {
            fileAsString = FileUtils.readFileToString(new File(uri.toURL().getFile()));
        } catch (IOException e) {
            throw new RuntimeException("Error reading URI file", e);
        }

        List<Writable> writables = readValues(uri, fileAsString);
        out.add(new org.datavec.api.records.impl.Record(writables, metaData));
    }

    return out;
}
 

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
@Test
public void testsBasicMeta() throws Exception {
    //As per testBasic - but also loading metadata
    RecordReader rr2 = new CSVRecordReader(0, ',');
    rr2.initialize(new FileSplit(new ClassPathResource("iris.txt").getTempFileFromArchive()));

    RecordReaderMultiDataSetIterator rrmdsi = new RecordReaderMultiDataSetIterator.Builder(10)
                    .addReader("reader", rr2).addInput("reader", 0, 3).addOutputOneHot("reader", 4, 3).build();

    rrmdsi.setCollectMetaData(true);

    int count = 0;
    while (rrmdsi.hasNext()) {
        MultiDataSet mds = rrmdsi.next();
        MultiDataSet fromMeta = rrmdsi.loadFromMetaData(mds.getExampleMetaData(RecordMetaData.class));
        assertEquals(mds, fromMeta);
        count++;
    }
    assertEquals(150 / 10, count);
}
 

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
@Test
public void testSplittingCSVMeta() throws Exception {
    //Here's the idea: take Iris, and split it up into 2 inputs and 2 output arrays
    //Inputs: columns 0 and 1-2
    //Outputs: columns 3, and 4->OneHot
    RecordReader rr2 = new CSVRecordReader(0, ',');
    rr2.initialize(new FileSplit(new ClassPathResource("iris.txt").getTempFileFromArchive()));

    RecordReaderMultiDataSetIterator rrmdsi = new RecordReaderMultiDataSetIterator.Builder(10)
                    .addReader("reader", rr2).addInput("reader", 0, 0).addInput("reader", 1, 2)
                    .addOutput("reader", 3, 3).addOutputOneHot("reader", 4, 3).build();
    rrmdsi.setCollectMetaData(true);

    int count = 0;
    while (rrmdsi.hasNext()) {
        MultiDataSet mds = rrmdsi.next();
        MultiDataSet fromMeta = rrmdsi.loadFromMetaData(mds.getExampleMetaData(RecordMetaData.class));
        assertEquals(mds, fromMeta);
        count++;
    }
    assertEquals(150 / 10, count);
}
 

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
@Override
public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
    List<Record> out = new ArrayList<>();
    for (RecordMetaData meta : recordMetaDatas) {
        URI uri = meta.getURI();
        File f = new File(uri);

        List<Writable> next;
        try (DataInputStream dis = new DataInputStream(new BufferedInputStream(new FileInputStream(f)))) {
            next = record(uri, dis);
        }
        out.add(new org.datavec.api.records.impl.Record(next, meta));
    }
    return out;
}
 

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
@Test
public void testMetaData() throws IOException {

    ClassPathResource cpr = new ClassPathResource("/testimages/class0/0.jpg");
    File parentDir = cpr.getFile().getParentFile().getParentFile();
    //        System.out.println(f.getAbsolutePath());
    //        System.out.println(f.getParentFile().getParentFile().getAbsolutePath());
    ParentPathLabelGenerator labelMaker = new ParentPathLabelGenerator();
    ImageRecordReader rr = new ImageRecordReader(32, 32, 3, labelMaker);
    rr.initialize(new FileSplit(parentDir));

    List<List<Writable>> out = new ArrayList<>();
    while (rr.hasNext()) {
        List<Writable> l = rr.next();
        out.add(l);
        assertEquals(2, l.size());
    }

    assertEquals(6, out.size());

    rr.reset();
    List<List<Writable>> out2 = new ArrayList<>();
    List<Record> out3 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();

    while (rr.hasNext()) {
        Record r = rr.nextRecord();
        out2.add(r.getRecord());
        out3.add(r);
        meta.add(r.getMetaData());
        //            System.out.println(r.getMetaData() + "\t" + r.getRecord().get(1));
    }

    assertEquals(out, out2);

    List<Record> fromMeta = rr.loadFromMetaData(meta);
    assertEquals(out3, fromMeta);
}
 

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
@Override
public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
    List<Record> out = new ArrayList<>();

    for (Record fileContents : super.loadFromMetaData(recordMetaDatas)) {
        INDArray transform = tfidfVectorizer.transform(fileContents);

        org.datavec.api.records.impl.Record record = new org.datavec.api.records.impl.Record(
                        new ArrayList<>(Collections.<Writable>singletonList(new NDArrayWritable(transform))),
                        new RecordMetaDataURI(fileContents.getMetaData().getURI(), TfidfRecordReader.class));

        if (appendLabel)
            record.getRecord().add(fileContents.getRecord().get(fileContents.getRecord().size() - 1));
        out.add(record);
    }

    return out;
}
 

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
@Override
public List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
    List<SequenceRecord> out = new ArrayList<>();
    for (RecordMetaData meta : recordMetaDatas) {
        File f = new File(meta.getURI());

        List<List<Writable>> list = loadData(f, null);
        out.add(new org.datavec.api.records.impl.SequenceRecord(list, meta));
    }

    return out;
}
 

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
@Ignore
@Test
public void testNativeCodecReaderMeta() throws Exception {
    File file = new ClassPathResource("fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new NativeCodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    assertEquals(500, record.size()); //500 frames

    reader.reset();
    SequenceRecord seqR = reader.nextSequence();
    assertEquals(record, seqR.getSequenceRecord());
    RecordMetaData meta = seqR.getMetaData();
    //        System.out.println(meta);
    assertTrue(meta.getURI().toString().endsWith("fire_lowres.mp4"));

    SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta);
    assertEquals(seqR, fromMeta);
}
 

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
@Test
public void testNextRecord() throws Exception {
    try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) {
        Record r = reader.nextRecord();
        List<Writable> fields = r.getRecord();
        RecordMetaData meta = r.getMetaData();
        assertNotNull(r);
        assertNotNull(fields);
        assertNotNull(meta);
        assertEquals(new Text("Bolivian Dark"), fields.get(0));
        assertEquals(new Text("14-001"), fields.get(1));
        assertEquals(new DoubleWritable(8.95), fields.get(2));
        assertEquals(RecordMetaDataJdbc.class, meta.getClass());
    }
}
 

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
@Test
public void testNextRecordAndRecover() throws Exception {
    try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) {
        Record r = reader.nextRecord();
        List<Writable> fields = r.getRecord();
        RecordMetaData meta = r.getMetaData();
        Record recovered = reader.loadFromMetaData(meta);
        List<Writable> fieldsRecovered = recovered.getRecord();
        assertEquals(fields.size(), fieldsRecovered.size());
        for (int i = 0; i < fields.size(); i++) {
            assertEquals(fields.get(i), fieldsRecovered.get(i));
        }
    }
}
 

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
/**
 * Load multiple records from the given a list of {@link RecordMetaData} instances<br>
 *
 * @param recordMetaDatas Metadata for the records that we want to load from
 * @return Multiple records for the given RecordMetaData instances
 * @throws IOException If I/O error occurs during loading
 */
@Override
public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
    List<Record> records = sequenceRecordReader.loadFromMetaData(recordMetaDatas);
    for (Record record : records)
        record.setRecord(transformProcess.execute(record.getRecord()));
    return records;
}
 

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
@Override
public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
    List<Record> out = new ArrayList<>();

    for (RecordMetaData meta : recordMetaDatas) {
        URI uri = meta.getURI();

        File f = new File(uri);
        List<Writable> list = loadFromFile(f);
        out.add(new org.datavec.api.records.impl.Record(list, meta));
    }

    return out;
}
 

import org.datavec.api.records.metadata.RecordMetaData; //导入依赖的package包/类
/**
 * Return next Record.
 *
 * @return
 */
@Override
public Record nextRecord() {
    List<Writable> next = next();
    URI uri = (locations == null || locations.length < 1 ? null : locations[splitIndex]);
    RecordMetaData meta = new RecordMetaDataLine(this.lineIndex - 1, uri, SVMLightRecordReader.class); //-1 as line number has been incremented already...
    return new org.datavec.api.records.impl.Record(next, meta);
}