Java RecordReader.next方法代码示例

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void test() throws Exception {

    CSVRecordReader rr = new CSVRecordReader(0, ',');
    rr.initialize(new FileSplit(new ClassPathResource("iris.dat").getFile()));

    CSVRecordReader rr2 = new CSVRecordReader(0, ',');
    rr2.initialize(new FileSplit(new ClassPathResource("iris.dat").getFile()));

    RecordReader rrC = new ConcatenatingRecordReader(rr, rr2);

    int count = 0;
    while(rrC.hasNext()){
        rrC.next();
        count++;
    }

    assertEquals(300, count);
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
private static void testJacksonRecordReader(RecordReader rr) {

        List<Writable> json0 = rr.next();
        List<Writable> exp0 = Arrays.asList((Writable) new Text("aValue0"), new Text("bValue0"), new Text("cxValue0"));
        assertEquals(exp0, json0);

        List<Writable> json1 = rr.next();
        List<Writable> exp1 =
                        Arrays.asList((Writable) new Text("aValue1"), new Text("MISSING_B"), new Text("cxValue1"));
        assertEquals(exp1, json1);

        List<Writable> json2 = rr.next();
        List<Writable> exp2 =
                        Arrays.asList((Writable) new Text("aValue2"), new Text("bValue2"), new Text("MISSING_CX"));
        assertEquals(exp2, json2);

        assertFalse(rr.hasNext());

        //Test reset
        rr.reset();
        assertEquals(exp0, rr.next());
        assertEquals(exp1, rr.next());
        assertEquals(exp2, rr.next());
        assertFalse(rr.hasNext());
    }
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void testCsvRRSerializationResults() throws Exception {
    int skipLines = 3;
    RecordReader r1 = new CSVRecordReader(skipLines, '\t');
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ObjectOutputStream os = new ObjectOutputStream(baos);
    os.writeObject(r1);
    byte[] bytes = baos.toByteArray();
    ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes));
    RecordReader r2 = (RecordReader) ois.readObject();

    File f = new ClassPathResource("iris_tab_delim.txt").getFile();

    r1.initialize(new FileSplit(f));
    r2.initialize(new FileSplit(f));

    int count = 0;
    while(r1.hasNext()){
        List<Writable> n1 = r1.next();
        List<Writable> n2 = r2.next();
        assertEquals(n1, n2);
        count++;
    }

    assertEquals(150-skipLines, count);
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Override
public List<Writable> next() {
    List<Writable> out = null;
    for( RecordReader rr : readers){
        if(rr.hasNext()){
            out = rr.next();
            break;
        }
    }
    invokeListeners(out);
    return out;
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
    public void testReadWrite() throws Exception {
        Configuration conf = new Configuration();
        conf.set(FileRecordReader.APPEND_LABEL, "true");
        conf.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false);
        conf.setInt(LibSvmRecordReader.NUM_FEATURES, 4);
        File out = new File("iris.libsvm.out");
        if (out.exists())
            out.delete();
//        conf.set(FileRecordWriter.PATH, out.getAbsolutePath()); // setting output path via config not safe
        RecordReader libSvmRecordReader = new LibSvmRecordReader();
        libSvmRecordReader.initialize(conf, new FileSplit(new ClassPathResource("iris.libsvm").getFile()));

        RecordWriter writer = new LibSvmRecordWriter(out, true);
        writer.setConf(conf);
        List<List<Writable>> data = new ArrayList<>();
        while (libSvmRecordReader.hasNext()) {
            List<Writable> record = libSvmRecordReader.next();
            writer.write(record);
            data.add(record);
        }
        writer.close();

        out.deleteOnExit();
        List<List<Writable>> test = new ArrayList<>();
        RecordReader testLibSvmRecordReader = new LibSvmRecordReader();
        testLibSvmRecordReader.initialize(conf, new FileSplit(out));
        while (testLibSvmRecordReader.hasNext())
            test.add(testLibSvmRecordReader.next());
        assertEquals(data, test);
    }
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void testReadMissing() throws Exception {
    RecordReader rr = new LibSvmRecordReader();
    Configuration conf = new Configuration();
    conf.set(LineRecordReader.APPEND_LABEL, "true");
    conf.set(LibSvmRecordReader.NUM_FEATURES, "4");
    conf.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false);
    rr.initialize(conf, new FileSplit(new ClassPathResource("libsvm_with_multiple_missing.libsvm").getFile()));

    List<List<Double>> expected = new ArrayList<>(11);
    expected.add(Arrays.asList(1.0, 2.0, 3.0, 4.0, 1.0));
    expected.add(Arrays.asList(0.0, 2.0, 3.0, 4.0, 2.0));
    expected.add(Arrays.asList(1.0, 0.0, 3.0, 4.0, 3.0));
    expected.add(Arrays.asList(1.0, 2.0, 0.0, 4.0, 4.0));
    expected.add(Arrays.asList(1.0, 2.0, 3.0, 0.0, 5.0));
    expected.add(Arrays.asList(0.0, 0.0, 3.0, 4.0, 6.0));
    expected.add(Arrays.asList(1.0, 0.0, 0.0, 4.0, 7.0));
    expected.add(Arrays.asList(1.0, 2.0, 0.0, 0.0, 8.0));
    expected.add(Arrays.asList(0.0, 0.0, 0.0, 4.0, 9.0));
    expected.add(Arrays.asList(1.0, 0.0, 0.0, 0.0, 10.0));
    expected.add(Arrays.asList(0.0, 0.0, 0.0, 0.0, 11.0));


    int count = 0;
    while (rr.hasNext()) {
        List<Writable> record = new ArrayList<>(rr.next());
        //            System.out.println(record);
        assertEquals(record.size(), 5);
        List<Double> exp = expected.get(count++);
        for (int j = 0; j < exp.size(); j++) {
            assertEquals(exp.get(j), record.get(j).toDouble(), 0.0);
        }
    }
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void testReadZeroIndexed() throws Exception {
    Configuration conf = new Configuration();
    conf.setInt(LibSvmRecordReader.NUM_FEATURES, 5);
    conf.set(FileRecordReader.APPEND_LABEL, "true");
    conf.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, true);
    RecordReader libSvmRecordReader = new LibSvmRecordReader();
    libSvmRecordReader.initialize(conf, new FileSplit(new ClassPathResource("iris.libsvm").getFile()));

    Configuration confZero = new Configuration();
    confZero.setInt(LibSvmRecordReader.NUM_FEATURES, 4);
    confZero.set(FileRecordReader.APPEND_LABEL, "true");
    confZero.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, true);
    RecordReader libSvmRecordReaderZero = new LibSvmRecordReader();
    libSvmRecordReaderZero.initialize(confZero,
                    new FileSplit(new ClassPathResource("iris_zero_indexed.libsvm").getFile()));

    Configuration confNoZero = new Configuration();
    confNoZero.setInt(LibSvmRecordReader.NUM_FEATURES, 4);
    confNoZero.set(FileRecordReader.APPEND_LABEL, "true");
    confNoZero.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false);
    RecordReader libSvmRecordReaderNoZero = new LibSvmRecordReader();
    libSvmRecordReaderNoZero.initialize(confNoZero, new FileSplit(new ClassPathResource("iris.libsvm").getFile()));

    while (libSvmRecordReader.hasNext()) {
        List<Writable> record = libSvmRecordReader.next();
        record.remove(0);
        List<Writable> recordZero = libSvmRecordReaderZero.next();
        assertEquals(record, recordZero);
        List<Writable> recordNoZero = libSvmRecordReaderNoZero.next();
        assertEquals(record, recordNoZero);
    }
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
private static void testJacksonRecordReader(RecordReader rr) {
	while (rr.hasNext()) {
    	List<Writable> json0 = rr.next();
    	//System.out.println(json0);
    	assert(json0.size() > 0);
	}
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void testSparseData() throws Exception {
    RecordReader svmLightRecordReader = new SVMLightRecordReader();
    Configuration conf = new Configuration();
    conf.set(SVMLightRecordReader.NUM_ATTRIBUTES, "784");
    svmLightRecordReader.initialize(conf, new FileSplit(new ClassPathResource("mnist_svmlight.txt").getFile()));
    assertTrue(svmLightRecordReader.hasNext());
    List<Writable> record = svmLightRecordReader.next();
    assertEquals(785, record.size());
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void testReadZeroIndexed() throws Exception {
    Configuration conf = new Configuration();
    conf.setInt(SVMLightRecordReader.NUM_FEATURES, 5);
    conf.set(FileRecordReader.APPEND_LABEL, "true");
    conf.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, true);
    RecordReader libSvmRecordReader = new SVMLightRecordReader();
    libSvmRecordReader.initialize(conf, new FileSplit(new ClassPathResource("iris.libsvm").getFile()));

    Configuration confZero = new Configuration();
    confZero.setInt(SVMLightRecordReader.NUM_FEATURES, 4);
    confZero.set(FileRecordReader.APPEND_LABEL, "true");
    confZero.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, true);
    RecordReader libSvmRecordReaderZero = new SVMLightRecordReader();
    libSvmRecordReaderZero.initialize(confZero,
                    new FileSplit(new ClassPathResource("iris_zero_indexed.libsvm").getFile()));

    Configuration confNoZero = new Configuration();
    confNoZero.setInt(SVMLightRecordReader.NUM_FEATURES, 4);
    confNoZero.set(FileRecordReader.APPEND_LABEL, "true");
    confNoZero.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false);
    RecordReader libSvmRecordReaderNoZero = new SVMLightRecordReader();
    libSvmRecordReaderNoZero.initialize(confNoZero, new FileSplit(new ClassPathResource("iris.libsvm").getFile()));

    while (libSvmRecordReader.hasNext()) {
        List<Writable> record = libSvmRecordReader.next();
        record.remove(0);
        List<Writable> recordZero = libSvmRecordReaderZero.next();
        assertEquals(record, recordZero);
        List<Writable> recordNoZero = libSvmRecordReaderNoZero.next();
        assertEquals(record, recordNoZero);
    }
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void testLineReader() throws Exception {
    String tempDir = System.getProperty("java.io.tmpdir");
    File tmpdir = new File(tempDir, "tmpdir-testLineReader");
    if (tmpdir.exists())
        tmpdir.delete();
    tmpdir.mkdir();

    File tmp1 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp1.txt"));
    File tmp2 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp2.txt"));
    File tmp3 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp3.txt"));

    FileUtils.writeLines(tmp1, Arrays.asList("1", "2", "3"));
    FileUtils.writeLines(tmp2, Arrays.asList("4", "5", "6"));
    FileUtils.writeLines(tmp3, Arrays.asList("7", "8", "9"));

    InputSplit split = new FileSplit(tmpdir);

    RecordReader reader = new LineRecordReader();
    reader.initialize(split);

    int count = 0;
    List<List<Writable>> list = new ArrayList<>();
    while (reader.hasNext()) {
        List<Writable> l = reader.next();
        assertEquals(1, l.size());
        list.add(l);
        count++;
    }

    assertEquals(9, count);

    try {
        FileUtils.deleteDirectory(tmpdir);
    } catch (Exception e) {
        e.printStackTrace();
    }
}