Java RecordReader.hasNext方法代码示例

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void test() throws Exception {

    CSVRecordReader rr = new CSVRecordReader(0, ',');
    rr.initialize(new FileSplit(new ClassPathResource("iris.dat").getFile()));

    CSVRecordReader rr2 = new CSVRecordReader(0, ',');
    rr2.initialize(new FileSplit(new ClassPathResource("iris.dat").getFile()));

    RecordReader rrC = new ConcatenatingRecordReader(rr, rr2);

    int count = 0;
    while(rrC.hasNext()){
        rrC.next();
        count++;
    }

    assertEquals(300, count);
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void testCsvRecordReader() throws Exception {
    SerializerInstance si = sc.env().serializer().newInstance();
    assertTrue(si instanceof KryoSerializerInstance);

    RecordReader r1 = new CSVRecordReader(1,'\t');
    RecordReader r2 = serDe(r1, si);

    File f = new ClassPathResource("iris_tab_delim.txt").getFile();
    r1.initialize(new FileSplit(f));
    r2.initialize(new FileSplit(f));

    while(r1.hasNext()){
        assertEquals(r1.next(), r2.next());
    }
    assertFalse(r2.hasNext());
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void testCsvRRSerializationResults() throws Exception {
    int skipLines = 3;
    RecordReader r1 = new CSVRecordReader(skipLines, '\t');
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    ObjectOutputStream os = new ObjectOutputStream(baos);
    os.writeObject(r1);
    byte[] bytes = baos.toByteArray();
    ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes));
    RecordReader r2 = (RecordReader) ois.readObject();

    File f = new ClassPathResource("iris_tab_delim.txt").getFile();

    r1.initialize(new FileSplit(f));
    r2.initialize(new FileSplit(f));

    int count = 0;
    while(r1.hasNext()){
        List<Writable> n1 = r1.next();
        List<Writable> n2 = r2.next();
        assertEquals(n1, n2);
        count++;
    }

    assertEquals(150-skipLines, count);
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Override
public void fit(RecordReader reader, RecordCallBack callBack) {
    while (reader.hasNext()) {
        Record record = reader.nextRecord();
        String s = toString(record.getRecord());
        Tokenizer tokenizer = tokenizerFactory.create(s);
        cache.incrementNumDocs(1);
        doWithTokens(tokenizer);
        if (callBack != null)
            callBack.onRecord(record);


    }
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Override
protected int poll() throws Exception {
    Exchange exchange = endpoint.createExchange();
    InputSplit split = inputFromExchange(exchange);
    RecordReader reader = inputFormat.createReader(split, configuration);
    int numMessagesPolled = 0;
    while (reader.hasNext()) {
        // create a message body
        while (reader.hasNext()) {
            exchange.getIn().setBody(reader.next());

            try {
                // send message to next processor in the route
                getProcessor().process(exchange);
                numMessagesPolled++; // number of messages polled
            } finally {
                // log exception if an exception occurred and was not handled
                if (exchange.getException() != null) {
                    getExceptionHandler().handleException("Error processing exchange", exchange,
                                    exchange.getException());
                }
            }
        }


    }

    return numMessagesPolled;
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void testDataVec() throws Exception {
    MockEndpoint mock = getMockEndpoint("mock:result");
    //1
    mock.expectedMessageCount(1);

    RecordReader reader = new CSVRecordReader();
    reader.initialize(new FileSplit(new ClassPathResource("iris.dat").getFile()));
    Collection<Collection<Writable>> recordAssertion = new ArrayList<>();
    while (reader.hasNext())
        recordAssertion.add(reader.next());
    mock.expectedBodiesReceived(recordAssertion);
    assertMockEndpointsSatisfied();
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
/**
 * Write all values from the specified record reader to the specified record writer.
 * Optionally, close the record writer on completion
 *
 * @param reader Record reader (source of data)
 * @param writer Record writer (location to write data)
 * @param closeOnCompletion if true: close the record writer once complete, via {@link RecordWriter#close()}
 * @throws IOException If underlying reader/writer throws an exception
 */
public static void convert(RecordReader reader, RecordWriter writer, boolean closeOnCompletion) throws IOException {

    if(!reader.hasNext()){
        throw new UnsupportedOperationException("Cannot convert RecordReader: reader has no next element");
    }

    while(reader.hasNext()){
        writer.write(reader.next());
    }

    if(closeOnCompletion){
        writer.close();
    }
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Override
public List<Writable> next() {
    List<Writable> out = null;
    for( RecordReader rr : readers){
        if(rr.hasNext()){
            out = rr.next();
            break;
        }
    }
    invokeListeners(out);
    return out;
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Override
public boolean hasNext() {
    for (RecordReader reader : readers) {
        if(reader.hasNext()){
            return true;
        }
    }
    return false;
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Override
public boolean hasNext() {
    boolean readersHasNext = true;
    for (RecordReader reader : readers) {
        readersHasNext = readersHasNext && reader.hasNext();
    }
    return readersHasNext;
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void testAppendingLabelsMetaData() throws Exception {
    ClassPathResource cpr = new ClassPathResource("json/json_test_0.txt");
    String path = cpr.getFile().getAbsolutePath().replace("0", "%d");

    InputSplit is = new NumberedFileInputSplit(path, 0, 2);

    //Insert at the end:
    RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1,
                    new LabelGen());
    rr.initialize(is);

    List<List<Writable>> out = new ArrayList<>();
    while (rr.hasNext()) {
        out.add(rr.next());
    }
    assertEquals(3, out.size());

    rr.reset();

    List<List<Writable>> out2 = new ArrayList<>();
    List<Record> outRecord = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    while (rr.hasNext()) {
        Record r = rr.nextRecord();
        out2.add(r.getRecord());
        outRecord.add(r);
        meta.add(r.getMetaData());
    }

    assertEquals(out, out2);

    List<Record> fromMeta = rr.loadFromMetaData(meta);
    assertEquals(outRecord, fromMeta);
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
    public void testReadWrite() throws Exception {
        Configuration conf = new Configuration();
        conf.set(FileRecordReader.APPEND_LABEL, "true");
        conf.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false);
        conf.setInt(LibSvmRecordReader.NUM_FEATURES, 4);
        File out = new File("iris.libsvm.out");
        if (out.exists())
            out.delete();
//        conf.set(FileRecordWriter.PATH, out.getAbsolutePath()); // setting output path via config not safe
        RecordReader libSvmRecordReader = new LibSvmRecordReader();
        libSvmRecordReader.initialize(conf, new FileSplit(new ClassPathResource("iris.libsvm").getFile()));

        RecordWriter writer = new LibSvmRecordWriter(out, true);
        writer.setConf(conf);
        List<List<Writable>> data = new ArrayList<>();
        while (libSvmRecordReader.hasNext()) {
            List<Writable> record = libSvmRecordReader.next();
            writer.write(record);
            data.add(record);
        }
        writer.close();

        out.deleteOnExit();
        List<List<Writable>> test = new ArrayList<>();
        RecordReader testLibSvmRecordReader = new LibSvmRecordReader();
        testLibSvmRecordReader.initialize(conf, new FileSplit(out));
        while (testLibSvmRecordReader.hasNext())
            test.add(testLibSvmRecordReader.next());
        assertEquals(data, test);
    }
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void testReadMissing() throws Exception {
    RecordReader rr = new LibSvmRecordReader();
    Configuration conf = new Configuration();
    conf.set(LineRecordReader.APPEND_LABEL, "true");
    conf.set(LibSvmRecordReader.NUM_FEATURES, "4");
    conf.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false);
    rr.initialize(conf, new FileSplit(new ClassPathResource("libsvm_with_multiple_missing.libsvm").getFile()));

    List<List<Double>> expected = new ArrayList<>(11);
    expected.add(Arrays.asList(1.0, 2.0, 3.0, 4.0, 1.0));
    expected.add(Arrays.asList(0.0, 2.0, 3.0, 4.0, 2.0));
    expected.add(Arrays.asList(1.0, 0.0, 3.0, 4.0, 3.0));
    expected.add(Arrays.asList(1.0, 2.0, 0.0, 4.0, 4.0));
    expected.add(Arrays.asList(1.0, 2.0, 3.0, 0.0, 5.0));
    expected.add(Arrays.asList(0.0, 0.0, 3.0, 4.0, 6.0));
    expected.add(Arrays.asList(1.0, 0.0, 0.0, 4.0, 7.0));
    expected.add(Arrays.asList(1.0, 2.0, 0.0, 0.0, 8.0));
    expected.add(Arrays.asList(0.0, 0.0, 0.0, 4.0, 9.0));
    expected.add(Arrays.asList(1.0, 0.0, 0.0, 0.0, 10.0));
    expected.add(Arrays.asList(0.0, 0.0, 0.0, 0.0, 11.0));


    int count = 0;
    while (rr.hasNext()) {
        List<Writable> record = new ArrayList<>(rr.next());
        //            System.out.println(record);
        assertEquals(record.size(), 5);
        List<Double> exp = expected.get(count++);
        for (int j = 0; j < exp.size(); j++) {
            assertEquals(exp.get(j), record.get(j).toDouble(), 0.0);
        }
    }
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void testRealData() throws Exception {
    RecordReader rr = new LibSvmRecordReader();
    Configuration conf = new Configuration();
    conf.setInt(LibSvmRecordReader.NUM_FEATURES, 5);
    conf.set(LineRecordReader.APPEND_LABEL, "true");
    rr.initialize(conf, new FileSplit(new ClassPathResource("iris.libsvm").getFile()));

    int count = 0;
    while (rr.hasNext()) {
        count++;
        System.out.println(rr.next());
    }
    System.out.println(count);
}
 

import org.datavec.api.records.reader.RecordReader; //导入方法依赖的package包/类
@Test
public void testRegexLineRecordReaderMeta() throws Exception {
    String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)";

    RecordReader rr = new RegexLineRecordReader(regex, 1);
    rr.initialize(new FileSplit(new ClassPathResource("/logtestdata/logtestfile0.txt").getFile()));

    List<List<Writable>> list = new ArrayList<>();
    while (rr.hasNext()) {
        list.add(rr.next());
    }
    assertEquals(3, list.size());

    List<Record> list2 = new ArrayList<>();
    List<List<Writable>> list3 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    rr.reset();
    int count = 1; //Start by skipping 1 line
    while (rr.hasNext()) {
        Record r = rr.nextRecord();
        list2.add(r);
        list3.add(r.getRecord());
        meta.add(r.getMetaData());

        assertEquals(count++, ((RecordMetaDataLine) r.getMetaData()).getLineNumber());
    }

    List<Record> fromMeta = rr.loadFromMetaData(meta);

    assertEquals(list, list3);
    assertEquals(list2, fromMeta);
}