Java ImageRecordReader类代码示例

import org.datavec.image.recordreader.ImageRecordReader; //导入依赖的package包/类
/**
 * This method returns the iterator. Scales all intensity values: it divides them by 255.
 *
 * @param data the dataset to use
 * @param seed the seed for the random number generator
 * @param batchSize the batch size to use
 * @return the iterator
 * @throws Exception
 */
@Override
public DataSetIterator getDataSetIterator(Instances data, int seed, int batchSize)
    throws Exception {

  batchSize = Math.min(data.numInstances(), batchSize);
  validate(data);
  ImageRecordReader reader = getImageRecordReader(data);

  final int labelIndex = 1; // Use explicit label index position
  final int numPossibleLabels = data.numClasses();
  DataSetIterator tmpIter =
      new RecordReaderDataSetIterator(reader, batchSize, labelIndex, numPossibleLabels);
  DataNormalization scaler = new ImagePreProcessingScaler(0, 1);
  scaler.fit(tmpIter);
  tmpIter.setPreProcessor(scaler);
  return tmpIter;
}
 

import org.datavec.image.recordreader.ImageRecordReader; //导入依赖的package包/类
/** Test */
@Test
public void testGetImageRecordReader() throws Exception {
  final Instances metaData = DatasetLoader.loadMiniMnistMeta();
  Method method =
      ImageInstanceIterator.class.getDeclaredMethod("getImageRecordReader", Instances.class);
  method.setAccessible(true);
  this.idi.setTrainBatchSize(1);
  final ImageRecordReader irr = (ImageRecordReader) method.invoke(this.idi, metaData);

  Set<String> labels = new HashSet<>();
  for (Instance inst : metaData) {
    String label = inst.stringValue(1);
    String itLabel = irr.next().get(1).toString();
    Assert.assertEquals(label, itLabel);
    labels.add(label);
  }
  Assert.assertEquals(10, labels.size());
  Assert.assertTrue(labels.containsAll(irr.getLabels()));
  Assert.assertTrue(irr.getLabels().containsAll(labels));
}
 

import org.datavec.image.recordreader.ImageRecordReader; //导入依赖的package包/类
@Test
public void testLRN() throws Exception {
    List<String> labels = new ArrayList<>(Arrays.asList("Zico", "Ziwang_Xu"));
    String rootDir = new ClassPathResource("lfwtest").getFile().getAbsolutePath();

    RecordReader reader = new ImageRecordReader(28, 28, 3);
    reader.initialize(new FileSplit(new File(rootDir)));
    DataSetIterator recordReader = new RecordReaderDataSetIterator(reader, 10, 1, labels.size());
    labels.remove("lfwtest");
    NeuralNetConfiguration.ListBuilder builder = (NeuralNetConfiguration.ListBuilder) incompleteLRN();
    builder.setInputType(InputType.convolutional(28, 28, 3));

    MultiLayerConfiguration conf = builder.build();

    ConvolutionLayer layer2 = (ConvolutionLayer) conf.getConf(3).getLayer();
    assertEquals(6, layer2.getNIn());

}
 

import org.datavec.image.recordreader.ImageRecordReader; //导入依赖的package包/类
private void check(BufferedImage image) throws Exception
{
    ImageIO.write(image, "png", new File("tmp.png")); //saves the image to the tmp.png file
    ImageRecordReader reader = new ImageRecordReader(150, 150, 3);
    reader.initialize(new FileSplit(new File("tmp.png"))); //reads the tmp.png file
    DataSetIterator dataIter = new RecordReaderDataSetIterator(reader, 1);
    while (dataIter.hasNext())
    {
        //Normalize the data from the file
        DataNormalization normalization = new NormalizerMinMaxScaler();
        DataSet set = dataIter.next();
        normalization.fit(set);
        normalization.transform(set);

        INDArray array = MainGUI.model.output(set.getFeatures(), false); //send the data to the model and get the results

        //Process the results and print them in an understandable format (percentage scores)
        String txt = "";

        DecimalFormat df = new DecimalFormat("#.00");

        for (int i = 0; i < array.length(); i++)
        {
            txt += MainGUI.labels.get(i) + ": " + (array.getDouble(i)*100 < 1 ? "0" : "") + df.format((array.getDouble(i)*100)) + "%\n";
        }

        probabilityArea.setText(txt);
    }

    reader.close();
}
 

import org.datavec.image.recordreader.ImageRecordReader; //导入依赖的package包/类
private DataSetIterator createInternal(InputSplit inputSplit) throws IOException {
    ImageTransform imageTransform = imageTransformFactory.create();
    int width = imageTransformConfigurationResource.getScaledWidth();
    int height = imageTransformConfigurationResource.getScaledHeight();
    int channels = imageTransformConfigurationResource.getChannels();
    int batchSize = networkConfigurationResource.getBatchSize();
    int outputs = networkConfigurationResource.getOutputs();
    ImageRecordReader recordReader = new ImageRecordReader(height, width, channels, pathLabelGenerator);
    recordReader.initialize(inputSplit, imageTransform);
    RecordReaderDataSetIterator recordReaderDataSetIterator = new RecordReaderDataSetIterator(recordReader, batchSize, 1, outputs);
    DataNormalization scaler = new ImagePreProcessingScaler(0, 1);
    scaler.fit(recordReaderDataSetIterator);
    recordReaderDataSetIterator.setPreProcessor(scaler);
    return recordReaderDataSetIterator;
}
 

import org.datavec.image.recordreader.ImageRecordReader; //导入依赖的package包/类
@Override
protected ImageRecordReader getImageRecordReader(Instances data) throws Exception {
  ArffMetaDataLabelGenerator labelGenerator =
      new ArffMetaDataLabelGenerator(data, getImagesLocation().toString());
  ResizeImageTransform rit = new ResizeImageTransform(width, height);
  ImageRecordReader reader =
      new ImageRecordReader(height, width, getNumChannels(), labelGenerator, rit);
  CollectionInputSplit cis = new CollectionInputSplit(labelGenerator.getPathURIs());
  reader.initialize(cis);
  return reader;
}
 

import org.datavec.image.recordreader.ImageRecordReader; //导入依赖的package包/类
/**
 * Returns the image recorder.
 *
 * @param data the dataset to use
 * @return the image recorder
 * @throws Exception
 */
protected ImageRecordReader getImageRecordReader(Instances data) throws Exception {
  ArffMetaDataLabelGenerator labelGenerator =
      new ArffMetaDataLabelGenerator(data, getImagesLocation().toString());
  ImageRecordReader reader =
      new ImageRecordReader(getHeight(), getWidth(), getNumChannels(), labelGenerator);
  CollectionInputSplit cis = new CollectionInputSplit(labelGenerator.getPathURIs());
  reader.initialize(cis);

  return reader;
}
 

import org.datavec.image.recordreader.ImageRecordReader; //导入依赖的package包/类
public RecordReader getRecordReader(int batchSize, int numExamples, int[] imgDim, int numLabels,
                PathLabelGenerator labelGenerator, boolean train, double splitTrainTest, Random rng) {
    load(batchSize, numExamples, numLabels, labelGenerator, splitTrainTest, rng);
    RecordReader recordReader =
                    new ImageRecordReader(imgDim[0], imgDim[1], imgDim[2], labelGenerator, imageTransform);

    try {
        InputSplit data = train ? inputSplit[0] : inputSplit[1];
        recordReader.initialize(data);
    } catch (IOException | InterruptedException e) {
        e.printStackTrace();
    }
    return recordReader;
}
 

import org.datavec.image.recordreader.ImageRecordReader; //导入依赖的package包/类
@Test
public void testLfwReader() throws Exception {
    String subDir = "lfw-a/lfw";
    File path = new File(FilenameUtils.concat(System.getProperty("user.home"), subDir));
    FileSplit fileSplit = new FileSplit(path, LFWLoader.ALLOWED_FORMATS, new Random(42));
    BalancedPathFilter pathFilter = new BalancedPathFilter(new Random(42), LFWLoader.LABEL_PATTERN, 1, 1, 1);
    InputSplit[] inputSplit = fileSplit.sample(pathFilter, 1);
    RecordReader rr = new ImageRecordReader(250, 250, 3, LFWLoader.LABEL_PATTERN);
    rr.initialize(inputSplit[0]);
    List<String> exptedLabel = rr.getLabels();

    RecordReader rr2 = new LFWLoader(new int[] {250, 250, 3}, true).getRecordReader(1, 1, 1, new Random(42));

    assertEquals(exptedLabel.get(0), rr2.getLabels().get(0));
}
 

import org.datavec.image.recordreader.ImageRecordReader; //导入依赖的package包/类
@Override
public RecordReader createReader(InputSplit split, Configuration conf) throws IOException, InterruptedException {
    RecordReader reader = new ImageRecordReader();
    reader.initialize(conf, split);
    return reader;
}
 

import org.datavec.image.recordreader.ImageRecordReader; //导入依赖的package包/类
@Test
public void testImagesRRDMSI() throws Exception {
    File parentDir = Files.createTempDir();
    parentDir.deleteOnExit();
    String str1 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Zico/");
    String str2 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Ziwang_Xu/");

    File f1 = new File(str1);
    File f2 = new File(str2);
    f1.mkdirs();
    f2.mkdirs();

    writeStreamToFile(new File(FilenameUtils.concat(f1.getPath(), "Zico_0001.jpg")),
                    new ClassPathResource("lfwtest/Zico/Zico_0001.jpg").getInputStream());
    writeStreamToFile(new File(FilenameUtils.concat(f2.getPath(), "Ziwang_Xu_0001.jpg")),
                    new ClassPathResource("lfwtest/Ziwang_Xu/Ziwang_Xu_0001.jpg").getInputStream());


    int outputNum = 2;
    Random r = new Random(12345);
    ParentPathLabelGenerator labelMaker = new ParentPathLabelGenerator();

    ImageRecordReader rr1 = new ImageRecordReader(10, 10, 1, labelMaker);
    ImageRecordReader rr1s = new ImageRecordReader(5, 5, 1, labelMaker);

    rr1.initialize(new FileSplit(parentDir));
    rr1s.initialize(new FileSplit(parentDir));


    MultiDataSetIterator trainDataIterator = new RecordReaderMultiDataSetIterator.Builder(1).addReader("rr1", rr1)
                    .addReader("rr1s", rr1s).addInput("rr1", 0, 0).addInput("rr1s", 0, 0)
                    .addOutputOneHot("rr1s", 1, outputNum).build();

    //Now, do the same thing with ImageRecordReader, and check we get the same results:
    ImageRecordReader rr1_b = new ImageRecordReader(10, 10, 1, labelMaker);
    ImageRecordReader rr1s_b = new ImageRecordReader(5, 5, 1, labelMaker);
    rr1_b.initialize(new FileSplit(parentDir));
    rr1s_b.initialize(new FileSplit(parentDir));

    DataSetIterator dsi1 = new RecordReaderDataSetIterator(rr1_b, 1, 1, 2);
    DataSetIterator dsi2 = new RecordReaderDataSetIterator(rr1s_b, 1, 1, 2);

    for (int i = 0; i < 2; i++) {
        MultiDataSet mds = trainDataIterator.next();

        DataSet d1 = dsi1.next();
        DataSet d2 = dsi2.next();

        assertEquals(d1.getFeatureMatrix(), mds.getFeatures(0));
        assertEquals(d2.getFeatureMatrix(), mds.getFeatures(1));
        assertEquals(d1.getLabels(), mds.getLabels(0));
    }
}
 

import org.datavec.image.recordreader.ImageRecordReader; //导入依赖的package包/类
@Test
public void testImagesRRDMSI_Batched() throws Exception {
    File parentDir = Files.createTempDir();
    parentDir.deleteOnExit();
    String str1 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Zico/");
    String str2 = FilenameUtils.concat(parentDir.getAbsolutePath(), "Ziwang_Xu/");

    File f1 = new File(str1);
    File f2 = new File(str2);
    f1.mkdirs();
    f2.mkdirs();

    writeStreamToFile(new File(FilenameUtils.concat(f1.getPath(), "Zico_0001.jpg")),
                    new ClassPathResource("lfwtest/Zico/Zico_0001.jpg").getInputStream());
    writeStreamToFile(new File(FilenameUtils.concat(f2.getPath(), "Ziwang_Xu_0001.jpg")),
                    new ClassPathResource("lfwtest/Ziwang_Xu/Ziwang_Xu_0001.jpg").getInputStream());

    int outputNum = 2;
    ParentPathLabelGenerator labelMaker = new ParentPathLabelGenerator();

    ImageRecordReader rr1 = new ImageRecordReader(10, 10, 1, labelMaker);
    ImageRecordReader rr1s = new ImageRecordReader(5, 5, 1, labelMaker);

    URI[] uris = new FileSplit(parentDir).locations();

    rr1.initialize(new CollectionInputSplit(uris));
    rr1s.initialize(new CollectionInputSplit(uris));

    MultiDataSetIterator trainDataIterator = new RecordReaderMultiDataSetIterator.Builder(2).addReader("rr1", rr1)
                    .addReader("rr1s", rr1s).addInput("rr1", 0, 0).addInput("rr1s", 0, 0)
                    .addOutputOneHot("rr1s", 1, outputNum).build();

    //Now, do the same thing with ImageRecordReader, and check we get the same results:
    ImageRecordReader rr1_b = new ImageRecordReader(10, 10, 1, labelMaker);
    ImageRecordReader rr1s_b = new ImageRecordReader(5, 5, 1, labelMaker);
    rr1_b.initialize(new FileSplit(parentDir));
    rr1s_b.initialize(new FileSplit(parentDir));

    DataSetIterator dsi1 = new RecordReaderDataSetIterator(rr1_b, 2, 1, 2);
    DataSetIterator dsi2 = new RecordReaderDataSetIterator(rr1s_b, 2, 1, 2);

    MultiDataSet mds = trainDataIterator.next();

    DataSet d1 = dsi1.next();
    DataSet d2 = dsi2.next();

    assertEquals(d1.getFeatureMatrix(), mds.getFeatures(0));
    assertEquals(d2.getFeatureMatrix(), mds.getFeatures(1));
    assertEquals(d1.getLabels(), mds.getLabels(0));

    //Check label assignment:

    File currentFile = rr1_b.getCurrentFile();
    INDArray expLabels;
    if(currentFile.getAbsolutePath().contains("Zico")){
        expLabels = Nd4j.create(new double[][] {{0, 1}, {1, 0}});
    } else {
        expLabels = Nd4j.create(new double[][] {{1, 0}, {0, 1}});
    }

    assertEquals(expLabels, d1.getLabels());
    assertEquals(expLabels, d2.getLabels());
}