Python iterators.SerialIterator方法代码示例

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def test_iterator_repeat(self):
        dataset = [1, 2, 3, 4, 5, 6]
        it = iterators.SerialIterator(dataset, 2, shuffle=False)
        for i in range(3):
            self.assertEqual(it.epoch, i)
            self.assertAlmostEqual(it.epoch_detail, i + 0 / 6)
            if i == 0:
                self.assertIsNone(it.previous_epoch_detail)
            else:
                self.assertAlmostEqual(it.previous_epoch_detail, i - 2 / 6)
            self.assertEqual(it.next(), [1, 2])
            self.assertFalse(it.is_new_epoch)
            self.assertAlmostEqual(it.epoch_detail, i + 2 / 6)
            self.assertAlmostEqual(it.previous_epoch_detail, i + 0 / 6)
            self.assertEqual(it.next(), [3, 4])
            self.assertFalse(it.is_new_epoch)
            self.assertAlmostEqual(it.epoch_detail, i + 4 / 6)
            self.assertAlmostEqual(it.previous_epoch_detail, i + 2 / 6)
            self.assertEqual(it.next(), [5, 6])
            self.assertTrue(it.is_new_epoch)
            self.assertAlmostEqual(it.epoch_detail, i + 6 / 6)
            self.assertAlmostEqual(it.previous_epoch_detail, i + 4 / 6) 

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def test_iterator_not_repeat(self):
        dataset = [1, 2, 3, 4, 5, 6]
        it = iterators.SerialIterator(dataset, 2, repeat=False, shuffle=False)

        self.assertAlmostEqual(it.epoch_detail, 0 / 6)
        self.assertIsNone(it.previous_epoch_detail)
        self.assertEqual(it.next(), [1, 2])
        self.assertAlmostEqual(it.epoch_detail, 2 / 6)
        self.assertAlmostEqual(it.previous_epoch_detail, 0 / 6)
        self.assertEqual(it.next(), [3, 4])
        self.assertAlmostEqual(it.epoch_detail, 4 / 6)
        self.assertAlmostEqual(it.previous_epoch_detail, 2 / 6)
        self.assertEqual(it.next(), [5, 6])
        self.assertTrue(it.is_new_epoch)
        self.assertEqual(it.epoch, 1)
        self.assertAlmostEqual(it.epoch_detail, 6 / 6)
        self.assertAlmostEqual(it.previous_epoch_detail, 4 / 6)
        for i in range(2):
            self.assertRaises(StopIteration, it.next) 

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def test_iterator_not_repeat_not_even(self):
        dataset = [1, 2, 3, 4, 5]
        it = iterators.SerialIterator(dataset, 2, repeat=False, shuffle=False)

        self.assertAlmostEqual(it.epoch_detail, 0 / 5)
        self.assertIsNone(it.previous_epoch_detail)
        self.assertEqual(it.next(), [1, 2])
        self.assertAlmostEqual(it.epoch_detail, 2 / 5)
        self.assertAlmostEqual(it.previous_epoch_detail, 0 / 5)
        self.assertEqual(it.next(), [3, 4])
        self.assertAlmostEqual(it.epoch_detail, 4 / 5)
        self.assertAlmostEqual(it.previous_epoch_detail, 2 / 5)
        self.assertEqual(it.next(), [5])
        self.assertTrue(it.is_new_epoch)
        self.assertEqual(it.epoch, 1)
        self.assertAlmostEqual(it.epoch_detail, 5 / 5)
        self.assertAlmostEqual(it.previous_epoch_detail, 4 / 5)
        self.assertRaises(StopIteration, it.next) 

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def check_custom(comm, length, bs):
    assert bs > 0
    assert length > 0
    a = list(range(comm.rank, length, comm.size))
    b = list(range(comm.rank, length, comm.size))
    c = list(range(comm.rank, length, comm.size))

    model = ExampleModel()
    dataset = TupleDataset(a, b, c)
    iterator = SerialIterator(dataset, bs, shuffle=False, repeat=False)
    evaluator = CustomMultiNodeEvaluator(comm, iterator, model)

    result = evaluator(None)

    iterator.reset()
    expected = comm.allreduce_obj(sum(2 for batch in iterator))

    if comm.rank == 0:
        assert expected == result

    else:
        assert result is None 

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def setUp(self):
        bboxes = [generate_random_bbox(5, (256, 324), 24, 120)
                  for _ in range(10)]
        areas = [[np.array([(bb[2] - bb[0]) * bb[3] - bb[0]]) for bb in bbox]
                 for bbox in bboxes]
        labels = 2 * np.ones((10, 5), dtype=np.int32)
        crowdeds = np.zeros((10, 5))
        self.dataset = TupleDataset(
            np.random.uniform(size=(10, 3, 32, 48)),
            bboxes, labels, areas, crowdeds)
        self.link = _DetectionStubLink(bboxes, labels)
        self.iterator = SerialIterator(
            self.dataset, 5, repeat=False, shuffle=False)
        self.evaluator = DetectionCOCOEvaluator(
            self.iterator, self.link, label_names=('cls0', 'cls1', 'cls2'))
        self.expected_ap = 1 

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def setUp(self):
        self.label_names = ('a', 'b', 'c')
        imgs = np.random.uniform(size=(1, 3, 2, 3))
        # There are labels for 'a' and 'b', but none for 'c'.
        pred_labels = np.array([[[1, 1, 1], [0, 0, 1]]])
        gt_labels = np.array([[[1, 0, 0], [0, -1, 1]]])

        self.iou_a = 1 / 3
        self.iou_b = 2 / 4
        self.pixel_accuracy = 3 / 5
        self.class_accuracy_a = 1 / 3
        self.class_accuracy_b = 2 / 2
        self.miou = np.mean((self.iou_a, self.iou_b))
        self.mean_class_accuracy = np.mean(
            (self.class_accuracy_a, self.class_accuracy_b))

        self.dataset = TupleDataset(imgs, gt_labels)
        self.link = _SemanticSegmentationStubLink(pred_labels)
        self.iterator = SerialIterator(
            self.dataset, 5, repeat=False, shuffle=False)
        self.evaluator = SemanticSegmentationEvaluator(
            self.iterator, self.link, self.label_names) 

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def test_apply_to_iterator_with_infinite_iterator(self):
        def func(*in_values):
            n_sample = len(in_values[0])
            return [np.random.uniform(size=(48, 64)) for _ in range(n_sample)]

        dataset = []
        for _ in range(5):
            H, W = np.random.randint(8, 16, size=2)
            dataset.append(np.random.randint(0, 256, size=(3, H, W)))

        iterator = SerialIterator(dataset, 2)

        in_values, out_values, rest_values = apply_to_iterator(func, iterator)

        for _ in range(10):
            next(in_values[0])

        for _ in range(10):
            next(out_values[0]) 

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def _test_r2_score_evaluator(inputs):
    predictor = DummyPredictor()
    x0, x1, _ = inputs
    dataset = NumpyTupleDataset(x0, x1)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = R2ScoreEvaluator(iterator, predictor, name='train')
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    expected = r2_score(x0, x1)
    pytest.approx(observation['target/r2_score'], expected)

    # --- test __call__ ---
    result = evaluator()
    pytest.approx(result['train/main/r2_score'], expected) 

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def _test_r2_score_evaluator_ignore_nan(inputs):
    predictor = DummyPredictor()
    x0, _, x2 = inputs
    dataset = NumpyTupleDataset(x0, x2)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = R2ScoreEvaluator(
        iterator, predictor, name='train', ignore_nan=True)
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    expected = r2_score(x0, x2, ignore_nan=True)
    pytest.approx(observation['target/r2_score'], expected)

    # --- test __call__ ---
    result = evaluator()
    pytest.approx(result['train/main/r2_score'], expected) 

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def _test_r2_score_evaluator_ignore_nan_with_nonnan_value(inputs):
    predictor = DummyPredictor()
    x0, x1, _ = inputs
    dataset = NumpyTupleDataset(x0, x1)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = R2ScoreEvaluator(
        iterator, predictor, name='train', ignore_nan=True)
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    expected = r2_score(x0, x1, ignore_nan=True)
    pytest.approx(observation['target/r2_score'], expected)

    # --- test __call__ ---
    result = evaluator()
    pytest.approx(result['train/main/r2_score'], expected) 

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def _test_prc_auc_evaluator_default_args(data0):

    predictor = DummyPredictor()
    dataset = NumpyTupleDataset(*data0)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = PRCAUCEvaluator(
        iterator, predictor, name='train',
        pos_labels=1, ignore_labels=None
    )
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    expected_prc_auc = 0.7916
    pytest.approx(observation['target/prc_auc'], expected_prc_auc)

    # --- test __call__ ---
    result = evaluator()
    pytest.approx(result['train/main/prc_auc'], expected_prc_auc) 

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def _test_prc_auc_evaluator_with_labels(data1):
    """test `pos_labels` and `ignore_labels` behavior"""

    predictor = DummyPredictor()
    dataset = NumpyTupleDataset(*data1)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = PRCAUCEvaluator(
        iterator, predictor, name='val',
        pos_labels=[1, 2], ignore_labels=-1,
    )

    # --- test evaluate ---
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    expected_prc_auc = 0.7916
    pytest.approx(observation['target/prc_auc'], expected_prc_auc)

    # --- test __call__ ---
    result = evaluator()
    pytest.approx(result['val/main/prc_auc'], expected_prc_auc) 

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def _test_prc_auc_evaluator_raise_error(data, raise_value_error=True):

    predictor = DummyPredictor()
    dataset = NumpyTupleDataset(*data)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = PRCAUCEvaluator(
        iterator, predictor, name='train',
        pos_labels=1, ignore_labels=None,
        raise_value_error=raise_value_error
    )
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    return observation['target/prc_auc'] 

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def _test_roc_auc_evaluator_default_args(data0):

    predictor = DummyPredictor()
    dataset = NumpyTupleDataset(*data0)

    iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False)
    evaluator = ROCAUCEvaluator(
        iterator, predictor, name='train',
        pos_labels=1, ignore_labels=None
    )
    repo = chainer.Reporter()
    repo.add_observer('target', predictor)
    with repo:
        observation = evaluator.evaluate()

    expected_roc_auc = 0.75
    # print('observation ', observation)
    assert observation['target/roc_auc'] == expected_roc_auc

    # --- test __call__ ---
    result = evaluator()
    # print('result ', result)
    assert result['train/main/roc_auc'] == expected_roc_auc 

# 需要导入模块: from chainer import iterators [as 别名]
# 或者: from chainer.iterators import SerialIterator [as 别名]
def train(network, loss, X_tr, Y_tr, X_te, Y_te, n_epochs=30, gamma=1):
    model= Objective(network, loss=loss, gamma=gamma)

    #optimizer = optimizers.SGD()
    optimizer = optimizers.Adam()
    optimizer.setup(model)

    train = tuple_dataset.TupleDataset(X_tr, Y_tr)
    test = tuple_dataset.TupleDataset(X_te, Y_te)

    train_iter = iterators.SerialIterator(train, batch_size=1, shuffle=True)
    test_iter = iterators.SerialIterator(test, batch_size=1, repeat=False,
                                         shuffle=False)
    updater = training.StandardUpdater(train_iter, optimizer)
    trainer = training.Trainer(updater, (n_epochs, 'epoch'))

    trainer.run()