Python container.ContentContainer类代码示例

 def _update_units(self, request, unit_inventory):
     """
     Update units that have been updated on the parent since
     added or last updated in the child inventory.
     :param request: A synchronization request.
     :type request: SyncRequest
     :param unit_inventory: The inventory of both parent and child content units.
     :type unit_inventory: UnitInventory
     """
     download_list = []
     units = unit_inventory.updated_units()
     listener = ContentDownloadListener(self, request)
     for unit, unit_ref in units:
         storage_path = unit[constants.STORAGE_PATH]
         if storage_path:
             self._reset_storage_path(unit)
             unit_url, destination = self._url_and_destination(unit_inventory.base_URL, unit)
             _request = listener.create_request(unit_url, destination, unit, unit_ref)
             download_list.append(_request)
         else:
             unit = unit_ref.fetch()
             self.add_unit(request, unit)
     if not download_list:
         return
     container = ContentContainer()
     request.summary.sources = container.download(
         request.downloader,
         download_list,
         listener)
     request.summary.errors.extend(listener.error_list)

    def test_download_to_stream(self):
        request_list = []
        _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10)
        _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
        _dir = self.populate_content(PRIMARY, 0, 20)
        # unit-world
        for n in range(0, 10):
            request = Request(
                cataloged[n].type_id,
                cataloged[n].unit_key,
                'file://%s/unit_%d' % (_dir, n),
                StringIO())
            request_list.append(request)
        # primary
        for n in range(11, 20):
            unit_key = {
                'name': 'unit_%d' % n,
                'version': '1.0.%d' % n,
                'release': '1',
                'checksum': str(uuid4())
            }
            request = Request(
                TYPE_ID,
                unit_key,
                'file://%s/unit_%d' % (_dir, n),
                StringIO())
            request_list.append(request)
        downloader = LocalFileDownloader(DownloaderConfig())
        listener = Mock()
        container = ContentContainer(path=self.tmp_dir)
        container.threaded = False
        container.refresh = Mock()

        # test
        report = container.download(downloader, request_list, listener)

        # validation
        # unit-world
        for i in range(0, 10):
            request = request_list[i]
            self.assertTrue(request.downloaded)
            self.assertEqual(len(request.errors), 0)
            fp = request.destination
            s = fp.getvalue()
            self.assertTrue(UNIT_WORLD in s)
        # primary
        for i in range(11, len(request_list)):
            request = request_list[i]
            self.assertTrue(request.downloaded)
            self.assertEqual(len(request.errors), 0)
            fp = request.destination
            s = fp.getvalue()
            self.assertTrue(PRIMARY in s)
        self.assertEqual(report.total_sources, 2)
        self.assertEqual(len(report.downloads), 2)
        self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9)
        self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
        self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 10)
        self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 0)

    def test_download(self, fake_load):
        sources = []
        for n in range(3):
            s = ContentSource('s-%d' % n, {})
            s.get_downloader = Mock()
            sources.append(s)

        fake_load.return_value = sources

        request_list = []
        for n in range(6):
            r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
            r.find_sources = Mock(return_value=sources[n % 3:])
            request_list.append(r)

        collated = [
            {
                sources[0]: ['nectar-1'],
                sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
                sources[2]: ['nectar-5', 'nectar-6']
            },
            {}
        ]
        fake_collated = Mock(side_effect=collated)

        fake_listener = Mock()
        canceled = FakeEvent()
        fake_primary = PrimarySource(Mock())

        # test
        container = ContentContainer('')
        container.refresh = Mock()
        container.collated = fake_collated
        report = container.download(canceled, fake_primary, request_list, fake_listener)

        # validation
        container.refresh.assert_called_with(canceled)

        for r in request_list:
            r.find_sources.assert_called_with(fake_primary, container.sources)

        self.assertEqual(report.total_passes, 1)
        self.assertEqual(report.total_sources, len(sources))
        self.assertEqual(len(report.downloads), 3)
        for source in sources:
            self.assertEqual(report.downloads[source.id].total_succeeded, 0)
            self.assertEqual(report.downloads[source.id].total_failed, 0)

        for source in sources:
            source.get_downloader.assert_called_with()
            downloader = source.get_downloader()
            listener = downloader.event_listener
            self.assertEqual(listener.cancel_event, canceled)
            self.assertEqual(listener.downloader, downloader)
            self.assertEqual(listener.listener, fake_listener)
            downloader.download.assert_called_with(collated[0][source])

    def test_purge_orphans(self, fake_manager, fake_load):
        fake_load.return_value = {"A": 1, "B": 2, "C": 3}

        # test
        container = ContentContainer("")

        # validation
        container.purge_orphans()

        fake_manager().purge_orphans.assert_called_with(fake_load.return_value.keys())

    def test_purge_orphans(self, fake_manager, fake_load):
        fake_load.return_value = {'A': 1, 'B': 2, 'C': 3}

        # test
        container = ContentContainer('')

        # validation
        container.purge_orphans()

        fake_manager().purge_orphans.assert_called_with(fake_load.return_value.keys())

    def test_download_canceled_after_collated(self, fake_load):
        sources = []
        for n in range(3):
            s = ContentSource('s-%d' % n, {})
            s.get_downloader = Mock()
            sources.append(s)

        fake_load.return_value = sources

        request_list = []
        for n in range(6):
            r = Request('T', {}, 'url-%d' % n, 'path-%d' % n)
            r.find_sources = Mock(return_value=sources[n % 3:])
            request_list.append(r)

        collated = [
            {
                sources[0]: ['nectar-1'],
                sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'],
                sources[2]: ['nectar-5', 'nectar-6']
            },
            {}
        ]
        fake_collated = Mock(side_effect=collated)

        fake_listener = Mock()
        canceled = Mock()
        canceled.isSet.side_effect = [False, True, True]
        fake_primary = PrimarySource(Mock())

        # test
        container = ContentContainer('')
        container.refresh = Mock()
        container.collated = fake_collated
        report = container.download(canceled, fake_primary, request_list, fake_listener)

        # validation
        container.refresh.assert_called_with(canceled)

        for r in request_list:
            r.find_sources.assert_called_with(fake_primary, container.sources)

        called = 0
        for s in sources:
            if s.get_downloader.called:
                called += 1

        self.assertEqual(called, 1)
        self.assertEqual(report.total_passes, 1)
        self.assertEqual(report.total_sources, len(sources))
        self.assertEqual(len(report.downloads), 1)
        self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0)
        self.assertEqual(report.downloads[sources[2].id].total_failed, 0)

    def test_purge_orphans(self):
        _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10)
        _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10)
        _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10)
        collection = ContentCatalog.get_collection()
        self.assertEqual(collection.find().count(), 30)
        container = ContentContainer(path=self.tmp_dir)

        # test
        container.purge_orphans()

        # validation
        self.assertEqual(collection.find().count(), 20)
        self.assertEqual(collection.find({'source_id': ORPHANED}).count(), 0)
        self.assertEqual(collection.find({'source_id': UNDERGROUND}).count(), 10)
        self.assertEqual(collection.find({'source_id': UNIT_WORLD}).count(), 10)

    def test_refresh_exception(self, mock_refresh):
        container = ContentContainer(path=self.tmp_dir)

        # test
        report = container.refresh(force=True)

        # validation
        self.assertEqual(len(report), 2)
        for r in report:
            self.assertFalse(r.succeeded)
            self.assertEqual(r.added_count, 0)
            self.assertEqual(r.deleted_count, 0)
            self.assertEqual(len(r.errors), 1)
        collection = ContentCatalog.get_collection()
        self.assertEqual(mock_refresh.call_count, 2)
        self.assertEqual(collection.find().count(), 0)

    def test_forced_refresh(self, fake_manager, fake_load):
        sources = {}
        for n in range(3):
            s = ContentSource('s-%d' % n, {})
            s.refresh = Mock()
            sources[s.id] = s

        fake_manager().has_entries.return_value = True
        fake_load.return_value = sources

        # test
        container = ContentContainer('')
        container.refresh(force=True)

        # validation
        for s in sources.values():
            s.refresh.assert_called_with()

    def test_refresh_failure(self, mock_plugin):
        container = ContentContainer(path=self.tmp_dir)

        # test
        report = container.refresh(force=True)

        # validation
        self.assertEqual(len(report), 5)
        for r in report:
            self.assertFalse(r.succeeded)
            self.assertEqual(r.added_count, 0)
            self.assertEqual(r.deleted_count, 0)
            self.assertEqual(len(r.errors), 1)
        plugin = mock_plugin.return_value[0]
        collection = ContentCatalog.get_collection()
        self.assertEqual(plugin.refresh.call_count, 5)
        self.assertEqual(collection.find().count(), 0)

    def test_download_canceled_before_collated(self, fake_load):
        fake_load.return_value = []
        canceled = FakeEvent()
        canceled.set()

        # test
        container = ContentContainer('')
        container.refresh = Mock()
        container.collated = Mock()
        report = container.download(canceled, None, [], None)

        container.refresh.assert_called_with(canceled)

        self.assertFalse(container.collated.called)
        self.assertEqual(report.total_passes, 0)
        self.assertEqual(report.total_sources, 0)
        self.assertEqual(len(report.downloads), 0)

    def test_refresh_canceled(self, fake_load):
        sources = {}
        for n in range(3):
            s = ContentSource('s-%d' % n, {})
            s.refresh = Mock()
            sources[s.id] = s

        fake_load.return_value = sources

        # test
        canceled = FakeEvent()
        canceled.set()
        container = ContentContainer('')
        container.refresh(canceled, force=True)

        # validation
        for s in sources.values():
            self.assertFalse(s.refresh.called)

    def test_refresh(self, fake_manager, fake_load):
        sources = {}
        for n in range(3):
            s = ContentSource('s-%d' % n, {})
            s.refresh = Mock(return_value=[n])
            s.get_downloader = Mock()
            sources[s.id] = s

        fake_manager().has_entries.return_value = False
        fake_load.return_value = sources

        # test
        container = ContentContainer('')
        report = container.refresh()

        # validation
        for s in sources.values():
            s.refresh.assert_called_with()

        self.assertEqual(sorted(report), [0, 1, 2])

    def test_threaded_download(self, fake_load, fake_refresh, fake_primary, fake_batch):
        path = Mock()
        downloader = Mock()
        requests = Mock()
        listener = Mock()

        _batch = Mock()
        _batch.download.return_value = 123
        fake_batch.return_value = _batch

        # test
        container = ContentContainer(path)
        report = container.download(downloader, requests, listener)

        # validation
        fake_load.assert_called_with(path)
        fake_refresh.assert_called_with()
        fake_primary.assert_called_with(downloader)
        fake_batch.assert_called_with(fake_primary(), container, requests, listener)
        _batch.assert_called_with()
        self.assertEqual(report, _batch.return_value)

    def test_refresh(self, mock_plugin):
        container = ContentContainer(path=self.tmp_dir)

        # test
        report = container.refresh(force=True)

        # validation
        plugin = mock_plugin.return_value[0]
        self.assertEqual(plugin.refresh.call_count, 5)
        self.assertEqual(len(report), 5)
        for r in report:
            self.assertTrue(r.succeeded)
            self.assertEqual(r.added_count, 100)
            self.assertEqual(r.deleted_count, 0)
        calls = iter(plugin.refresh.call_args_list)
        for source in ContentSource.load_all(self.tmp_dir).values():
            for url in source.urls:
                args = calls.next()[0]
                self.assertTrue(isinstance(args[0], CatalogerConduit))
                self.assertEqual(args[1], source.descriptor)
                self.assertEqual(args[2], url)