Python repo_publish.RepoPublishConduit类代码示例

    def _init_publisher(self):

        repo = Repository(self.repo_id, working_dir=self.working_dir)
        self.repo = repo

        conduit = RepoPublishConduit(repo.id, YUM_DISTRIBUTOR_ID)
        conduit.last_publish = mock.Mock(return_value=None)
        conduit.get_repo_scratchpad = mock.Mock(return_value={})

        config_defaults = {'http': True,
                           'https': True,
                           'relative_url': None,
                           'http_publish_dir': os.path.join(self.published_dir, 'http'),
                           'https_publish_dir': os.path.join(self.published_dir, 'https')}
        config = PluginCallConfiguration(None, None)
        config.default_config.update(config_defaults)

        self.publisher = publish.BaseYumRepoPublisher(repo, conduit, config, YUM_DISTRIBUTOR_ID,
                                                      working_dir=self.working_dir)
        self.publisher.get_checksum_type = mock.Mock(return_value=None)

        # mock out the repomd_file_context, so _publish_<step> can be called
        # outside of the publish() method
        self.publisher.repomd_file_context = mock.MagicMock()
        self.publisher.all_steps = mock.MagicMock()

class RepoPublishConduitTests(base.PulpServerTests):

    def clean(self):
        super(RepoPublishConduitTests, self).clean()

        mock_plugins.reset()
        model.Repository.drop_collection()
        RepoDistributor.get_collection().remove()

    @mock.patch('pulp.server.managers.repo.importer.model.Repository.objects')
    def setUp(self, mock_repo_qs):
        super(RepoPublishConduitTests, self).setUp()
        mock_plugins.install()
        manager_factory.initialize()

        self.distributor_manager = manager_factory.repo_distributor_manager()

        # Populate the database with a repo with units
        self.distributor_manager.add_distributor('repo-1', 'mock-distributor', {}, True,
                                                 distributor_id='dist-1')

        self.conduit = RepoPublishConduit('repo-1', 'dist-1')

    def tearDown(self):
        super(RepoPublishConduitTests, self).tearDown()
        mock_plugins.reset()

    def test_str(self):
        """
        Makes sure the __str__ implementation doesn't crash.
        """
        str(self.conduit)

    def test_last_publish(self):
        """
        Tests retrieving the last publish time in both the unpublish and previously published cases.
        """

        # Test - Unpublished
        unpublished = self.conduit.last_publish()
        self.assertTrue(unpublished is None)

        # Setup - Previous publish
        last_publish = datetime.datetime(2015, 4, 29, 20, 23, 56, 0)
        repo_dist = RepoDistributor.get_collection().find_one({'repo_id': 'repo-1'})
        repo_dist['last_publish'] = last_publish
        RepoDistributor.get_collection().save(repo_dist, safe=True)

        # Test - Last publish
        found = self.conduit.last_publish()
        self.assertTrue(isinstance(found, datetime.datetime))  # check returned format
        self.assertEqual(repo_dist['last_publish'], found)

    @mock.patch('pulp.plugins.conduits.repo_publish.RepoDistributor')
    def test_last_publish_with_error(self, mock_dist):
        """
        Test the handling of an error getting last_publish information.
        """
        mock_dist.get_collection().find_one.return_value = None
        self.assertRaises(DistributorConduitException, self.conduit.last_publish)

 def test_import_cached_manifest_missing_units(self, *unused):
     # Setup
     self.populate()
     with mock_config.patch({'server': {'storage_dir': self.parentfs}}):
         dist = NodesHttpDistributor()
         working_dir = os.path.join(self.childfs, 'working_dir')
         os.makedirs(working_dir)
         repo = Repository(self.REPO_ID, working_dir)
         configuration = self.dist_conf()
         conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR)
         dist.publish_repo(repo, conduit, configuration)
         model.Distributor.objects.delete()
         RepoContentUnit.get_collection().remove()
         unit_db.clean()
         self.define_plugins()
         publisher = dist.publisher(repo, configuration)
         manifest_path = publisher.manifest_path()
         manifest = Manifest(manifest_path)
         manifest.read()
         shutil.copy(manifest_path, os.path.join(working_dir, MANIFEST_FILE_NAME))
         # Test
         importer = NodesHttpImporter()
         manifest_url = pathlib.url_join(publisher.base_url, manifest_path)
         configuration = {
             constants.MANIFEST_URL_KEYWORD: manifest_url,
             constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY,
         }
         configuration = PluginCallConfiguration(configuration, {})
         conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER, Mock())
     with mock_config.patch({'server': {'storage_dir': self.childfs}}):
         with patch('pulp_node.constants.CONTENT_PATH', self.parentfs):
             importer.sync_repo(repo, conduit, configuration)
         # Verify
         units = conduit.get_units()
         self.assertEquals(len(units), self.NUM_UNITS)

 def test_import(self):
     # Setup
     self.populate()
     pulp_conf.set('server', 'storage_dir', self.parentfs)
     dist = NodesHttpDistributor()
     repo = Repository(self.REPO_ID)
     cfg = {
         'protocol':'file',
         'http':{'alias':self.alias},
         'https':{'alias':self.alias},
         'file':{'alias':self.alias},
     }
     conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR)
     dist.publish_repo(repo, conduit, cfg)
     Repo.get_collection().remove()
     RepoDistributor.get_collection().remove()
     RepoContentUnit.get_collection().remove()
     unit_db.clean()
     # Test
     importer = NodesHttpImporter()
     publisher = dist.publisher(repo, cfg)
     manifest_url = 'file://' + publisher.manifest_path()
     cfg = dict(manifest_url=manifest_url, strategy=constants.MIRROR_STRATEGY)
     conduit = RepoSyncConduit(
         self.REPO_ID,
         constants.HTTP_IMPORTER,
         RepoContentUnit.OWNER_TYPE_IMPORTER,
         constants.HTTP_IMPORTER)
     importer.sync_repo(repo, conduit, cfg)
     # Verify
     units = conduit.get_units()
     self.assertEquals(len(units), self.NUM_UNITS)

class RepoPublishConduitTests(base.PulpServerTests):

    def clean(self):
        super(RepoPublishConduitTests, self).clean()

        mock_plugins.reset()
        model.Repository.objects.delete()
        model.Distributor.objects.delete()

    @mock.patch('pulp.server.controllers.distributor.model.Repository.objects')
    def setUp(self, mock_repo_qs):
        super(RepoPublishConduitTests, self).setUp()
        mock_plugins.install()
        manager_factory.initialize()

        # Populate the database with a repo with units
        dist_controller.add_distributor('repo-1', 'mock-distributor', {}, True,
                                        distributor_id='dist-1')

        self.conduit = RepoPublishConduit('repo-1', 'dist-1')

    def tearDown(self):
        super(RepoPublishConduitTests, self).tearDown()
        mock_plugins.reset()

    def test_str(self):
        """
        Makes sure the __str__ implementation doesn't crash.
        """
        str(self.conduit)

    def test_last_publish(self):
        """
        Tests retrieving the last publish time in both the unpublish and previously published cases.
        """

        # Test - Unpublished
        unpublished = self.conduit.last_publish()
        self.assertTrue(unpublished is None)

        # Setup - Previous publish
        last_publish = datetime.datetime(2015, 4, 29, 20, 23, 56, 0)
        repo_dist = model.Distributor.objects.get_or_404(repo_id='repo-1')
        repo_dist['last_publish'] = last_publish
        repo_dist.save()

        # Test - Last publish
        found = self.conduit.last_publish()
        self.assertTrue(isinstance(found, datetime.datetime))  # check returned format

        self.assertEqual(found.tzinfo, dateutils.utc_tz())
        self.assertEqual(repo_dist['last_publish'], found.replace(tzinfo=None))

    @mock.patch('pulp.plugins.conduits.repo_publish.model.Distributor.objects')
    def test_last_publish_with_error(self, m_dist_qs):
        """
        Test the handling of an error getting last_publish information.
        """
        m_dist_qs.only.return_value.get_or_404.side_effect = exceptions.MissingResource
        self.assertRaises(DistributorConduitException, self.conduit.last_publish)

class RepoPublishConduitTests(base.PulpServerTests):

    def clean(self):
        super(RepoPublishConduitTests, self).clean()

        mock_plugins.reset()

        Repo.get_collection().remove()
        RepoDistributor.get_collection().remove()

    def setUp(self):
        super(RepoPublishConduitTests, self).setUp()
        mock_plugins.install()
        manager_factory.initialize()

        self.repo_manager = manager_factory.repo_manager()
        self.distributor_manager = manager_factory.repo_distributor_manager()

        # Populate the database with a repo with units
        self.repo_manager.create_repo('repo-1')
        self.distributor_manager.add_distributor('repo-1', 'mock-distributor', {}, True, distributor_id='dist-1')

        self.conduit = RepoPublishConduit('repo-1', 'dist-1')

    def test_str(self):
        """
        Makes sure the __str__ implementation doesn't crash.
        """
        str(self.conduit)

    def test_last_publish(self):
        """
        Tests retrieving the last publish time in both the unpublish and previously published cases.
        """

        # Test - Unpublished
        unpublished = self.conduit.last_publish()
        self.assertTrue(unpublished is None)

        # Setup - Previous publish
        last_publish = datetime.datetime.now()
        repo_dist = RepoDistributor.get_collection().find_one({'repo_id' : 'repo-1'})
        repo_dist['last_publish'] = dateutils.format_iso8601_datetime(last_publish)
        RepoDistributor.get_collection().save(repo_dist, safe=True)

        # Test - Last publish
        found = self.conduit.last_publish()
        self.assertTrue(isinstance(found, datetime.datetime)) # check returned format
        self.assertEqual(repo_dist['last_publish'], dateutils.format_iso8601_datetime(found))

    @mock.patch('pulp.server.managers.repo.publish.RepoPublishManager.last_publish')
    def test_last_publish_with_error(self, mock_call):
        # Setup
        mock_call.side_effect = Exception()

        # Test
        self.assertRaises(DistributorConduitException, self.conduit.last_publish)

 def setUp(self):
     self.working_directory = tempfile.mkdtemp()
     self.content_dir = os.path.join(self.working_directory, 'content')
     self.working_dir = os.path.join(self.working_directory, 'work')
     os.makedirs(self.working_dir)
     self.repo = Repository(id='foo', working_dir=self.working_dir)
     config = PluginCallConfiguration(None, None)
     conduit = RepoPublishConduit(self.repo.id, 'foo_repo')
     conduit.get_repo_scratchpad = Mock(return_value={u'tags': {}})
     self.parent = steps.PluginStep('test-step', self.repo, conduit, config)

 def setUp(self):
     self.temp_dir = tempfile.mkdtemp()
     self.working_directory = os.path.join(self.temp_dir, 'working')
     self.publish_directory = os.path.join(self.temp_dir, 'publish')
     self.content_directory = os.path.join(self.temp_dir, 'content')
     os.makedirs(self.working_directory)
     os.makedirs(self.publish_directory)
     os.makedirs(self.content_directory)
     repo = Repository('foo_repo_id', working_dir=self.working_directory)
     config = PluginCallConfiguration(None, None)
     conduit = RepoPublishConduit(repo.id, 'foo_repo')
     conduit.get_repo_scratchpad = Mock(return_value={u'tags': {}})
     self.parent = PublishStep('test-step', repo, conduit, config)

 def test_import_unit_files_already_exist_size_mismatch(self, *mocks):
     # Setup
     self.populate()
     pulp_conf.set('server', 'storage_dir', self.parentfs)
     dist = NodesHttpDistributor()
     working_dir = os.path.join(self.childfs, 'working_dir')
     os.makedirs(working_dir)
     repo = Repository(self.REPO_ID, working_dir)
     cfg = self.dist_conf()
     conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR)
     dist.publish_repo(repo, conduit, cfg)
     Repo.get_collection().remove()
     RepoDistributor.get_collection().remove()
     RepoContentUnit.get_collection().remove()
     unit_db.clean()
     self.define_plugins()
     parent_content = os.path.join(self.parentfs, 'content')
     child_content = os.path.join(self.childfs, 'content')
     shutil.copytree(parent_content, child_content)
     for fn in os.listdir(child_content):
         path = os.path.join(child_content, fn)
         if os.path.isdir(path):
             continue
         with open(path, 'w') as fp:
             fp.truncate()
     # Test
     importer = NodesHttpImporter()
     publisher = dist.publisher(repo, cfg)
     manifest_url = pathlib.url_join(publisher.base_url, publisher.manifest_path())
     configuration = {
         constants.MANIFEST_URL_KEYWORD: manifest_url,
         constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY,
     }
     configuration = PluginCallConfiguration(configuration, {})
     conduit = RepoSyncConduit(
         self.REPO_ID,
         constants.HTTP_IMPORTER,
         RepoContentUnit.OWNER_TYPE_IMPORTER,
         constants.HTTP_IMPORTER)
     pulp_conf.set('server', 'storage_dir', self.childfs)
     importer.sync_repo(repo, conduit, configuration)
     # Verify
     units = conduit.get_units()
     self.assertEquals(len(units), self.NUM_UNITS)
     mock_importer_config_to_nectar_config = mocks[0]
     mock_importer_config_to_nectar_config.assert_called_with(configuration.flatten())

    def setUp(self, mock_repo_qs):
        super(RepoPublishConduitTests, self).setUp()
        mock_plugins.install()
        manager_factory.initialize()

        # Populate the database with a repo with units
        dist_controller.add_distributor('repo-1', 'mock-distributor', {}, True,
                                        distributor_id='dist-1')

        self.conduit = RepoPublishConduit('repo-1', 'dist-1')

 def test_import_cached_manifest_matched(self, mock_fetch, *unused):
     # Setup
     self.populate()
     pulp_conf.set('server', 'storage_dir', self.parentfs)
     dist = NodesHttpDistributor()
     working_dir = os.path.join(self.childfs, 'working_dir')
     os.makedirs(working_dir)
     repo = Repository(self.REPO_ID, working_dir)
     configuration = self.dist_conf()
     conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR)
     dist.publish_repo(repo, conduit, configuration)
     Repo.get_collection().remove()
     RepoDistributor.get_collection().remove()
     RepoContentUnit.get_collection().remove()
     unit_db.clean()
     self.define_plugins()
     publisher = dist.publisher(repo, configuration)
     manifest_path = publisher.manifest_path()
     units_path = os.path.join(os.path.dirname(manifest_path), UNITS_FILE_NAME)
     manifest = Manifest(manifest_path)
     manifest.read()
     shutil.copy(manifest_path, os.path.join(working_dir, MANIFEST_FILE_NAME))
     shutil.copy(units_path, os.path.join(working_dir, UNITS_FILE_NAME))
     # Test
     importer = NodesHttpImporter()
     manifest_url = pathlib.url_join(publisher.base_url, manifest_path)
     configuration = {
         constants.MANIFEST_URL_KEYWORD: manifest_url,
         constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY,
     }
     configuration = PluginCallConfiguration(configuration, {})
     conduit = RepoSyncConduit(
         self.REPO_ID,
         constants.HTTP_IMPORTER,
         RepoContentUnit.OWNER_TYPE_IMPORTER,
         constants.HTTP_IMPORTER)
     pulp_conf.set('server', 'storage_dir', self.childfs)
     importer.sync_repo(repo, conduit, configuration)
     # Verify
     units = conduit.get_units()
     self.assertEquals(len(units), self.NUM_UNITS)
     self.assertFalse(mock_fetch.called)

 def test_import(self, *mocks):
     # Setup
     self.populate()
     max_concurrency = 5
     max_bandwidth = 12345
     pulp_conf.set('server', 'storage_dir', self.parentfs)
     dist = NodesHttpDistributor()
     working_dir = os.path.join(self.childfs, 'working_dir')
     os.makedirs(working_dir)
     repo = Repository(self.REPO_ID, working_dir)
     cfg = self.dist_conf()
     conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR)
     dist.publish_repo(repo, conduit, cfg)
     Repo.get_collection().remove()
     RepoDistributor.get_collection().remove()
     RepoContentUnit.get_collection().remove()
     unit_db.clean()
     self.define_plugins()
     # Test
     importer = NodesHttpImporter()
     publisher = dist.publisher(repo, cfg)
     manifest_url = pathlib.url_join(publisher.base_url, publisher.manifest_path())
     configuration = {
         constants.MANIFEST_URL_KEYWORD: manifest_url,
         constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY,
         importer_constants.KEY_MAX_DOWNLOADS: max_concurrency,
         importer_constants.KEY_MAX_SPEED: max_bandwidth,
     }
     configuration = PluginCallConfiguration(configuration, {})
     conduit = RepoSyncConduit(
         self.REPO_ID,
         constants.HTTP_IMPORTER,
         RepoContentUnit.OWNER_TYPE_IMPORTER,
         constants.HTTP_IMPORTER)
     pulp_conf.set('server', 'storage_dir', self.childfs)
     importer.sync_repo(repo, conduit, configuration)
     # Verify
     units = conduit.get_units()
     self.assertEquals(len(units), self.NUM_UNITS)
     mock_importer_config_to_nectar_config = mocks[0]
     mock_importer_config_to_nectar_config.assert_called_with(configuration.flatten())

    def _init_publisher(self):

        repo = Repository(self.repo_id, working_dir=self.working_dir)

        conduit = RepoPublishConduit(repo.id, YUM_DISTRIBUTOR_ID)
        conduit.get_repo_scratchpad = mock.Mock(return_value={})

        config_defaults = {
            "http": True,
            "https": True,
            "relative_url": None,
            "http_publish_dir": os.path.join(self.published_dir, "http"),
            "https_publish_dir": os.path.join(self.published_dir, "https"),
        }
        config = PluginCallConfiguration(None, None)
        config.default_config.update(config_defaults)

        self.publisher = publish.Publisher(repo, conduit, config)

        # mock out the repomd_file_context, so _publish_<step> can be called
        # outside of the publish() method
        self.publisher.repomd_file_context = mock.MagicMock()

 def test_import_unit_files_already_exist(self, mock_get_working, *mocks):
     # Setup
     self.populate()
     mock_get_working.return_value = self.temp_dir
     with mock_config.patch({'server': {'storage_dir': self.parentfs}}):
         dist = NodesHttpDistributor()
         working_dir = os.path.join(self.childfs, 'working_dir')
         os.makedirs(working_dir)
         repo = Repository(self.REPO_ID, working_dir)
         cfg = self.dist_conf()
         conduit = RepoPublishConduit(self.REPO_ID, constants.HTTP_DISTRIBUTOR)
         dist.publish_repo(repo, conduit, cfg)
         model.Distributor.objects.delete()
         RepoContentUnit.get_collection().remove()
         unit_db.clean()
         self.define_plugins()
         parent_content = os.path.join(self.parentfs, 'content')
         child_content = os.path.join(self.childfs, 'content')
         shutil.copytree(parent_content, child_content)
         # Test
         importer = NodesHttpImporter()
         publisher = dist.publisher(repo, cfg)
         manifest_url = pathlib.url_join(publisher.base_url, publisher.manifest_path())
         configuration = {
             constants.MANIFEST_URL_KEYWORD: manifest_url,
             constants.STRATEGY_KEYWORD: constants.MIRROR_STRATEGY,
         }
         configuration = PluginCallConfiguration(configuration, {})
         conduit = RepoSyncConduit(self.REPO_ID, constants.HTTP_IMPORTER, Mock())
     with mock_config.patch({'server': {'storage_dir': self.childfs}}):
         with patch('pulp_node.constants.CONTENT_PATH', self.parentfs):
             importer.sync_repo(repo, conduit, configuration)
         # Verify
         units = conduit.get_units()
         self.assertEquals(len(units), self.NUM_UNITS)
         mock_importer_config_to_nectar_config = mocks[0]
         mock_importer_config_to_nectar_config.assert_called_with(configuration.flatten())