Python cache.PROGRAM_CACHE_KEY_TPL类代码示例

    def test_get_many_with_missing(self, mock_cache, mock_warning, mock_info):
        programs = ProgramFactory.create_batch(3)

        all_programs = {
            PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in programs
        }

        partial_programs = {
            PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in programs[:2]
        }

        def fake_get_many(keys):
            if len(keys) == 1:
                return {PROGRAM_CACHE_KEY_TPL.format(uuid=programs[-1]['uuid']): programs[-1]}
            else:
                return partial_programs

        mock_cache.get.return_value = [program['uuid'] for program in programs]
        mock_cache.get_many.side_effect = fake_get_many

        actual_programs = get_programs(self.site)

        # All 3 cached programs should be returned. An info message should be
        # logged about the one that was initially missing, but the code should
        # be able to stitch together all the details.
        self.assertEqual(
            set(program['uuid'] for program in actual_programs),
            set(program['uuid'] for program in all_programs.values())
        )
        self.assertFalse(mock_warning.called)
        mock_info.assert_called_with('Failed to get details for 1 programs. Retrying.')

        for program in actual_programs:
            key = PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid'])
            self.assertEqual(program, all_programs[key])

def get_programs_by_uuids(uuids):
    """
    Gets a list of programs for the provided uuids
    """
    # a list of UUID objects would be a perfectly reasonable parameter to provide
    uuid_strings = [six.text_type(handle) for handle in uuids]

    programs = cache.get_many([PROGRAM_CACHE_KEY_TPL.format(uuid=handle) for handle in uuid_strings])
    programs = list(programs.values())

    # The get_many above sometimes fails to bring back details cached on one or
    # more Memcached nodes. It doesn't look like these keys are being evicted.
    # 99% of the time all keys come back, but 1% of the time all the keys stored
    # on one or more nodes are missing from the result of the get_many. One
    # get_many may fail to bring these keys back, but a get_many occurring
    # immediately afterwards will succeed in bringing back all the keys. This
    # behavior can be mitigated by trying again for the missing keys, which is
    # what we do here. Splitting the get_many into smaller chunks may also help.
    missing_uuids = set(uuid_strings) - set(program['uuid'] for program in programs)
    if missing_uuids:
        logger.info(
            u'Failed to get details for {count} programs. Retrying.'.format(count=len(missing_uuids))
        )

        retried_programs = cache.get_many([PROGRAM_CACHE_KEY_TPL.format(uuid=uuid) for uuid in missing_uuids])
        programs += list(retried_programs.values())

        still_missing_uuids = set(uuid_strings) - set(program['uuid'] for program in programs)
        for uuid in still_missing_uuids:
            logger.warning(missing_details_msg_tpl.format(uuid=uuid))

    return programs

    def test_pathways_multiple_pages(self):
        """
        Verify that the command properly caches credit pathways when multiple pages are returned from its endpoint
        """
        UserFactory(username=self.catalog_integration.service_username)
        new_pathways = PathwayFactory.create_batch(40)
        for new_pathway in new_pathways:
            new_pathway['programs'] = []
        pathways = self.pathways + new_pathways

        programs = {
            PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in self.programs
        }

        self.mock_list()
        for uuid in self.uuids:
            program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
            self.mock_detail(uuid, program)

        # mock 3 pages of credit pathways, starting at the last
        self.mock_pathways(pathways[40:], page_number=3, final=True)
        self.mock_pathways(pathways[20:40], page_number=2, final=False)
        self.mock_pathways(pathways[:20], page_number=1, final=False)

        call_command('cache_programs')

        pathways_dict = {
            PATHWAY_CACHE_KEY_TPL.format(id=pathway['id']): pathway for pathway in pathways
        }
        pathway_keys = list(pathways_dict.keys())

        cached_pathway_keys = cache.get(SITE_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=self.site_domain))
        self.assertEqual(
            set(cached_pathway_keys),
            set(pathway_keys)
        )

        cached_pathways = cache.get_many(pathway_keys)
        self.assertEqual(
            set(cached_pathways),
            set(pathways_dict)
        )

        # We can't use a set comparison here because these values are dictionaries
        # and aren't hashable. We've already verified that all pathways came out
        # of the cache above, so all we need to do here is verify the accuracy of
        # the data itself.
        for key, pathway in cached_pathways.items():
            # cached pathways store just program uuids instead of the full programs, transform before comparing
            pathways_dict[key]['program_uuids'] = [program['uuid'] for program in pathways_dict[key]['programs']]
            del pathways_dict[key]['programs']

            self.assertEqual(pathway, pathways_dict[key])

def get_programs(site, uuid=None):
    """Read programs from the cache.

    The cache is populated by a management command, cache_programs.

    Arguments:
        site (Site): django.contrib.sites.models object

    Keyword Arguments:
        uuid (string): UUID identifying a specific program to read from the cache.

    Returns:
        list of dict, representing programs.
        dict, if a specific program is requested.
    """
    missing_details_msg_tpl = 'Failed to get details for program {uuid} from the cache.'

    if uuid:
        program = cache.get(PROGRAM_CACHE_KEY_TPL.format(uuid=uuid))
        if not program:
            logger.warning(missing_details_msg_tpl.format(uuid=uuid))

        return program
    uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), [])
    if not uuids:
        logger.warning('Failed to get program UUIDs from the cache.')

    programs = cache.get_many([PROGRAM_CACHE_KEY_TPL.format(uuid=uuid) for uuid in uuids])
    programs = list(programs.values())

    # The get_many above sometimes fails to bring back details cached on one or
    # more Memcached nodes. It doesn't look like these keys are being evicted.
    # 99% of the time all keys come back, but 1% of the time all the keys stored
    # on one or more nodes are missing from the result of the get_many. One
    # get_many may fail to bring these keys back, but a get_many occurring
    # immediately afterwards will succeed in bringing back all the keys. This
    # behavior can be mitigated by trying again for the missing keys, which is
    # what we do here. Splitting the get_many into smaller chunks may also help.
    missing_uuids = set(uuids) - set(program['uuid'] for program in programs)
    if missing_uuids:
        logger.info(
            'Failed to get details for {count} programs. Retrying.'.format(count=len(missing_uuids))
        )

        retried_programs = cache.get_many([PROGRAM_CACHE_KEY_TPL.format(uuid=uuid) for uuid in missing_uuids])
        programs += list(retried_programs.values())

        still_missing_uuids = set(uuids) - set(program['uuid'] for program in programs)
        for uuid in still_missing_uuids:
            logger.warning(missing_details_msg_tpl.format(uuid=uuid))

    return programs

 def _create_catalog_program(self, catalog_org):
     """ helper method to create a cached catalog program """
     program = ProgramFactory.create(
         authoring_organizations=[catalog_org]
     )
     cache.set(PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']), program, None)
     return program

    def process_pathways(self, site, pathways, programs):
        """
        For each program, add references to each pathway it is a part of.
        For each pathway, replace the "programs" dict with "program_uuids",
        which only contains uuids (since program data is already cached)
        """
        processed_pathways = {}
        failure = False
        for pathway in pathways:
            try:
                pathway_id = pathway['id']
                pathway_cache_key = PATHWAY_CACHE_KEY_TPL.format(id=pathway_id)
                processed_pathways[pathway_cache_key] = pathway
                uuids = []

                for program in pathway['programs']:
                    program_uuid = program['uuid']
                    program_cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=program_uuid)
                    programs[program_cache_key]['pathway_ids'].append(pathway_id)
                    uuids.append(program_uuid)

                del pathway['programs']
                pathway['program_uuids'] = uuids
            except:  # pylint: disable=bare-except
                logger.error('Failed to process pathways for {domain}'.format(domain=site.domain))
                failure = True
        return processed_pathways, programs, failure

    def test_handle_pathways(self):
        """
        Verify that the command requests and caches credit pathways
        """

        UserFactory(username=self.catalog_integration.service_username)

        programs = {
            PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in self.programs
        }

        pathways = {
            PATHWAY_CACHE_KEY_TPL.format(id=pathway['id']): pathway for pathway in self.pathways
        }

        self.mock_list()
        self.mock_pathways(self.pathways)

        for uuid in self.uuids:
            program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
            self.mock_detail(uuid, program)

        call_command('cache_programs')

        cached_pathway_keys = cache.get(SITE_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=self.site_domain))
        pathway_keys = list(pathways.keys())
        self.assertEqual(
            set(cached_pathway_keys),
            set(pathway_keys)
        )

        cached_pathways = cache.get_many(pathway_keys)
        self.assertEqual(
            set(cached_pathways),
            set(pathways)
        )

        # We can't use a set comparison here because these values are dictionaries
        # and aren't hashable. We've already verified that all pathways came out
        # of the cache above, so all we need to do here is verify the accuracy of
        # the data itself.
        for key, pathway in cached_pathways.items():
            # cached pathways store just program uuids instead of the full programs, transform before comparing
            pathways[key]['program_uuids'] = [program['uuid'] for program in pathways[key]['programs']]
            del pathways[key]['programs']

            self.assertEqual(pathway, pathways[key])

    def test_handle_programs(self):
        """
        Verify that the command requests and caches program UUIDs and details.
        """
        # Ideally, this user would be created in the test setup and deleted in
        # the one test case which covers the case where the user is missing. However,
        # that deletion causes "OperationalError: no such table: wiki_attachmentrevision"
        # when run on Jenkins.
        UserFactory(username=self.catalog_integration.service_username)

        programs = {
            PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in self.programs
        }

        self.mock_list()
        self.mock_pathways(self.pathways)

        for uuid in self.uuids:
            program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
            self.mock_detail(uuid, program)

        call_command('cache_programs')

        cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
        self.assertEqual(
            set(cached_uuids),
            set(self.uuids)
        )

        program_keys = list(programs.keys())
        cached_programs = cache.get_many(program_keys)
        # Verify that the keys were all cache hits.
        self.assertEqual(
            set(cached_programs),
            set(programs)
        )

        # We can't use a set comparison here because these values are dictionaries
        # and aren't hashable. We've already verified that all programs came out
        # of the cache above, so all we need to do here is verify the accuracy of
        # the data itself.
        for key, program in cached_programs.items():
            # cached programs have a pathways field added to them, remove before comparing
            del program['pathway_ids']
            self.assertEqual(program, programs[key])

    def test_handle_missing_programs(self):
        """
        Verify that a problem retrieving a program doesn't prevent the command
        from retrieving and caching other programs, but does cause it to exit
        with a non-zero exit code.
        """
        UserFactory(username=self.catalog_integration.service_username)

        all_programs = {
            PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in self.programs
        }
        partial_programs = {
            PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in self.programs[:2]
        }

        self.mock_list()

        for uuid in self.uuids[:2]:
            program = partial_programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
            self.mock_detail(uuid, program)

        with self.assertRaises(SystemExit) as context:
            call_command('cache_programs')

        self.assertEqual(context.exception.code, 1)

        cached_uuids = cache.get(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site_domain))
        self.assertEqual(
            set(cached_uuids),
            set(self.uuids)
        )

        program_keys = list(all_programs.keys())
        cached_programs = cache.get_many(program_keys)
        # One of the cache keys should result in a cache miss.
        self.assertEqual(
            set(cached_programs),
            set(partial_programs)
        )

        for key, program in cached_programs.items():
            # cached programs have a pathways field added to them, remove before comparing
            del program['pathway_ids']
            self.assertEqual(program, partial_programs[key])

 def setup_catalog_cache(self, program_uuid, organization_key):
     """
     helper function to initialize a cached program with an single authoring_organization
     """
     catalog_org = CatalogOrganizationFactory.create(key=organization_key)
     program = ProgramFactory.create(
         uuid=program_uuid,
         authoring_organizations=[catalog_org]
     )
     cache.set(PROGRAM_CACHE_KEY_TPL.format(uuid=program_uuid), program, None)

    def test_handle_missing_pathways(self):
        """
        Verify that the command raises an exception when it fails to retrieve pathways.
        """
        UserFactory(username=self.catalog_integration.service_username)

        programs = {
            PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program for program in self.programs
        }

        self.mock_list()

        for uuid in self.uuids:
            program = programs[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)]
            self.mock_detail(uuid, program)

        with self.assertRaises(SystemExit) as context:
            call_command('cache_programs')
        self.assertEqual(context.exception.code, 1)

        cached_pathways = cache.get(SITE_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=self.site_domain))
        self.assertEqual(cached_pathways, [])

 def fetch_program_details(self, client, uuids):
     programs = {}
     failure = False
     for uuid in uuids:
         try:
             cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)
             logger.info('Requesting details for program {uuid}.'.format(uuid=uuid))
             program = client.programs(uuid).get(exclude_utm=1)
             programs[cache_key] = program
         except:  # pylint: disable=bare-except
             logger.exception('Failed to retrieve details for program {uuid}.'.format(uuid=uuid))
             failure = True
             continue
     return programs, failure

    def test_catalog_program_missing_org(self):
        """
        Test OrganizationDoesNotExistException is thrown if the cached program does not
        have an authoring organization.
        """
        program = ProgramFactory.create(
            uuid=self.program_uuid,
            authoring_organizations=[]
        )
        cache.set(PROGRAM_CACHE_KEY_TPL.format(uuid=self.program_uuid), program, None)

        organization = OrganizationFactory.create(short_name=self.organization_key)
        provider = SAMLProviderConfigFactory.create(organization=organization)
        self.create_social_auth_entry(self.user, provider, self.external_user_id)

        with pytest.raises(OrganizationDoesNotExistException):
            get_user_by_program_id(self.external_user_id, self.program_uuid)

    def test_get_one(self, mock_warning, _mock_info):
        expected_program = ProgramFactory()
        expected_uuid = expected_program['uuid']

        self.assertEqual(get_programs(self.site, uuid=expected_uuid), None)
        mock_warning.assert_called_once_with(
            'Failed to get details for program {uuid} from the cache.'.format(uuid=expected_uuid)
        )
        mock_warning.reset_mock()

        cache.set(
            PROGRAM_CACHE_KEY_TPL.format(uuid=expected_uuid),
            expected_program,
            None
        )

        actual_program = get_programs(self.site, uuid=expected_uuid)
        self.assertEqual(actual_program, expected_program)
        self.assertFalse(mock_warning.called)

    def test_get_from_course(self, mock_warning, _mock_info):
        expected_program = ProgramFactory()
        expected_course = expected_program['courses'][0]['course_runs'][0]['key']

        self.assertEqual(get_programs(course=expected_course), [])

        cache.set(
            COURSE_PROGRAMS_CACHE_KEY_TPL.format(course_run_id=expected_course),
            [expected_program['uuid']],
            None
        )
        cache.set(
            PROGRAM_CACHE_KEY_TPL.format(uuid=expected_program['uuid']),
            expected_program,
            None
        )

        actual_program = get_programs(course=expected_course)
        self.assertEqual(actual_program, [expected_program])
        self.assertFalse(mock_warning.called)