Python search_engine_base.SearchEngine类代码示例

 def _display_name_filter(self, library, display_name):
     """ Filters library children by capa type"""
     search_engine = SearchEngine.get_search_engine(index="library_index")
     if search_engine:
         log.warn("search engine found")
         filter_clause = {
             "library": unicode(normalize_key_for_search(library.location.library_key)),
             # "content_type": CapaDescriptor.INDEX_CONTENT_TYPE,
             # "display_name": display_name
         }
         search_result = search_engine.search(field_dictionary=filter_clause)
         new_results = search_result.get('results', [])
         results = []
         for r in new_results:
             v = self.deep_search(["display_name"], r)
             if v['display_name'] == display_name:
                 results.append(r)
         return [LibraryUsageLocator.from_string(item['data']['id']) for item in results]
     else:
         log.warn("search engine NOT found")
         #return [key for key in library.children if self._filter_child_name(key, display_name)]
         results = []
         for r in library.children:
             p = self.store.get_item(r, 1)
             v = {}
             for field in p.fields.values():
                 v[field.name] = field.read_json(p)
             # v = p.get_explicitly_set_fields_by_scope(Scope.settings)
             if v.get('display_name') == display_name:
                 results.append(r)
         return results

    def test_search_from_url(self):
        """ ensure that we get the error back when the backend fails """
        searcher = SearchEngine.get_search_engine(TEST_INDEX_NAME)
        searcher.index(
            "courseware_content",
            [
                {
                    "id": "FAKE_ID_1",
                    "content": {
                        "text": "Little Darling, it's been a long long lonely winter"
                    }
                }
            ]
        )
        searcher.index(
            "courseware_content",
            [
                {
                    "id": "FAKE_ID_2",
                    "content": {
                        "text": "Little Darling, it's been a year since sun been gone"
                    }
                }
            ]
        )
        searcher.index("test_doc", [{"id": "FAKE_ID_3", "content": {"text": "Here comes the sun"}}])

        code, results = post_request({"search_string": "sun"})
        self.assertGreater(code, 499)
        self.assertEqual(results["error"], 'An error occurred when searching for "sun"')

        with self.assertRaises(StandardError):
            searcher.search(query_string="test search")

 def engine(cls):
     """
     Return course team search engine (if feature is enabled).
     """
     try:
         return SearchEngine.get_search_engine(index=cls.INDEX_NAME)
     except ConnectionError as err:
         logging.error('Error connecting to elasticsearch: %s', err)
         raise ElasticSearchConnectionError

def delete_course_task(user_id, course_key_string):

    profile = UserProfile.objects.get(pk=user_id)
    user = User.objects.get(pk=profile.user_id)

    course_key = CourseKey.from_string(course_key_string)
    delete_course_and_groups(course_key, user.id)
    searcher = SearchEngine.get_search_engine(CoursewareSearchIndexer.INDEX_NAME)
    if searcher != None:
        CoursewareSearchIndexer.remove_deleted_items(searcher, CourseKey.from_string(course_key_string), [])
        searcher.remove(CourseAboutSearchIndexer.DISCOVERY_DOCUMENT_TYPE, [course_key_string])

    def setUp(self):
        """
        Set up tests.
        """
        super(ReindexCourseTeamTest, self).setUp()

        self.team1 = CourseTeamFactory(course_id=COURSE_KEY1, team_id='team1')
        self.team2 = CourseTeamFactory(course_id=COURSE_KEY1, team_id='team2')
        self.team3 = CourseTeamFactory(course_id=COURSE_KEY1, team_id='team3')

        self.search_engine = SearchEngine.get_search_engine(index='index_course_team')

    def handle(self, *args, **options):
        """
        By convention set by Django developers, this method actually executes command's actions.
        So, there could be no better docstring than emphasize this once again.
        """
        course_ids = options['course_ids']
        all_option = options['all']
        setup_option = options['setup']
        index_all_courses_option = all_option or setup_option

        if (not len(course_ids) and not index_all_courses_option) or \
                (len(course_ids) and index_all_courses_option):
            raise CommandError("reindex_course requires one or more <course_id>s OR the --all or --setup flags.")

        store = modulestore()

        if index_all_courses_option:
            index_name = CoursewareSearchIndexer.INDEX_NAME
            doc_type = CoursewareSearchIndexer.DOCUMENT_TYPE
            if setup_option:
                try:
                    # try getting the ElasticSearch engine
                    searcher = SearchEngine.get_search_engine(index_name)
                except exceptions.ElasticsearchException as exc:
                    logging.exception(u'Search Engine error - %s', exc)
                    return

                index_exists = searcher._es.indices.exists(index=index_name)  # pylint: disable=protected-access
                doc_type_exists = searcher._es.indices.exists_type(  # pylint: disable=protected-access
                    index=index_name,
                    doc_type=doc_type
                )

                index_mapping = searcher._es.indices.get_mapping(  # pylint: disable=protected-access
                    index=index_name,
                    doc_type=doc_type
                ) if index_exists and doc_type_exists else {}

                if index_exists and index_mapping:
                    return

            # if reindexing is done during devstack setup step, don't prompt the user
            if setup_option or query_yes_no(self.CONFIRMATION_PROMPT, default="no"):
                # in case of --setup or --all, get the list of course keys from all courses
                # that are stored in the modulestore
                course_keys = [course.id for course in modulestore().get_courses()]
            else:
                return
        else:
            # in case course keys are provided as arguments
            course_keys = map(self._parse_course_key, course_ids)

        for course_key in course_keys:
            CoursewareSearchIndexer.do_course_reindex(store, course_key)

    def test_task_indexing_course(self):
        """ Making sure that the receiver correctly fires off the task when invoked by signal """
        searcher = SearchEngine.get_search_engine(CoursewareSearchIndexer.INDEX_NAME)
        response = searcher.search(field_dictionary={"course": unicode(self.course.id)})
        self.assertEqual(response["total"], 0)

        listen_for_course_publish(self, self.course.id)

        # Note that this test will only succeed if celery is working in inline mode
        response = searcher.search(field_dictionary={"course": unicode(self.course.id)})
        self.assertEqual(response["total"], 3)

    def test_task_library_update(self):
        """ Making sure that the receiver correctly fires off the task when invoked by signal """
        searcher = SearchEngine.get_search_engine(LibrarySearchIndexer.INDEX_NAME)
        library_search_key = unicode(normalize_key_for_search(self.library.location.library_key))
        response = searcher.search(field_dictionary={"library": library_search_key})
        self.assertEqual(response["total"], 0)

        listen_for_library_update(self, self.library.location.library_key)

        # Note that this test will only succeed if celery is working in inline mode
        response = searcher.search(field_dictionary={"library": library_search_key})
        self.assertEqual(response["total"], 2)

    def remove_deleted_items(cls, structure_key):
        """ Remove item from Course About Search_index """
        searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
        if not searcher:
            return

        response = searcher.search(
            doc_type=cls.DISCOVERY_DOCUMENT_TYPE,
            field_dictionary=cls._get_location_info(structure_key)
        )
        result_ids = [result["data"]["id"] for result in response["results"]]
        searcher.remove(cls.DISCOVERY_DOCUMENT_TYPE, result_ids)

 def test_abstract_impl(self):
     """ Make sure that if one tries to use the abstract base, then operations yeild NotImplementedError s """
     abstract = SearchEngine("test_index_name")
     test_string = "A test string"
     with self.assertRaises(NotImplementedError):
         abstract.index("test_doc", [{"name": test_string}])
     with self.assertRaises(NotImplementedError):
         abstract.search(test_string)
     with self.assertRaises(NotImplementedError):
         abstract.remove("test_doc", ["test_id"])

 def _problem_type_filter(self, library, capa_type):
     """ Filters library children by capa type"""
     search_engine = SearchEngine.get_search_engine(index="library_index")
     if search_engine:
         filter_clause = {
             "library": unicode(normalize_key_for_search(library.location.library_key)),
             "content_type": CapaDescriptor.INDEX_CONTENT_TYPE,
             "problem_types": capa_type
         }
         search_result = search_engine.search(field_dictionary=filter_clause)
         results = search_result.get('results', [])
         return [LibraryUsageLocator.from_string(item['data']['id']) for item in results]
     else:
         return [key for key in library.children if self._filter_child(key, capa_type)]

def delete_temp_user_task(request, user_id):

    profile = UserProfile.objects.get(pk=user_id)
    user = User.objects.get(pk=profile.user_id)

    courses = [format_course_for_view(c) for c in get_courses_accessible_to_user(request, user)[0]]
    libraries = [format_library_for_view(lib, user) for lib in accessible_libraries_list(user)]

    for course in courses:
        course_key = CourseKey.from_string(course["course_key"])
        delete_course_and_groups(course_key, user.id)
        searcher = SearchEngine.get_search_engine(CoursewareSearchIndexer.INDEX_NAME)
        if searcher != None:
            CoursewareSearchIndexer.remove_deleted_items(searcher, CourseKey.from_string(course_key_string), [])
            searcher.remove(CourseAboutSearchIndexer.DISCOVERY_DOCUMENT_TYPE, [course_key_string])

    for library in libraries:
        library_key = CourseKey.from_string(library['library_key'])
        delete_course_and_groups(library_key, user.id)

    def add_to_search_index(modulestore, location, delete=False, raise_on_error=False):
        """
        Add to courseware search index from given location and its children
        """
        error_list = []
        # TODO - inline for now, need to move this out to a celery task
        searcher = SearchEngine.get_search_engine(INDEX_NAME)
        if not searcher:
            return

        if isinstance(location, CourseLocator):
            course_key = location
        else:
            course_key = location.course_key

        location_info = {
            "course": unicode(course_key),
        }

        def _fetch_item(item_location):
            """ Fetch the item from the modulestore location, log if not found, but continue """
            try:
                if isinstance(item_location, CourseLocator):
                    item = modulestore.get_course(item_location)
                else:
                    item = modulestore.get_item(item_location, revision=ModuleStoreEnum.RevisionOption.published_only)
            except ItemNotFoundError:
                log.warning('Cannot find: %s', item_location)
                return None

            return item

        def index_item_location(item_location, current_start_date):
            """ add this item to the search index """
            item = _fetch_item(item_location)
            if not item:
                return

            is_indexable = hasattr(item, "index_dictionary")
            # if it's not indexable and it does not have children, then ignore
            if not is_indexable and not item.has_children:
                return

            # if it has a defined start, then apply it and to it's children
            if item.start and (not current_start_date or item.start > current_start_date):
                current_start_date = item.start

            if item.has_children:
                for child_loc in item.children:
                    index_item_location(child_loc, current_start_date)

            item_index = {}
            item_index_dictionary = item.index_dictionary() if is_indexable else None

            # if it has something to add to the index, then add it
            if item_index_dictionary:
                try:
                    item_index.update(location_info)
                    item_index.update(item_index_dictionary)
                    item_index['id'] = unicode(item.scope_ids.usage_id)
                    if current_start_date:
                        item_index['start_date'] = current_start_date

                    searcher.index(DOCUMENT_TYPE, item_index)
                except Exception as err:  # pylint: disable=broad-except
                    # broad exception so that index operation does not fail on one item of many
                    log.warning('Could not index item: %s - %s', item_location, unicode(err))
                    error_list.append(_('Could not index item: {}').format(item_location))

        def remove_index_item_location(item_location):
            """ remove this item from the search index """
            item = _fetch_item(item_location)
            if item:
                if item.has_children:
                    for child_loc in item.children:
                        remove_index_item_location(child_loc)

                searcher.remove(DOCUMENT_TYPE, unicode(item.scope_ids.usage_id))

        try:
            if delete:
                remove_index_item_location(location)
            else:
                index_item_location(location, None)
        except Exception as err:  # pylint: disable=broad-except
            # broad exception so that index operation does not prevent the rest of the application from working
            log.exception(
                "Indexing error encountered, courseware index may be out of date %s - %s",
                course_key,
                unicode(err)
            )
            error_list.append(_('General indexing error occurred'))

        if raise_on_error and error_list:
            raise SearchIndexingError(_('Error(s) present during indexing'), error_list)

 def searcher(self):
     """ Centralized call to getting the search engine for the test """
     return SearchEngine.get_search_engine(self.INDEX_NAME)

    def index_about_information(cls, modulestore, course):
        """
        Add the given course to the course discovery index

        Arguments:
        modulestore - modulestore object to use for operations

        course - course object from which to take properties, locate about information
        """
        searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
        if not searcher:
            return

        course_id = unicode(course.id)
        course_info = {
            'id': course_id,
            'course': course_id,
            'content': {},
            'image_url': course_image_url(course),
        }

        # load data for all of the 'about' modules for this course into a dictionary
        about_dictionary = {
            item.location.name: item.data
            for item in modulestore.get_items(course.id, qualifiers={"category": "about"})
        }

        about_context = {
            "course": course,
            "about_dictionary": about_dictionary,
        }

        for about_information in cls.ABOUT_INFORMATION_TO_INCLUDE:
            # Broad exception handler so that a single bad property does not scupper the collection of others
            try:
                section_content = about_information.get_value(**about_context)
            except:  # pylint: disable=bare-except
                section_content = None
                log.warning(
                    "Course discovery could not collect property %s for course %s",
                    about_information.property_name,
                    course_id,
                    exc_info=True,
                )

            if section_content:
                if about_information.index_flags & AboutInfo.ANALYSE:
                    analyse_content = section_content
                    if isinstance(section_content, basestring):
                        analyse_content = strip_html_content_to_text(section_content)
                    course_info['content'][about_information.property_name] = analyse_content
                    if about_information.property_name == "more_info":
                        course_info[about_information.property_name] = analyse_content
                if about_information.index_flags & AboutInfo.PROPERTY:
                    course_info[about_information.property_name] = section_content

        # Broad exception handler to protect around and report problems with indexing
        try:
            searcher.index(cls.DISCOVERY_DOCUMENT_TYPE, [course_info])
        except:  # pylint: disable=bare-except
            log.exception(
                "Course discovery indexing error encountered, course discovery index may be out of date %s",
                course_id,
            )
            raise

        log.debug(
            "Successfully added %s course to the course discovery index",
            course_id
        )