Python QueryParser.remove_plugin_class方法代码示例

# 需要导入模块: from whoosh.qparser import QueryParser [as 别名]
# 或者: from whoosh.qparser.QueryParser import remove_plugin_class [as 别名]
    def __init__(self, index_dir, var_path):
        self._index = None
        try:
            self._index = wh_index.open_dir(index_dir)
        except wh_index.IndexError:
            raise IndexError

        self._var_reader = self._make_var_reader(var_path)

        op = OperatorsPlugin(
            And=r"\bAND\b|&", Or=None,  # r"\bOR\b|\|",
            Not=r"\bNOT\b|\s+-", AndMaybe=None, Require=None)
        parser = QueryParser('content', _schema,
                             termclass=my_variations(self._var_reader))
        parser.remove_plugin_class(RangePlugin)
        parser.remove_plugin_class(BoostPlugin)
        parser.remove_plugin_class(WildcardPlugin)
        parser.replace_plugin(op)
        self._parser = parser

        parser_wild = QueryParser('content', _schema,
                                  termclass=my_variations(self._var_reader))
        parser_wild.remove_plugin_class(RangePlugin)
        parser_wild.remove_plugin_class(BoostPlugin)
        parser_wild.replace_plugin(op)
        self._parser_wild = parser_wild

        op_filter = OperatorsPlugin(And=r"\bAND\b", Or=r"\bOR\b",
                                    Not=None, AndMaybe=None, Require=None)
        asf_parser = QueryParser('asfilter', _schema)
        asf_parser.replace_plugin(op_filter)
        self._asf_parser = asf_parser

# 需要导入模块: from whoosh.qparser import QueryParser [as 别名]
# 或者: from whoosh.qparser.QueryParser import remove_plugin_class [as 别名]
def predict_TF_IDF(data, docs_per_q):
    # index docs
    exclude = set(string.punctuation)

    res = []

    for idx, row in data.iterrows():
        print row["id"]
        # get answers words
        w_A = set(utils.tokenize(row["answerA"]))
        w_B = set(utils.tokenize(row["answerB"]))
        w_C = set(utils.tokenize(row["answerC"]))
        w_D = set(utils.tokenize(row["answerD"]))

        sc_A = 0
        sc_B = 0
        sc_C = 0
        sc_D = 0

        q_punc = row["question"]  # first thing to debug if not working
        question = "".join(ch for ch in q_punc if ch not in exclude)
        qp = QueryParser("content", schema=schema, group=qparser.OrGroup)
        qp.add_plugin(qparser.FuzzyTermPlugin())
        qp.remove_plugin_class(qparser.PhrasePlugin)
        qp.add_plugin(qparser.SequencePlugin())
        q = qp.parse(unicode(question, "utf-8"))
        # q = qp.parse('physics')
        # cp = qparser.CompoundsPlugin( AndMaybe="&~")
        with ix.searcher() as s, ix.searcher(weighting=scoring.TF_IDF()) as scoring_searcher_tfidf:
            results = s.search(q, limit=docs_per_q)
            """
            u_id = unicode(uuid.uuid1())
            if not os.path.exists("/home/evan/Desktop/Kaggle/allen/glove/kaggle_allen/data/whoosh7/%s" % u_id):
                os.mkdir("/home/evan/Desktop/Kaggle/allen/glove/kaggle_allen/data/whoosh7/%s" % u_id)
            q_ix = index.create_in("/home/evan/Desktop/Kaggle/allen/glove/kaggle_allen/data/whoosh7/%s" % u_id, schema)
            q_writer = q_ix.writer()
            for document in results:
                q_writer.add_document(article_title=document['article_title'], content=document['content'])
            q_writer.commit()
            """
            # with q_ix.searcher(weighting=scoring.TF_IDF()) as scoring_searcher_tfidf
            for document in results:
                doc_parser = QueryParser("content", schema=schema)
                doc_q = doc_parser.parse(u"article_title:%s" % document["article_title"])
                for w in w_A:
                    try:
                        sc_A += (
                            scoring.TF_IDF()
                            .scorer(scoring_searcher_tfidf, "content", w)
                            .score(doc_q.matcher(scoring_searcher_tfidf))
                        )
                    except TermNotFound:
                        pass
                for w in w_B:
                    try:
                        sc_B += (
                            scoring.TF_IDF()
                            .scorer(scoring_searcher_tfidf, "content", w)
                            .score(doc_q.matcher(scoring_searcher_tfidf))
                        )
                    except TermNotFound:
                        pass
                for w in w_C:
                    try:
                        sc_C += (
                            scoring.TF_IDF()
                            .scorer(scoring_searcher_tfidf, "content", w)
                            .score(doc_q.matcher(scoring_searcher_tfidf))
                        )
                    except TermNotFound:
                        pass
                for w in w_D:
                    try:
                        sc_D += (
                            scoring.TF_IDF()
                            .scorer(scoring_searcher_tfidf, "content", w)
                            .score(doc_q.matcher(scoring_searcher_tfidf))
                        )
                    except TermNotFound:
                        pass

        res.append(["A", "B", "C", "D"][np.argmax([sc_A, sc_B, sc_C, sc_D])])

    return res

# 需要导入模块: from whoosh.qparser import QueryParser [as 别名]
# 或者: from whoosh.qparser.QueryParser import remove_plugin_class [as 别名]
from whoosh.index import create_in
from whoosh.fields import *
schema = Schema(title=TEXT(stored=True), content=TEXT)
ix = create_in("indexdir", schema)
writer = ix.writer()
writer.add_document(title=u"First document", content=u"This is the first document we've added!")
writer.add_document(title=u"Second document", content=u"The second one is even more interesting!")
writer.add_document(title=u"Third document", content=u"letter first, stamp second, mail third")
writer.add_document(title=u"Fourth document", content=u"stamp first, mail third")
writer.add_document(title=u"Fivth document", content=u"letter first,  mail third")
writer.add_document(title=u"Sixth document", content=u"letters first, stamps second, mail third")
writer.add_document(title=u"Seventh document", content=u"stamp first, letters second, mial third")
writer.commit()


from whoosh.qparser import QueryParser, FuzzyTermPlugin, PhrasePlugin, SequencePlugin
with ix.searcher() as searcher:
    parser = QueryParser(u"content", ix.schema)
    parser.add_plugin(FuzzyTermPlugin())
    parser.remove_plugin_class(PhrasePlugin)
    parser.add_plugin(SequencePlugin())
    query = parser.parse(u"Apple iphone 6")
    print query
    results = searcher.search(query)
    print "nb of results =", len(results)
    for r in results:
        print r

# 需要导入模块: from whoosh.qparser import QueryParser [as 别名]
# 或者: from whoosh.qparser.QueryParser import remove_plugin_class [as 别名]
# schema: a :class:`whoosh.fields.Schema` object to use when parsing. The appropriate fields in the schema will be used to
#     tokenize terms/phrases before they are turned into query objects.
#     You can specify None for the schema to create a parser that does not analyze the text of the query, usually for testing purposes.
parser = QueryParser("content", ix.schema)  # ix.schema 和 schema 是相同的东西
print(len(parser.plugins), parser.plugins)  # 11
# [<whoosh.qparser.plugins.WhitespacePlugin>, <whoosh.qparser.plugins.WhitespacePlugin>, <whoosh.qparser.plugins.SingleQuotePlugin>,
#  <whoosh.qparser.plugins.FieldsPlugin>,     <whoosh.qparser.plugins.WildcardPlugin>,   <whoosh.qparser.plugins.PhrasePlugin>,
#  <whoosh.qparser.plugins.RangePlugin>,      <whoosh.qparser.plugins.GroupPlugin>,      <whoosh.qparser.plugins.OperatorsPlugin>,
#  <whoosh.qparser.plugins.BoostPlugin>,      <whoosh.qparser.plugins.EveryPlugin>]
## default_set(): Returns the default list of plugins to use.
print(len(parser.default_set()), parser.default_set())  # 10
# [<whoosh.qparser.plugins.WhitespacePlugin>, <whoosh.qparser.plugins.SingleQuotePlugin>, <whoosh.qparser.plugins.FieldsPlugin>,
#  <whoosh.qparser.plugins.WildcardPlugin>,   <whoosh.qparser.plugins.PhrasePlugin>,      <whoosh.qparser.plugins.RangePlugin>,
#  <whoosh.qparser.plugins.GroupPlugin>,      <whoosh.qparser.plugins.OperatorsPlugin>,   <whoosh.qparser.plugins.BoostPlugin>,
#  <whoosh.qparser.plugins.EveryPlugin>]
parser.remove_plugin_class(whoosh.qparser.plugins.WildcardPlugin)
print(len(parser.plugins), len(parser.default_set()))  # 10 10
parser.add_plugin(qparser.PrefixPlugin)
print(len(parser.plugins), len(parser.default_set()))  # 11 10
## parse(text, normalize=True, debug=False) Parses the input string and returns a :class:`whoosh.query.Query` object/tree.
query = parser.parse('document')
## search(q, **kwargs) Runs a :class:`whoosh.query.Query` object on this searcher and returns a :class:`Results` object.
# See :doc:`/searching` for more information.
results = searcher.search(query)  # 检索 "content" 中出现 "document"
print(results)  # <Top 1 Results for Term('content', 'document') runtime=0.0015511049998622184>
print(type(results))  # <class 'whoosh.searching.Results'>

## 查询方法二: 上面两行是只用方法, 下面一行也形
## find(defaultfield, querystring, **kwargs)
results = searcher.find("title", "document")  # 检索标题中出现 'document' 的文档
print(results)  # <Top 2 Results for Term('title', 'document') runtime=0.0008875329999682435>