Python util.now函数代码示例

 def index(self, lib):
     print "Indexing with %s..." % lib
     
     options = self.options
     chunk = int(options.chunk)
     skip = int(options.skip)
     upto = int(options.upto)
     count = 0
     skipc = skip
     
     starttime = chunkstarttime = now()
     lib.indexer()
     for d in self.spec.documents():
         skipc -= 1
         if not skipc:
             lib.index_document(d)
             count += 1
             skipc = skip
             if chunk and not count % chunk:
                 t = now()
                 sofar = t - starttime
                 print "Done %d docs, %0.3f secs for %d, %0.3f total, %0.3f docs/s" % (count, t - chunkstarttime, chunk, sofar, count/sofar)
                 chunkstarttime = t
             if count > upto:
                 break
     
     spooltime = now()
     print "Spool time:", spooltime - starttime
     lib.finish()
     committime = now()
     print "Commit time:", committime - spooltime
     print "Total time to index", count, "documents:",  committime - starttime

    def search(self, lib):
        lib.searcher()

        t = now()
        q = lib.query()
        print "Query:", q
        r = lib.find(q)
        print "Search time:", now() - t

        t = now()
        self.spec.print_results(lib.results(r))
        print "Print time:", now() - t

    def search_file(self, lib):
        f = open(self.options.termfile, "rb")
        terms = [line.strip() for line in f]
        f.close()

        print "Searching %d terms with %s" % (len(terms), lib)
        lib.searcher()
        starttime = now()
        for r in lib.findterms(terms):
            pass
        searchtime = now() - starttime
        print "Search time:", searchtime, "searches/s:", float(len(terms)) / searchtime

 def finish(self, doccount, lengthfile, termtable, postingwriter):
     _fieldlength_totals = self._fieldlength_totals
     if not self.tasks:
         return
     
     pqueue = self.postingqueue
     rqueue = self.resultsqueue
     
     for _ in xrange(self.procs):
         pqueue.put((-1, doccount))
     
     #print "Joining..."
     t = now()
     for task in self.tasks:
         task.join()
     #print "Join:", now() - t
     
     #print "Getting results..."
     t = now()
     runs = []
     lenfilenames = []
     for task in self.tasks:
         taskruns, flentotals, flenmaxes, lenfilename = rqueue.get()
         runs.extend(taskruns)
         lenfilenames.append(lenfilename)
         for fieldnum, total in flentotals.iteritems():
             _fieldlength_totals[fieldnum] += total
         for fieldnum, length in flenmaxes.iteritems():
             if length > self._fieldlength_maxes.get(fieldnum, 0):
                 self._fieldlength_maxes[fieldnum] = length
     #print "Results:", now() - t
     
     #print "Writing lengths..."
     t = now()
     lw = LengthWriter(lengthfile, doccount)
     for lenfilename in lenfilenames:
         sublengths = LengthReader(StructFile(open(lenfilename, "rb")), doccount)
         lw.add_all(sublengths)
         os.remove(lenfilename)
     lw.close()
     lengths = lw.reader()
     #print "Lengths:", now() - t
     
     t = now()
     iterator = imerge([read_run(runname, count) for runname, count in runs])
     total = sum(count for runname, count in runs)
     write_postings(self.schema, termtable, lengths, postingwriter, iterator)
     for runname, count in runs:
         os.remove(runname)
     #print "Merge:", now() - t
     
     self.cleanup()

    def cache_messages(self, archive, cache):
        print("Caching messages in %s..." % cache)

        if not os.path.exists(archive):
            raise Exception("Archive file %r does not exist" % archive)

        t = now()
        f = open(cache, "wb")
        c = 0
        for d in self.get_messages(archive):
            c += 1
            dump(d, f)
            if not c % 1000: print(c)
        f.close()
        print("Cached messages in ", now() - t, "seconds")

    def prepare(self, top_searcher, q, context):
        """This method is called before a search.

        Subclasses can override this to perform set-up work, but
        they should still call the superclass's method because it sets several
        necessary attributes on the collector object:

        self.top_searcher
            The top-level searcher.
        self.q
            The query object
        self.context
            ``context.needs_current`` controls whether a wrapping collector
            requires that this collector's matcher be in a valid state at every
            call to ``collect()``. If this is ``False``, the collector is free
            to use faster methods that don't necessarily keep the matcher
            updated, such as ``matcher.all_ids()``.

        :param top_searcher: the top-level :class:`whoosh.searching.Searcher`
            object.
        :param q: the :class:`whoosh.query.Query` object being searched for.
        :param context: a :class:`whoosh.searching.SearchContext` object
            containing information about the search.
        """

        self.top_searcher = top_searcher
        self.q = q
        self.context = context

        self.starttime = now()
        self.runtime = None
        self.docset = set()

    def index(self, lib):
        print("Indexing with %s..." % lib)

        options = self.options
        every = None if options.every is None else int(options.every)
        merge = options.merge
        chunk = int(options.chunk)
        skip = int(options.skip)
        upto = int(options.upto)
        count = 0
        skipc = skip

        starttime = chunkstarttime = now()

        lib.indexer()

        for d in self.spec.documents():
            skipc -= 1
            if not skipc:
                lib.index_document(d)
                count += 1
                skipc = skip
                if chunk and not count % chunk:
                    t = now()
                    sofar = t - starttime
                    print(
                        "Done %d docs, %0.3f secs for %d, %0.3f total, %0.3f docs/s"
                        % (count, t - chunkstarttime, chunk, sofar,
                           count / sofar))
                    chunkstarttime = t
                if count > upto:
                    break
                if every and not count % every:
                    print("----Commit")
                    lib.finish(merge=merge)
                    lib.indexer(create=False)

        spooltime = now()
        print("Spool time:", spooltime - starttime)
        lib.finish(merge=merge)
        committime = now()
        print("Commit time:", committime - spooltime)
        totaltime = committime - starttime
        print("Total time to index %d documents: %0.3f secs (%0.3f minutes)" %
              (count, totaltime, totaltime / 60.0))
        print("Indexed %0.3f docs/s" % (count / totaltime))

 def _complex_sort_query(self, q, limit=None, reverse=False, filter=None):
     t = now()
     if self.arrays is None:
         self._complex_cache()
     comb = self.searcher._filter_to_comb(filter)
     docnums = [docnum for docnum in self.searcher.docs_for_query(q)
                if (not comb) or docnum in comb]
     docnums.sort(key=self._complex_key_fn, reverse=reverse)
     docset = set(docnums)
     
     # I artificially enforce the limit here, even thought the current
     # implementation can't use it, so that the results don't change based
     # on single- vs- multi-segment.
     if limit:
         docnums = docnums[:limit]
     runtime = now() - t
     return self._results(q, docnums, docset, runtime)

def test_20000_single():
    sc = fields.Schema(id=fields.ID(stored=True), text=fields.TEXT)
    with TempIndex(sc, "20000single") as ix:
        domain = ["alfa", "bravo", "charlie", "delta", "echo", "foxtrot",
                  "golf", "hotel", "india", "juliet", "kilo", "lima"]

        t = now()
        for i in xrange(20000):
            w = ix.writer()
            w.add_document(id=text_type(i),
                           text=u(" ").join(random.sample(domain, 5)))
            w.commit()
        print("Write single:", now() - t)

        t = now()
        ix.optimize()
        print("Optimize single:", now() - t)

    def sort_query(self, query, sortedby, reverse=False):
        if isinstance(sortedby, basestring):
            sorter = self._field_sorter(sortedby)
        elif isinstance(sortedby, (list, tuple)):
            sorter = scoring.MultiFieldSorter([self._field_sorter(fname)
                                               for fname in sortedby])
        elif isinstance(sortedby, Sorter):
            sorter = sortedby
        else:
            raise ValueError("sortedby argument (%R) must be a string, list,"
                             " or Sorter" % sortedby)

        t = now()
        sorted_docs = list(sorter.order(self, query.docs(self), reverse=reverse))
        runtime = now() - t
        
        return Results(self, query, sorted_docs, None, runtime)

def search(qstring, ixdir, basedir, limit=None, optimize=True, scores=True):
    ix = index.open_dir(ixdir)
    qp = qparser.QueryParser("title", ix.schema)
    q = qp.parse(qstring)

    with ix.searcher(weighting=scoring.PL2()) as s:
        if scores:
            r = s.search(q, limit=limit, optimize=optimize)
            for hit in r:
                print_record(hit.rank, basedir, hit["file"], hit["pos"])
            print("Found %d records in %0.06f seconds" % (len(r), r.runtime))
        else:
            t = now()
            for i, docnum in enumerate(s.docs_for_query(q)):
                if not limit or i < limit:
                    fields = s.stored_fields(docnum)
                    print_record(i, basedir, fields["file"], fields["pos"])
            print("Found %d records in %0.06f seconds" % (i, now() - t))

def make_index(basedir, ixdir, procs=4, limitmb=128, multisegment=True,
               glob="*.mrc"):
    if not os.path.exists(ixdir):
        os.mkdir(ixdir)

    # Multi-lingual stop words
    stoplist = (analysis.STOP_WORDS
                | set("de la der und le die et en al no von di du da "
                      "del zur ein".split()))
    # Schema
    ana = analysis.StemmingAnalyzer(stoplist=stoplist)
    schema = fields.Schema(title=fields.TEXT(analyzer=ana),
                           author=fields.TEXT(phrase=False),
                           subject=fields.TEXT(analyzer=ana, phrase=False),
                           file=fields.STORED, pos=fields.STORED,
                           )

    # MARC fields to extract
    mfields = set(subjectfields)  # Subjects
    mfields.update("100 110 111".split())  # Author
    mfields.add("245")  # Title

    print("Indexing with %d processor(s) and %d MB per processor"
          % (procs, limitmb))
    c = 0
    t = now()
    ix = index.create_in(ixdir, schema)
    with ix.writer(procs=procs, limitmb=limitmb,
                   multisegment=multisegment) as w:
        filenames = [filename for filename in os.listdir(basedir)
                     if fnmatch.fnmatch(filename, glob)]
        for filename in filenames:
            path = os.path.join(basedir, filename)
            print("Indexing", path)
            f = open(path, 'rb')
            for x, pos in read_file(f, mfields):
                w.add_document(title=uni(title(x)), author=uni(author(x)),
                               subject=uni(subjects(x)),
                               file=filename, pos=pos)
                c += 1
            f.close()
        print("Committing...")
    print("Indexed %d records in %0.02f minutes" % (c, (now() - t) / 60.0))

def test_20000_buffered():
    from whoosh.writing import BufferedWriter

    sc = fields.Schema(id=fields.ID(stored=True), text=fields.TEXT)
    with TempIndex(sc, "20000buffered") as ix:
        domain = ["alfa", "bravo", "charlie", "delta", "echo", "foxtrot",
                  "golf", "hotel", "india", "juliet", "kilo", "lima"]

        t = now()
        w = BufferedWriter(ix, limit=100, period=None)
        for i in xrange(20000):
            w.add_document(id=text_type(i),
                           text=u(" ").join(random.sample(domain, 5)))
        w.close()
        print("Write buffered:", now() - t)

        t = now()
        ix.optimize()
        print("Optimize buffered:", now() - t)

 def index_document(self, d):
     try:
         self.archive.indexDictionary(str(self.count), d)
     except ValueError:
         print "d=", d
         raise
     self.count += 1
     if not self.count % int(self.options.batch):
         t = now()
         self.archive.store(lazy=True)
         self.indexer(create=False)

    def finish(self):
        """This method is called after a search.

        Subclasses can override this to perform set-up work, but
        they should still call the superclass's method because it sets several
        necessary attributes on the collector object:

        self.runtime
            The time (in seconds) the search took.
        """

        self.runtime = now() - self.starttime