Python Filter类代码示例

def getGroupWide(folder='../test/patents/US_out/full/'):
    """Return a set of terms used across an entire set of files."""
    parser = NPParser.NPParser()
    filters = Settings.getDocumentFilters()
    if 'stops' in filters:
        filters.remove('stops')
    termlist = []
    filenames = [f for f in os.listdir(folder) if f[-4:]=='.txt']
    filtfname = os.path.join(folder, 'filter.save')
    if os.path.exists(filtfname):
            Filter._get_stemdict(filtfname)
    for f in filenames:
        nps = parser.getTerms(os.path.join(folder,f), filters)
        termlist.append(nps)
#    if not os.path.exists(filtfname):
#        Filter._save_stemdict(filtfname)
    all_terms = set()
    for termset in termlist:
        all_terms.update(termset)
    retlist = set()
    for term in all_terms:
        count = 0
        for termset in termlist:
            if term in termset:
                count += 1
        if count > len(filenames)*0.2:
            if 'stem' in filters:
                retlist.update(Filter.unstem(term))
            else:
                retlist.add(term)
    return retlist

 def getTerms(self, filename, filters=[], relaxed=False):
     """Input file, output a FreqDist of terms"""
     filterfname = os.path.join(os.path.dirname(filename), "filter.save")
     if os.path.exists(filename + ".nps") and os.path.exists(filterfname):
         f = open(filename + ".nps")
         old_filters, fd = pickle.load(f)
         f.close()
         if old_filters == filters:
             if not Filter.unstemdict:
                 Filter._get_stemdict(filterfname)
             return fd
     NPs = self.getNPs(filename)
     fd = FreqDist()
     for NP in NPs:
         # get the possible terms for each NP
         terms = self.extractPossibleTerms(NP, relaxed)
         # filter each term by some given criteria
         # this requires keeping case information until this point
         # filt = Filter.Filter() # class containing all filters
         for t in terms:
             for f in filters:
                 t = Filter.criteria[f](t)
             if t:
                 fd[t] += 1
     f = open(filename + ".nps", "w")
     pickle.dump((filters, fd), f)
     f.close()
     if os.path.exists(filterfname):
         os.remove(filterfname)
     return fd

    def check_file(self, pkg, filename):
        if filename.startswith('/usr/lib/debug') or pkg.isSource():
            return
        if not stat.S_ISREG(pkg.files()[filename].mode):
            return

        if len(pkg.grep(self.build_root_re, filename)):
            Filter.printError(pkg, "file-contains-buildroot", filename)

 def check_file(self, pkg, filename):
     beam = BeamFile(pkg.files()[filename].path)
     if 'debug_info' not in beam.compileinfo['options']:
         Filter.printWarning(
             pkg, "beam-compiled-without-debug_info", filename)
     if not self.source_re.match(Pkg.b2s(beam.compileinfo['source'].value)):
         Filter.printWarning(
             pkg, "beam-was-not-recompiled", filename,
             beam.compileinfo['source'].value)

def loopProcess(url):
  r = requests.get(url)
  Filter.proc(url, r.text, r.headers)
# push all url into raw queue
  mats = re.findall(PATTERN_URL, r.text)
  if mats is not None:
    for mat in mats:
      link = mat[0]
      if not link.startswith('http://'):
        link = url[:url.find('/', 8)] + '/' + link
      URLService.pushRawUrl(link)

    def check(self, pkg):
        ghosts = pkg.ghostFiles()
        for filename in pkg.files():
            if filename in ghosts:
                continue

            if not stat.S_ISREG(pkg.files()[filename].mode):
                continue

            if wrong_compression(os.path.join(pkg.dirname, filename)):
                Filter.printError(pkg, 'files-wrong-compression', filename)

def playFilteredSound():
    Zero.list = []
    Pole.list = []

    for key in entryDictionary.keys():
        entryDictionary[key].complexRoot.addToList()

    soundFilter = Filter(Zero.list, Pole.list)
    filteredSamples = soundFilter.filterStream(originalSamples)

    snd.play(getWaveBytes(inputSoundInfo, filteredSamples))

    def check_file(self, pkg, filename):
        if pkg.isSource() or not stat.S_ISREG(pkg.files()[filename].mode):
            return

        if pkg.grep(self.suspicious_dir, filename):
            Filter.printError(pkg, "invalid-pkgconfig-file", filename)

        pc_file = file(pkg.dirName() + "/" + filename, "r")
        for l in pc_file:
            if l.startswith('Libs:') and self.wronglib_dir.search(l):
                Filter.printError(pkg, 'pkgconfig-invalid-libs-dir',
                                  filename, l)

def saveSound():
    Zero.list = []
    Pole.list = []

    for key in entryDictionary.keys():
        entryDictionary[key].complexRoot.addToList()

    soundFilter = Filter(Zero.list, Pole.list)
    filteredSamples = soundFilter.filterStream(originalSamples)

    filename = tkFileDialog.asksaveasfilename(filetypes=[("Waveform Audio", ".wav")],
        defaultextension='.wav')
    saveSoundToFile(inputSoundInfo, filteredSamples, filename)

 def grep(self, regex, filename):
     """Grep regex from a file, return matching line numbers."""
     ret = []
     lineno = 0
     try:
         with open(os.path.join(
                 self.dirName() or '/', filename.lstrip('/'))) as in_file:
             for line in in_file:
                 lineno += 1
                 if regex.search(line):
                     ret.append(str(lineno))
                     break
     except Exception as e:
         Filter.printWarning(self, 'read-error', filename, e)
     return ret

def run(args):
    usage = 'Usage: ' + args[0] + ' path [-terms|-chem|-dna|-group]'
    logging.basicConfig(level=LEVEL)
    if len(args) < 2 or len(args) > 3:
        print usage
        return None
    path = None
    getFunc = None
    for arg in args[1:]:
        if os.path.isdir(arg):
            path = arg
        elif arg == '-terms':
            getFunc = getPatentTerms
        elif arg == '-chem':
            getFunc = getChemicals
        elif arg == '-dna':
            getFunc = getDNA
        elif arg == '-group':
            getFunc = getGroupWide
    if not path or not getFunc:
        print usage
        return None
    path = os.path.abspath(path)
    logging.info('RDG path: '+path)
    logging.info('Get Function: '+getFunc.func_name)
    if getFunc.func_name == 'getGroupWide':
        terms = getFunc(path)
    else:
        logging.debug('Collecting File ids...')
        filenames = [f for f in os.listdir(path) if f[-4:]=='.txt']
        terms = []
        logging.debug('Finding terms...')
        filtfname = os.path.join(path, 'filter.save')
        if getFunc.func_name == 'getPatentTerms' and os.path.exists(filtfname):
            Filter._get_stemdict(filtfname)
        for f in filenames:
            logging.debug('...'+f+'...')
            terms.extend(getFunc(os.path.join(path,f)))
#        if getFunc.func_name == 'getPatentTerms' and not os.path.exists(filtfname):
#            Filter._save_stemdict(filtfname)
        logging.debug('Clean up...')
        if getFunc.func_name == 'getPatentTerms':
            temp = set()
            for t in terms:
                temp.update(Filter.unstem(t))
            terms = temp
        terms = set(terms)
    return terms

 def __init__(self, recordFactory, sensorView):
     self.recordFactory = recordFactory
     self.records = []
     self.frecuency = 30.0
     self.a = 0
     self.sensorView = sensorView
     self.filter = Filter.filter()

    def check_file(self, pkg, filename):
        if filename.startswith('/usr/lib/debug') or pkg.isSource():
            return

        if not stat.S_ISREG(pkg.files()[filename].mode):
            return

        grep_date = pkg.grep(self.istoday, filename)

        if len(grep_date):
            grep_time = pkg.grep(self.looksliketime, filename)

            if len(grep_time):
                Filter.printError(pkg, "file-contains-date-and-time", filename)
            else:
                Filter.printWarning(pkg, "file-contains-current-date",
                                    filename)

    def check(self, pkg):

        if pkg.isSource():
            return

        md5s = {}
        sizes = {}
        files = pkg.files()
        configFiles = pkg.configFiles()

        for f, pkgfile in files.items():
            if f in pkg.ghostFiles():
                continue

            if not stat.S_ISREG(pkgfile.mode):
                continue

            md5s.setdefault(pkgfile.md5, set()).add(f)
            sizes[pkgfile.md5] = pkgfile.size

        sum = 0
        for f in md5s:
            duplicates = md5s[f]
            if len(duplicates) == 1:
                continue

            one = duplicates.pop()
            one_is_config = False
            if one in configFiles:
                one_is_config = True

            partition = get_prefix(one)

            st = os.stat(pkg.dirName() + '/' + one)
            diff = 1 + len(duplicates) - st[stat.ST_NLINK]
            if diff <= 0:
                for dupe in duplicates:
                    if partition != get_prefix(dupe):
                        Filter.printError(pkg, "hardlink-across-partition",
                                          one, dupe)
                    if one_is_config and dupe in configFiles:
                        Filter.printError(pkg, "hardlink-across-config-files",
                                          one, dupe)
                continue

            for dupe in duplicates:
                if partition != get_prefix(dupe):
                    diff = diff - 1
            sum += sizes[f] * diff
            if sizes[f] and diff > 0:
                Filter.printWarning(pkg, 'files-duplicate', one,
                                    ":".join(duplicates))

        if sum > 100000:
            Filter.printError(pkg, 'files-duplicated-waste', sum)

 def convert(self, raw_data):
     raw_data_s = Filter.s(raw_data)
     if raw_data == u'n/a':
         return 0
     elif raw_data == u'< 1 year':
         return 1
     elif raw_data == u'10+ years':
         return 11
     return int(raw_data[0]) + 1