Python Filter.filter方法代码示例

# 需要导入模块: from filter import Filter [as 别名]
# 或者: from filter.Filter import filter [as 别名]
def _generateArtifactList(options):
    # load configuration
    logging.info("Loading configuration...")
    config = Configuration()
    config.load(options)

    # build list
    logging.info("Building artifact list...")
    listBuilder = ArtifactListBuilder(config)
    artifactList = listBuilder.buildList()

    logging.debug("Generated list contents:")
    for gat in artifactList:
        priorityList = artifactList[gat]
        for priority in priorityList:
            versionList = priorityList[priority]
            for version in versionList:
                logging.debug("  %s:%s", gat, version)

    #filter list
    logging.info("Filtering artifact list...")
    listFilter = Filter(config)
    artifactList = listFilter.filter(artifactList)

    logging.debug("Filtered list contents:")
    for gat in artifactList:
        priorityList = artifactList[gat]
        for priority in priorityList:
            versionList = priorityList[priority]
            for version in versionList:
                logging.debug("  %s:%s", gat, version)

    logging.info("Artifact list generation done")
    return artifactList

# 需要导入模块: from filter import Filter [as 别名]
# 或者: from filter.Filter import filter [as 别名]
def test_filter():
    mappings = [{'source': 'title'}, {'source': 'description'}, {'source': 'adfasdf'}, {'no source': 'adfasdf'}]
    filter = Filter(mappings)
    result = {}
    result['items'] = [{
        'author': 'Fred',
        'guid': 1234,
        'title': 'Gee golly',
        'description': 'This is fun.'
    }]
    result = filter.filter(result)
    print result
    assert result['items'] == [{'title': 'Gee golly', 'description': 'This is fun.'}]

# 需要导入模块: from filter import Filter [as 别名]
# 或者: from filter.Filter import filter [as 别名]
class PushRedis(object):

    connect_interval = 100

    def __init__(self, pusher=None, kiss={}):
        self.pusher = pusher
        self.kiss = kiss

        self.target = self.kiss["target"]
        self.key_name = self.kiss["key_name"]
        self.split_key = self.kiss["split_key"]
        self.compile_key = self.kiss["compile_key"]
        self.extend_info = self.kiss["extend_info"]
        self.filt = Filter(self.target, self.compile_key)
        self.tailfile = Tailfile(self.target, self.split_key)

    def run(self):
        for i in self.tailfile:
            for j in self.filt.filter(i, self.extend_info):
                self.pusher.rpush(self.key_name, j)

# 需要导入模块: from filter import Filter [as 别名]
# 或者: from filter.Filter import filter [as 别名]
def worker(id, msg, host):
    url = msg['source'].strip()
    print "looking up etag for", url
    print host
    headers = {'User-Agent': firefox_user_agent}

    if url in host:
        if "etag" in host[url]:
            headers['If-None-Match'] = host[url]['etag']
        if "last-modified" in host[url]:
            headers['If-Modified-Since'] = host[url]['last-modified']

    req = urllib2.Request(url, None, headers)

    print "starting download"

    try:
        res = urllib2.urlopen(req)
        print "completed download"

    except urllib2.HTTPError as e:
        print HTTPResponses[e.code]
        save_job(_id=id, code=e.code)
        return

    except urllib2.URLError as e:
        if e.reason[0][0] == -3:
            print "request timed out"
            error = {'code': e.reason[0][0], 'msg': e.reason[0][1]}
            save_job(_id=id, error=error, code=500)
            return

    if 'parser' in msg and 'plugin_key' in msg['parser'] and msg['parser']['plugin_key'] in parsers:
        parser = globals()[msg['parser']['plugin_key']]()
        count, result = parser.parse(res.read())
        filter = Filter(msg['processor']['config']['mappings'])
        save_job(_id=id, count=count, result=filter.filter(result), code=200)

    host[url] = dict(res.info())
    print hosts.save(host)

# 需要导入模块: from filter import Filter [as 别名]
# 或者: from filter.Filter import filter [as 别名]
def _generateArtifactList(options, args):

    config = Configuration()
    if options.config or not args:
        # load configuration
        logging.info("Loading configuration...")
        config.load(options)
    else:
        # create configuration
        logging.info("Creating configuration...")
        config.create(options, args)

    # build list
    logging.info("Building artifact list...")
    listBuilder = ArtifactListBuilder(config)
    artifactList = listBuilder.buildList()

    logging.debug("Generated list contents:")
    _logAL(artifactList)

    #filter list
    logging.info("Filtering artifact list...")
    listFilter = Filter(config)
    artifactList = listFilter.filter(artifactList, options.threadnum)

    logging.debug("Filtered list contents:")
    _logAL(artifactList)

    logging.info("Artifact list generation done")

    if options.reportdir:
        logging.info("Generating repository analysis report")
        if hasattr(options, "reportname"):
            reporter.generate_report(options.reportdir, config, artifactList, options.reportname)
        else:
            reporter.generate_report(options.reportdir, config, artifactList, None)
        logging.info("Report has been generated")

    return artifactList

# 需要导入模块: from filter import Filter [as 别名]
# 或者: from filter.Filter import filter [as 别名]
from filter import Filter
import sys

# Get arguments from user
train = sys.argv[1]
messages = sys.argv[2]

# Open files for reading and writing
train_file = open(train, "rb")
messages_file = open(messages, "rb")
predictions_file = open("predictions.txt", "w")

# Create new filter and train it using the train-file
f = Filter()
f.train(train_file)

#filter the messages in messages_file, write results to predictions_file
f.filter(messages_file, predictions_file)

# Close all the files
train_file.close()
messages_file.close()
predictions_file.close()