Python Cache.get方法代码示例

# 需要导入模块: from intelmq.lib.cache import Cache [as 别名]
# 或者: from intelmq.lib.cache.Cache import get [as 别名]
class ReverseDnsExpertBot(Bot):

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           )

    def process(self):
        event = self.receive_message()

        keys = ["source.%s", "destination.%s"]

        for key in keys:
            ip_key = key % "ip"

            if not event.contains(ip_key):
                continue

            ip = event.get(ip_key)
            ip_version = IPAddress.version(ip)
            ip_integer = IPAddress.to_int(ip)

            if ip_version == 4:
                minimum = MINIMUM_BGP_PREFIX_IPV4

            elif ip_version == 6:
                minimum = MINIMUM_BGP_PREFIX_IPV6

            else:
                self.logger.warning("Invalid IP version {}".format(ip_version))
                self.send_message(event)
                self.acknowledge_message()

            cache_key = bin(ip_integer)[2: minimum + 2]
            cachevalue = self.cache.get(cache_key)

            result = None
            if cachevalue:
                result = cachevalue
            else:
                rev_name = reversename.from_address(ip)
                try:
                    result = resolver.query(rev_name, "PTR")
                    expiration = result.expiration
                    result = result[0]
                except dns.exception.DNSException as e:
                    if isinstance(e, dns.resolver.NXDOMAIN):
                        continue
                else:
                    ttl = datetime.fromtimestamp(expiration) - datetime.now()
                    self.cache.set(cache_key, str(result),
                                   ttl=int(ttl.total_seconds()))

            if result is not None:
                event.add(key % 'reverse_dns', str(result), force=True)

        self.send_message(event)
        self.acknowledge_message()

# 需要导入模块: from intelmq.lib.cache import Cache [as 别名]
# 或者: from intelmq.lib.cache.Cache import get [as 别名]
class CymruExpertBot(Bot):

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           getattr(self.parameters, "redis_cache_password",
                                   None)
                           )

    def process(self):
        event = self.receive_message()

        keys = ["source.%s", "destination.%s"]

        for key in keys:
            ip_key = key % "ip"

            if ip_key not in event:
                continue

            ip = event.get(ip_key)
            ip_version = IPAddress.version(ip)
            ip_integer = IPAddress.to_int(ip)

            if ip_version == 4:
                minimum = MINIMUM_BGP_PREFIX_IPV4

            elif ip_version == 6:
                minimum = MINIMUM_BGP_PREFIX_IPV6

            else:
                raise ValueError('Unexpected IP version '
                                 '{!r}.'.format(ip_version))

            cache_key = bin(ip_integer)[2: minimum + 2]
            result_json = self.cache.get(cache_key)

            if result_json:
                result = json.loads(result_json)
            else:
                result = Cymru.query(ip)
                if not result:
                    continue
                result_json = json.dumps(result)
                self.cache.set(cache_key, result_json)

            for result_key, result_value in result.items():
                if result_key == 'registry' and result_value == 'other':
                    continue
                event.add(key % result_key, result_value, overwrite=True)

        self.send_message(event)
        self.acknowledge_message()

# 需要导入模块: from intelmq.lib.cache import Cache [as 别名]
# 或者: from intelmq.lib.cache.Cache import get [as 别名]
class CymruExpertBot(Bot):

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           )

    def process(self):
        event = self.receive_message()

        if event is None:
            self.acknowledge_message()
            return

        keys = ["source.%s", "destination.%s"]

        for key in keys:
            ip_key = key % "ip"

            if not event.contains(ip_key):
                continue

            ip = event.value(ip_key)
            ip_version = IPAddress.version(ip)
            ip_integer = IPAddress.to_int(ip)

            if ip_version == 4:
                minimum = MINIMUM_BGP_PREFIX_IPV4

            elif ip_version == 6:
                minimum = MINIMUM_BGP_PREFIX_IPV6

            else:
                raise ValueError('Unexpected IP version '
                                 '{!r}.'.format(ip_version))

            cache_key = bin(ip_integer)[2: minimum + 2]
            result_json = self.cache.get(cache_key)

            if result_json:
                result = json.loads(result_json)
            else:
                result = Cymru.query(ip)
                result_json = json.dumps(result)
                self.cache.set(cache_key, result_json)

            for result_key, result_value in result.items():
                event.add(key % result_key, result_value, sanitize=True,
                          force=True)

        self.send_message(event)
        self.acknowledge_message()

# 需要导入模块: from intelmq.lib.cache import Cache [as 别名]
# 或者: from intelmq.lib.cache.Cache import get [as 别名]
class ForwardDnsExpertBot(Bot):

    def init(self):
        print("joooo")
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           )

    def process(self):
        event = self.receive_message()
        if event is None:
            self.acknowledge_message()
            return

        messageSent = False
        for key in ["source.%s", "destination.%s"]:
            ip_key = key % "ip"
            fqdn_key = key % "fqdn"

            if event.contains(ip_key) or not event.contains(fqdn_key):
                continue

            fqdn = event[fqdn_key]
            cache_key = hash(fqdn)
            cachevalue = self.cache.get(cache_key)

            result = None
            if cachevalue:
                result = cachevalue
            else:
                soc = socket.getaddrinfo(fqdn, 0)
                try:
                    result = set([address[4][0] for address in soc])
                except socket.error as msg:
                    print(msg)
                    continue
                else:
                    self.cache.set(cache_key, result)

            print("VEEOF",result)
            for ip in result:
                print("ZDEE!!",ip,ip_key,fqdn_key)
                event.add(ip_key, ip, sanitize=True, force=True)
                self.send_message(event)
                messageSent = True

        if not messageSent:
            self.send_message(event)
        self.acknowledge_message()

# 需要导入模块: from intelmq.lib.cache import Cache [as 别名]
# 或者: from intelmq.lib.cache.Cache import get [as 别名]
class MicrosoftInterflowCollectorBot(CollectorBot):

    def check_ttl_time(self):
        """
        Checks if the cache's TTL is big enough compared to the chosen
        time frame so that the bot does not process the same data over and
        over.
        """
        if isinstance(self.time_match, datetime):  # absolute
            now = datetime.now(tz=pytz.timezone('UTC'))
            if now - timedelta(seconds=self.parameters.redis_cache_ttl) > self.time_match:
                raise ValueError("The cache's TTL must be higher than 'not_older_than', "
                                 "otherwise the bot is processing the same data over and over again.")

    def init(self):
        if requests is None:
            raise ValueError('Could not import requests. Please install it.')

        self.set_request_parameters()
        self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
        if self.parameters.file_match:
            self.file_match = re.compile(self.parameters.file_match)
        else:
            self.file_match = None

        if self.parameters.not_older_than:
            try:
                self.time_match = timedelta(minutes=parse_relative(self.parameters.not_older_than))
            except ValueError:
                if sys.version_info >= (3, 6):
                    self.time_match = parser.parse(self.parameters.not_older_than).astimezone(pytz.utc)
                else:  # "astimezone() cannot be applied to a naive datetime" otherwise
                    if '+' not in self.parameters.not_older_than:
                        self.parameters.not_older_than += '+00:00'
                    self.time_match = parser.parse(self.parameters.not_older_than)
                self.logger.info("Filtering files absolute %r.", self.time_match)
                self.check_ttl_time()
            else:
                self.logger.info("Filtering files relative %r.", self.time_match)
                if timedelta(seconds=self.parameters.redis_cache_ttl) < self.time_match:
                    raise ValueError("The cache's TTL must be higher than 'not_older_than', "
                                     "otherwise the bot is processing the same data over and over again.")
        else:
            self.time_match = None

        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           getattr(self.parameters, "redis_cache_password",
                                   None)
                           )

    def process(self):
        self.check_ttl_time()
        self.logger.debug('Downloading file list.')
        files = requests.get(URL_LIST,
                             auth=self.auth,
                             proxies=self.proxy,
                             headers=self.http_header,
                             verify=self.http_verify_cert,
                             cert=self.ssl_client_cert,
                             timeout=self.http_timeout_sec)
        files.raise_for_status()
        self.logger.debug('Downloaded file list, %s entries.', len(files.json()))
        for file in files.json():
            if self.cache.get(file['Name']):
                self.logger.debug('Processed file %s already.', file['Name'])
                continue
            if self.file_match and not self.file_match.match(file['Name']):
                self.logger.debug('File %r does not match filename filter.', file['Name'])
                continue
            filetime = parser.parse(file['LastModified'])
            if isinstance(self.time_match, datetime) and filetime < self.time_match:
                self.logger.debug('File %r does not match absolute time filter.', file['Name'])
                continue
            else:
                now = datetime.now(tz=pytz.timezone('UTC'))
                if isinstance(self.time_match, timedelta) and filetime < (now - self.time_match):
                    self.logger.debug('File %r does not match relative time filter.', file['Name'])
                    continue

            self.logger.debug('Processing file %r.', file['Name'])
            download_url = URL_DOWNLOAD % file['Name']
            download = requests.get(download_url,
                                    auth=self.auth,
                                    proxies=self.proxy,
                                    headers=self.http_header,
                                    verify=self.http_verify_cert,
                                    cert=self.ssl_client_cert,
                                    timeout=self.http_timeout_sec)
            download.raise_for_status()
            if download_url.endswith('.gz'):
                raw = gzip.open(io.BytesIO(download.content)).read().decode()
            else:
                raw = download.text
            report = self.new_report()
            report.add('feed.url', download_url)
            report.add('raw', raw)
            self.send_message(report)
#.........这里部分代码省略.........

# 需要导入模块: from intelmq.lib.cache import Cache [as 别名]
# 或者: from intelmq.lib.cache.Cache import get [as 别名]
class ReverseDnsExpertBot(Bot):

    def init(self):
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           getattr(self.parameters, "redis_cache_password",
                                   None)
                           )

    def process(self):
        event = self.receive_message()

        keys = ["source.%s", "destination.%s"]

        for key in keys:
            ip_key = key % "ip"

            if ip_key not in event:
                continue

            ip = event.get(ip_key)
            ip_version = IPAddress.version(ip)
            ip_integer = IPAddress.to_int(ip)

            if ip_version == 4:
                minimum = MINIMUM_BGP_PREFIX_IPV4

            elif ip_version == 6:
                minimum = MINIMUM_BGP_PREFIX_IPV6

            cache_key = bin(ip_integer)[2: minimum + 2]
            cachevalue = self.cache.get(cache_key)
            
            result = None
            if cachevalue == DNS_EXCEPTION_VALUE:
                continue
            elif cachevalue:
                result = cachevalue
            else:
                rev_name = reversename.from_address(ip)
                try:
                    results = resolver.query(rev_name, "PTR")
                    expiration = results.expiration
                    for result in results:
                        # use first valid result
                        if event.is_valid('source.reverse_dns', str(result)):
                            break
                    else:
                        raise InvalidPTRResult
                except (dns.exception.DNSException, InvalidPTRResult) as e:
                    # Set default TTL for 'DNS query name does not exist' error
                    ttl = None if isinstance(e, dns.resolver.NXDOMAIN) else \
                        getattr(self.parameters, "cache_ttl_invalid_response",
                                60)
                    self.cache.set(cache_key, DNS_EXCEPTION_VALUE, ttl)
                    result = None

                else:
                    ttl = datetime.fromtimestamp(expiration) - datetime.now()
                    self.cache.set(cache_key, str(result),
                                   ttl=int(ttl.total_seconds()))

            if result is not None:
                event.add(key % 'reverse_dns', str(result), overwrite=True)

        self.send_message(event)
        self.acknowledge_message()

# 需要导入模块: from intelmq.lib.cache import Cache [as 别名]
# 或者: from intelmq.lib.cache.Cache import get [as 别名]
class MicrosoftInterflowCollectorBot(CollectorBot):
    def init(self):
        self.set_request_parameters()
        self.http_header['Ocp-Apim-Subscription-Key'] = self.parameters.api_key
        if self.parameters.file_match:
            self.file_match = re.compile(self.parameters.file_match)
        else:
            self.file_match = None

        if self.parameters.not_older_than:
            try:
                self.time_match = timedelta(minutes=parse_relative(self.parameters.not_older_than))
            except ValueError:
                self.time_match = parser.parse(self.parameters.not_older_than)
                self.logger.info("Filtering files absolute %r.", self.time_match)
            else:
                self.logger.info("Filtering files relative %r.", self.time_match)
        else:
            self.time_match = None

        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           getattr(self.parameters, "redis_cache_password",
                                   None)
                           )

    def process(self):
        self.logger.debug('Downloading file list.')
        files = requests.get(URL_LIST,
                             auth=self.auth,
                             proxies=self.proxy,
                             headers=self.http_header,
                             verify=self.http_verify_cert,
                             cert=self.ssl_client_cert,
                             timeout=self.http_timeout_sec)
        files.raise_for_status()
        self.logger.debug('Downloaded file list, %s entries.', len(files.json()))
        for file in files.json():
            if self.cache.get(file['Name']):
                self.logger.debug('Processed file %s already.', file['Name'])
                continue
            if self.file_match and not self.file_match.match(file['Name']):
                self.logger.debug('File %r does not match filename filter.', file['Name'])
                continue
            filetime = parser.parse(file['LastModified'])
            if isinstance(self.time_match, datetime) and filetime < self.time_match:
                self.logger.debug('File %r does not match absolute time filter.', file['Name'])
                continue
            else:
                now = datetime.now(tz=pytz.timezone('UTC'))
                if isinstance(self.time_match, timedelta) and filetime < (now - self.time_match):
                    self.logger.debug('File %r does not match relative time filter.', file['Name'])
                    continue

            self.logger.debug('Processing file %r.', file['Name'])
            download_url = URL_DOWNLOAD % file['Name']
            download = requests.get(download_url,
                                    auth=self.auth,
                                    proxies=self.proxy,
                                    headers=self.http_header,
                                    verify=self.http_verify_cert,
                                    cert=self.ssl_client_cert,
                                    timeout=self.http_timeout_sec)
            download.raise_for_status()
            if download_url.endswith('.gz'):
                raw = gzip.open(io.BytesIO(download.content)).read().decode()
            else:
                raw = download.text
            report = self.new_report()
            report.add('feed.url', download_url)
            report.add('raw', raw)
            self.send_message(report)
            self.cache.set(file['Name'], True)

# 需要导入模块: from intelmq.lib.cache import Cache [as 别名]
# 或者: from intelmq.lib.cache.Cache import get [as 别名]
class CymruExpertBot(Bot):
    
    def init(self):
        self.cache = Cache(
                            self.parameters.redis_cache_host,
                            self.parameters.redis_cache_port,
                            self.parameters.redis_cache_db,
                            self.parameters.redis_cache_ttl
                          )

    
    def process(self):
        event = self.receive_message()
        
        if not event:
            self.acknowledge_message()
            return

        keys = ["source_%s", "destination_%s"]
        
        for key in keys:
            ip = event.value(key % "ip")
            
            if not ip:
                self.send_message(event)
                self.acknowledge_message()
                return

            elif utils.is_ipv4(ip):
                ip_version = 4
                ip_integer = utils.ip_to_int(ip)
                cache_key = bin(ip_integer)[2 : MINIMUM_BGP_PREFIX_IPV4 + 2]

            elif utils.is_ipv6(ip):
                ip_version = 6
                ip_integer = utils.ip_to_int(ip)
                cache_key = bin(ip_integer)[2 : MINIMUM_BGP_PREFIX_IPV6 + 2]

            else:
                self.send_message(event)
                self.acknowledge_message()
                return


            result_json = self.cache.get(cache_key)

            if result_json:
                result = json.loads(result_json)
            else:
                result = Cymru.query(ip, ip_version)
                result_json = json.dumps(result)
                self.cache.set(cache_key, result_json)
            
            if "asn" in result:
                event.clear(key % 'asn')
                event.add(key % 'asn',        result['asn'])
                
            if "bgp_prefix" in result:
                event.clear(key % 'bgp_prefix')
                event.add(key % 'bgp_prefix', result['bgp_prefix'])
                
            if "registry" in result:
                event.clear(key % 'registry')
                event.add(key % 'registry',   result['registry'])
                
            if "allocated" in result:
                event.clear(key % 'allocated')
                event.add(key % 'allocated',  result['allocated'])
                
            if "as_name" in result:
                event.clear(key % 'as_name')
                event.add(key % 'as_name',    result['as_name'])
                
            if "cc" in result:
                event.clear(key % 'cymru_cc')
                event.add(key % 'cymru_cc',   result['cc'])

        self.send_message(event)
        self.acknowledge_message()

# 需要导入模块: from intelmq.lib.cache import Cache [as 别名]
# 或者: from intelmq.lib.cache.Cache import get [as 别名]
class RIPEExpertBot(Bot):
    QUERY = {
        'db_ip': 'https://rest.db.ripe.net/abuse-contact/{}.json',
        'db_asn': 'https://rest.db.ripe.net/abuse-contact/as{}.json',
        'stat': 'https://stat.ripe.net/data/abuse-contact-finder/data.json?resource={}',
        'stat_geolocation': 'https://stat.ripe.net/data/maxmind-geo-lite/data.json?resource={}',
    }

    REPLY_TO_DATA = {
        'db_ip': lambda x: clean_string(x['abuse-contacts']['email']),
        'db_asn': lambda x: clean_string(x['abuse-contacts']['email']),
        'stat': lambda x: clean_string(x['data']['anti_abuse_contacts']['abuse_c'][0]['email']),
        'stat_geolocation': lambda x: clean_geo(x['data']['located_resources'][0]['locations'][0]),
    }

    GEOLOCATION_REPLY_TO_INTERNAL = {
        ('cc', 'country'),
        ('latitude', 'latitude'),
        ('longitude', 'longitude'),
        ('city', 'city')
    }

    def init(self):
        if requests is None:
            raise ValueError("Could not import 'requests'. Please install the package.")

        self.__mode = getattr(self.parameters, 'mode', 'append')
        self.__query = {
            "db_asn": getattr(self.parameters, 'query_ripe_db_asn', True),
            "db_ip": getattr(self.parameters, 'query_ripe_db_ip', True),
            "stat_asn": getattr(self.parameters, 'query_ripe_stat_asn', True),
            "stat_ip": getattr(self.parameters, 'query_ripe_stat_ip', True),
            "stat_geo": getattr(self.parameters, 'query_ripe_stat_geolocation', True)
        }

        self.__initialize_http_session()
        self.__initialize_cache()

    def __initialize_http_session(self):
        self.http_session = requests.Session()
        self.set_request_parameters()
        self.http_session.proxies.update(self.proxy)
        self.http_session.headers.update(self.http_header)
        self.http_session.verify = self.http_verify_cert
        self.http_session.cert = self.ssl_client_cert

    def __initialize_cache(self):
        cache_host = getattr(self.parameters, 'redis_cache_host')
        cache_port = getattr(self.parameters, 'redis_cache_port')
        cache_db = getattr(self.parameters, 'redis_cache_db')
        cache_ttl = getattr(self.parameters, 'redis_cache_ttl')
        if cache_host and cache_port and cache_db and cache_ttl:
            self.__cache = Cache(cache_host, cache_port, cache_db, cache_ttl,
                                 getattr(self.parameters, "redis_cache_password", None))

    def process(self):
        with self.event_context() as event:
            for target in {'source.', 'destination.'}:
                abuse_key = target + "abuse_contact"
                abuse = set(event.get(abuse_key).split(',')) if self.__mode == 'append' and abuse_key in event else set()

                asn = event.get(target + "asn", None)
                if asn:
                    if self.__query['stat_asn']:
                        abuse.update(self.__perform_cached_query('stat', asn))
                    if self.__query['db_asn']:
                        abuse.update(self.__perform_cached_query('db_asn', asn))

                ip = event.get(target + "ip", None)
                if ip:
                    if self.__query['stat_ip']:
                        abuse.update(self.__perform_cached_query('stat', ip))
                    if self.__query['db_ip']:
                        abuse.update(self.__perform_cached_query('db_ip', ip))
                    if self.__query['stat_geo']:
                        info = self.__perform_cached_query('stat_geolocation', ip)

                        should_overwrite = self.__mode == 'replace'

                        for local_key, ripe_key in self.GEOLOCATION_REPLY_TO_INTERNAL:
                            if ripe_key in info:
                                event.add(target + "geolocation." + local_key, info[ripe_key], overwrite=should_overwrite)

                event.add(abuse_key, ','.join(abuse), overwrite=True)

    @contextmanager
    def event_context(self):
        event = self.receive_message()
        try:
            yield event
        finally:
            self.send_message(event)
            self.acknowledge_message()

    def __perform_cached_query(self, type, resource):
        cached_value = self.__cache.get('{}:{}'.format(type, resource))
        if cached_value:
            if cached_value == CACHE_NO_VALUE:
                return {}
            else:
#.........这里部分代码省略.........

# 需要导入模块: from intelmq.lib.cache import Cache [as 别名]
# 或者: from intelmq.lib.cache.Cache import get [as 别名]
class RIPENCCExpertBot(Bot):

    def init(self):
        self.query_db_asn = getattr(self.parameters, 'query_ripe_db_asn', True)
        self.query_db_ip = getattr(self.parameters, 'query_ripe_db_ip', True)
        self.query_stat_asn = getattr(self.parameters, 'query_ripe_stat', True)
        self.query_stat_ip = getattr(self.parameters, 'query_ripe_stat', True)
        self.cache = Cache(self.parameters.redis_cache_host,
                           self.parameters.redis_cache_port,
                           self.parameters.redis_cache_db,
                           self.parameters.redis_cache_ttl,
                           )

    def process(self):
        event = self.receive_message()

        if event is None:
            self.acknowledge_message()
            return

        for key in ['source.', 'destination.']:
            ip_key = key + "ip"
            abuse_key = key + "abuse_contact"
            asn_key = key + "asn"

            ip = event.get(ip_key, None)
            if not ip:
                continue
            ip_version = IPAddress.version(ip)
            ip_integer = IPAddress.to_int(ip)

            if ip_version == 4:
                minimum = MINIMUM_BGP_PREFIX_IPV4

            elif ip_version == 6:
                minimum = MINIMUM_BGP_PREFIX_IPV6

            else:
                raise ValueError('Unexpected IP version '
                                 '{!r}.'.format(ip_version))

            cache_key = bin(ip_integer)[2: minimum + 2]
            cache_result = self.cache.get(cache_key)

            abuse = (event.get(abuse_key).split(',') if abuse_key in event
                     else [])

            if cache_result:
                cache_result = ast.literal_eval(cache_result)
                cache_result = [n.strip() for n in cache_result]
                abuse.extend(cache_result)

            else:
                asn = event.get(asn_key, None)
                if self.query_db_asn and asn:
                    abuse.extend(lib.query_asn(asn))
                if self.query_db_ip and ip:
                    abuse.extend(lib.query_ripedb(ip))
                if self.query_stat_asn and asn:
                    abuse.extend(lib.query_ripestat(asn))
                if self.query_stat_ip and ip:
                    abuse.extend(lib.query_ripestat(ip))
                self.cache.set(cache_key,abuse)

            event.add(abuse_key, ','.join(filter(None, set(abuse))), force=True)

        self.send_message(event)
        self.acknowledge_message()