本文整理汇总了Python中redis.StrictRedis.mget方法的典型用法代码示例。如果您正苦于以下问题:Python StrictRedis.mget方法的具体用法?Python StrictRedis.mget怎么用?Python StrictRedis.mget使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类redis.StrictRedis
的用法示例。
在下文中一共展示了StrictRedis.mget方法的10个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: Storage
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import mget [as 别名]
class Storage(object):
def __init__(self, host=None, port=None, *args, **kwargs):
if host is None:
host = os.environ.get('REDIS_HOST', 'localhost')
if port is None:
port = os.environ.get('REDIS_PORT', '6379')
self.redis = StrictRedis(host, port, *args, **kwargs)
def get_labels(self):
return [x.decode('utf-8') for x in self.redis.smembers('labels')]
def get_images(self, label):
return self.redis.smembers(label)
def get_repr_image_for_labels(self, labels):
if not labels:
return {}
keys = [u'repr_img:{}'.format(label) for label in labels]
return dict(zip(labels, self.redis.mget(*keys)))
示例2: Analyzer
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import mget [as 别名]
class Analyzer(Thread):
"""
The Analyzer class which controls the analyzer thread and spawned processes.
"""
def __init__(self, parent_pid):
"""
Initialize the Analyzer
Create the :obj:`self.anomalous_metrics` list
Create the :obj:`self.exceptions_q` queue
Create the :obj:`self.anomaly_breakdown_q` queue
"""
super(Analyzer, self).__init__()
self.redis_conn = StrictRedis(unix_socket_path=settings.REDIS_SOCKET_PATH)
self.daemon = True
self.parent_pid = parent_pid
self.current_pid = getpid()
self.anomalous_metrics = Manager().list()
self.exceptions_q = Queue()
self.anomaly_breakdown_q = Queue()
self.mirage_metrics = Manager().list()
def check_if_parent_is_alive(self):
"""
Self explanatory
"""
try:
kill(self.current_pid, 0)
kill(self.parent_pid, 0)
except:
exit(0)
def spin_process(self, i, unique_metrics):
"""
Assign a bunch of metrics for a process to analyze.
Multiple get the assigned_metrics to the process from Redis.
For each metric:
- unpack the `raw_timeseries` for the metric.
- Analyse each timeseries against `ALGORITHMS` to determine if it is
anomalous.
- If anomalous add it to the :obj:`self.anomalous_metrics` list
- Add what algorithms triggered to the :obj:`self.anomaly_breakdown_q`
queue
- If :mod:`settings.ENABLE_CRUCIBLE` is ``True``:
- Add a crucible data file with the details about the timeseries and
anomaly.
- Write the timeseries to a json file for crucible.
Add keys and values to the queue so the parent process can collate for:\n
* :py:obj:`self.anomaly_breakdown_q`
* :py:obj:`self.exceptions_q`
"""
spin_start = time()
logger.info('spin_process started')
# Discover assigned metrics
keys_per_processor = int(ceil(float(len(unique_metrics)) / float(settings.ANALYZER_PROCESSES)))
if i == settings.ANALYZER_PROCESSES:
assigned_max = len(unique_metrics)
else:
assigned_max = min(len(unique_metrics), i * keys_per_processor)
# Fix analyzer worker metric assignment #94
# https://github.com/etsy/skyline/pull/94 @languitar:worker-fix
assigned_min = (i - 1) * keys_per_processor
assigned_keys = range(assigned_min, assigned_max)
# Compile assigned metrics
assigned_metrics = [unique_metrics[index] for index in assigned_keys]
# Check if this process is unnecessary
if len(assigned_metrics) == 0:
return
# Multi get series
raw_assigned = self.redis_conn.mget(assigned_metrics)
# Make process-specific dicts
exceptions = defaultdict(int)
anomaly_breakdown = defaultdict(int)
# Distill timeseries strings into lists
for i, metric_name in enumerate(assigned_metrics):
self.check_if_parent_is_alive()
try:
raw_series = raw_assigned[i]
unpacker = Unpacker(use_list=False)
unpacker.feed(raw_series)
timeseries = list(unpacker)
anomalous, ensemble, datapoint = run_selected_algorithm(timeseries, metric_name)
#.........这里部分代码省略.........
示例3: Analyzer
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import mget [as 别名]
class Analyzer(Thread):
def __init__(self, parent_pid):
"""
Initialize the Analyzer
"""
super(Analyzer, self).__init__()
self.redis_conn = StrictRedis(unix_socket_path = settings.REDIS_SOCKET_PATH)
self.daemon = True
self.parent_pid = parent_pid
self.current_pid = getpid()
self.lock = Lock()
self.exceptions = Manager().dict()
self.anomaly_breakdown = Manager().dict()
self.anomalous_metrics = Manager().list()
def check_if_parent_is_alive(self):
"""
Self explanatory
"""
try:
kill(self.current_pid, 0)
kill(self.parent_pid, 0)
except:
exit(0)
def spin_process(self, i, unique_metrics):
"""
Assign a bunch of metrics for a process to analyze.
"""
# Discover assigned metrics
keys_per_processor = int(ceil(float(len(unique_metrics)) / float(settings.ANALYZER_PROCESSES)))
if i == settings.ANALYZER_PROCESSES:
assigned_max = len(unique_metrics)
else:
assigned_max = i * keys_per_processor
assigned_min = assigned_max - keys_per_processor
assigned_keys = range(assigned_min, assigned_max)
# Compile assigned metrics
assigned_metrics = [unique_metrics[index] for index in assigned_keys]
# Check if this process is unnecessary
if len(assigned_metrics) == 0:
return
# Multi get series
raw_assigned = self.redis_conn.mget(assigned_metrics)
# Make process-specific dicts
exceptions = defaultdict(int)
anomaly_breakdown = defaultdict(int)
# Distill timeseries strings into lists
for i, metric_name in enumerate(assigned_metrics):
self.check_if_parent_is_alive()
try:
raw_series = raw_assigned[i]
unpacker = Unpacker(use_list = False)
unpacker.feed(raw_series)
timeseries = list(unpacker)
anomalous, ensemble, datapoint = run_selected_algorithm(timeseries)
# If it's anomalous, add it to list
if anomalous:
metric = [datapoint, metric_name]
self.anomalous_metrics.append(metric)
# Get the anomaly breakdown - who returned True?
for index, value in enumerate(ensemble):
if value:
algorithm = settings.ALGORITHMS[index]
anomaly_breakdown[algorithm] += 1
# It could have been deleted by the Roomba
except AttributeError:
exceptions['DeletedByRoomba'] += 1
except TooShort:
exceptions['TooShort'] += 1
except Stale:
exceptions['Stale'] += 1
except Incomplete:
exceptions['Incomplete'] += 1
except Boring:
exceptions['Boring'] += 1
except:
exceptions['Other'] += 1
logger.info(traceback.format_exc())
# Collate process-specific dicts to main dicts
with self.lock:
for key, value in anomaly_breakdown.items():
if key not in self.anomaly_breakdown:
self.anomaly_breakdown[key] = value
else:
self.anomaly_breakdown[key] += value
for key, value in exceptions.items():
if key not in self.exceptions:
#.........这里部分代码省略.........
示例4: Analyzer
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import mget [as 别名]
class Analyzer(Thread):
def __init__(self, parent_pid):
"""
Initialize the Analyzer
"""
super(Analyzer, self).__init__()
self.redis_conn = StrictRedis(unix_socket_path = settings.REDIS_SOCKET_PATH)
self.daemon = True
self.parent_pid = parent_pid
self.current_pid = getpid()
self.anomalous_metrics = Manager().list()
self.exceptions_q = Queue()
self.anomaly_breakdown_q = Queue()
def check_if_parent_is_alive(self):
"""
Self explanatory
"""
try:
kill(self.current_pid, 0)
kill(self.parent_pid, 0)
except:
exit(0)
def send_graphite_metric(self, name, value):
if settings.GRAPHITE_HOST != '':
sock = socket.socket()
sock.connect((settings.CARBON_HOST.replace('http://', ''), settings.CARBON_PORT))
sock.sendall('%s %s %i\n' % (name, value, time()))
sock.close()
return True
return False
def spin_process(self, i, unique_metrics):
"""
Assign a bunch of metrics for a process to analyze.
"""
# Discover assigned metrics
keys_per_processor = int(ceil(float(len(unique_metrics)) / float(settings.ANALYZER_PROCESSES)))
if i == settings.ANALYZER_PROCESSES:
assigned_max = len(unique_metrics)
else:
assigned_max = i * keys_per_processor
assigned_min = assigned_max - keys_per_processor
assigned_keys = range(assigned_min, assigned_max)
# Compile assigned metrics
assigned_metrics = [unique_metrics[index] for index in assigned_keys]
# Check if this process is unnecessary
if len(assigned_metrics) == 0:
return
# Multi get series
raw_assigned = self.redis_conn.mget(assigned_metrics)
# Make process-specific dicts
exceptions = defaultdict(int)
anomaly_breakdown = defaultdict(int)
# Distill timeseries strings into lists
for i, metric_name in enumerate(assigned_metrics):
self.check_if_parent_is_alive()
try:
raw_series = raw_assigned[i]
unpacker = Unpacker(use_list = False)
unpacker.feed(raw_series)
timeseries = list(unpacker)
anomalous, ensemble, datapoint = run_selected_algorithm(timeseries, metric_name)
# If it's anomalous, add it to list
if anomalous:
base_name = metric_name.replace(settings.FULL_NAMESPACE, '', 1)
metric = [datapoint, base_name]
self.anomalous_metrics.append(metric)
# Get the anomaly breakdown - who returned True?
for index, value in enumerate(ensemble):
if value:
algorithm = settings.ALGORITHMS[index]
anomaly_breakdown[algorithm] += 1
# It could have been deleted by the Roomba
except TypeError:
exceptions['DeletedByRoomba'] += 1
except TooShort:
exceptions['TooShort'] += 1
except Stale:
exceptions['Stale'] += 1
except Boring:
exceptions['Boring'] += 1
except:
exceptions['Other'] += 1
logger.info(traceback.format_exc())
# Add values to the queue so the parent process can collate
for key, value in anomaly_breakdown.items():
#.........这里部分代码省略.........
示例5: Boundary
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import mget [as 别名]
#.........这里部分代码省略.........
assigned_keys = range(assigned_min, assigned_max)
# Compile assigned metrics
assigned_metrics_and_algos = [boundary_metrics[index] for index in assigned_keys]
if ENABLE_BOUNDARY_DEBUG:
logger.info('debug - printing assigned_metrics_and_algos')
for assigned_metric_and_algo in assigned_metrics_and_algos:
logger.info('debug - assigned_metric_and_algo - %s' % str(assigned_metric_and_algo))
# Compile assigned metrics
assigned_metrics = []
for i in assigned_metrics_and_algos:
assigned_metrics.append(i[0])
# unique unhashed things
def unique_noHash(seq):
seen = set()
return [x for x in seq if str(x) not in seen and not seen.add(str(x))]
unique_assigned_metrics = unique_noHash(assigned_metrics)
if ENABLE_BOUNDARY_DEBUG:
logger.info('debug - unique_assigned_metrics - %s' % str(unique_assigned_metrics))
logger.info('debug - printing unique_assigned_metrics:')
for unique_assigned_metric in unique_assigned_metrics:
logger.info('debug - unique_assigned_metric - %s' % str(unique_assigned_metric))
# Check if this process is unnecessary
if len(unique_assigned_metrics) == 0:
return
# Multi get series
try:
raw_assigned = self.redis_conn.mget(unique_assigned_metrics)
except:
logger.error("failed to mget assigned_metrics from redis")
return
# Make process-specific dicts
exceptions = defaultdict(int)
anomaly_breakdown = defaultdict(int)
# Reset boundary_algortims
all_boundary_algorithms = []
for metric in BOUNDARY_METRICS:
all_boundary_algorithms.append(metric[1])
# The unique algorithms that are being used
boundary_algorithms = unique_noHash(all_boundary_algorithms)
if ENABLE_BOUNDARY_DEBUG:
logger.info('debug - boundary_algorithms - %s' % str(boundary_algorithms))
discover_run_metrics = []
# Distill metrics into a run list
for i, metric_name, in enumerate(unique_assigned_metrics):
self.check_if_parent_is_alive()
try:
if ENABLE_BOUNDARY_DEBUG:
logger.info('debug - unpacking timeseries for %s - %s' % (metric_name, str(i)))
raw_series = raw_assigned[i]
unpacker = Unpacker(use_list=False)
unpacker.feed(raw_series)
timeseries = list(unpacker)
except Exception as e:
示例6: Analyzer
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import mget [as 别名]
class Analyzer(Thread):
def __init__(self, parent_pid, storage):
"""
Initialize the Analyzer
"""
super(Analyzer, self).__init__()
self.redis_conn = StrictRedis(unix_socket_path = settings.REDIS_SOCKET_PATH)
self.daemon = True
self.parent_pid = parent_pid
self.current_pid = getpid()
self.lock = Lock()
self.exceptions = Manager().dict()
self.anomaly_breakdown = Manager().dict()
self.anomalous_metrics = Manager().list()
self.storage = storage
self.alerter = Alerter(storage)
def check_if_parent_is_alive(self):
"""
Self explanatory
"""
try:
kill(self.current_pid, 0)
kill(self.parent_pid, 0)
except:
exit(0)
def spin_process(self, i, unique_metrics):
"""
Assign a bunch of metrics for a process to analyze.
"""
# Discover assigned metrics
keys_per_processor = int(ceil(float(len(unique_metrics)) / float(settings.ANALYZER_PROCESSES)))
if i == settings.ANALYZER_PROCESSES:
assigned_max = len(unique_metrics)
else:
assigned_max = i * keys_per_processor
assigned_min = assigned_max - keys_per_processor
assigned_keys = range(assigned_min, assigned_max)
# Compile assigned metrics
assigned_metrics = [unique_metrics[index] for index in assigned_keys]
# Check if this process is unnecessary
if len(assigned_metrics) == 0:
return
# Multi get series
raw_assigned = self.redis_conn.mget(assigned_metrics)
# Make process-specific dicts
exceptions = defaultdict(int)
anomaly_breakdown = defaultdict(int)
# Distill timeseries strings into lists
for i, metric_name in enumerate(assigned_metrics):
self.check_if_parent_is_alive()
try:
raw_series = raw_assigned[i]
unpacker = Unpacker(use_list = False)
unpacker.feed(raw_series)
timeseries = list(unpacker)
anomalous, ensemble, datapoint, ts = run_selected_algorithm(timeseries)
# If it's anomalous, add it to list
if anomalous:
base_name = metric_name.replace(settings.FULL_NAMESPACE, '', 1)
metric = [datapoint, base_name, ts]
self.anomalous_metrics.append(metric)
# Get the anomaly breakdown - who returned True?
for index, value in enumerate(ensemble):
if value:
algorithm = settings.ALGORITHMS[index]
anomaly_breakdown[algorithm] += 1
# It could have been deleted by the Roomba
except AttributeError:
exceptions['DeletedByRoomba'] += 1
except TooShort:
exceptions['TooShort'] += 1
except Stale:
exceptions['Stale'] += 1
except Incomplete:
exceptions['Incomplete'] += 1
except Boring:
exceptions['Boring'] += 1
except:
exceptions['Other'] += 1
logger.info(traceback.format_exc())
# Collate process-specific dicts to main dicts
with self.lock:
for key, value in anomaly_breakdown.items():
if key not in self.anomaly_breakdown:
self.anomaly_breakdown[key] = value
else:
self.anomaly_breakdown[key] += value
#.........这里部分代码省略.........
示例7: Boundary
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import mget [as 别名]
class Boundary(Thread):
def __init__(self, parent_pid):
"""
Initialize the Boundary
"""
super(Boundary, self).__init__()
self.redis_conn = StrictRedis(unix_socket_path=REDIS_SOCKET)
self.daemon = True
self.parent_pid = parent_pid
self.current_pid = getpid()
self.boundary_metrics = Manager().list()
self.anomalous_metrics = Manager().list()
self.exceptions_q = Queue()
self.anomaly_breakdown_q = Queue()
def check_if_parent_is_alive(self):
"""
Self explanatory
"""
try:
kill(self.current_pid, 0)
kill(self.parent_pid, 0)
except:
exit(0)
def unique_noHash(self, seq):
seen = set()
return [x for x in seq if str(x) not in seen and not seen.add(str(x))]
# This is to make a dump directory in /tmp if ENABLE_BOUNDARY_DEBUG is True
# for dumping the metric timeseries data into for debugging purposes
def mkdir_p(self, path):
try:
os.makedirs(path)
return True
except OSError as exc:
# Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
def spin_process(self, i, boundary_metrics):
"""
Assign a bunch of metrics for a process to analyze.
"""
# Determine assigned metrics
bp = settings.BOUNDARY_PROCESSES
bm_range = len(boundary_metrics)
keys_per_processor = int(ceil(float(bm_range) / float(bp)))
if i == settings.BOUNDARY_PROCESSES:
assigned_max = len(boundary_metrics)
else:
# This is a skyine bug, the original skyline code uses 1 as the
# beginning position of the index, python indices begin with 0
# assigned_max = len(boundary_metrics)
# This closes the etsy/skyline pull request opened by @languitar on 17 Jun 2014
# https://github.com/etsy/skyline/pull/94 Fix analyzer worker metric assignment
assigned_max = min(len(boundary_metrics), i * keys_per_processor)
assigned_min = (i - 1) * keys_per_processor
assigned_keys = range(assigned_min, assigned_max)
# Compile assigned metrics
assigned_metrics_and_algos = [boundary_metrics[index] for index in assigned_keys]
if ENABLE_BOUNDARY_DEBUG:
logger.info('debug :: printing assigned_metrics_and_algos')
for assigned_metric_and_algo in assigned_metrics_and_algos:
logger.info('debug :: assigned_metric_and_algo - %s' % str(assigned_metric_and_algo))
# Compile assigned metrics
assigned_metrics = []
for i in assigned_metrics_and_algos:
assigned_metrics.append(i[0])
# unique unhashed things
def unique_noHash(seq):
seen = set()
return [x for x in seq if str(x) not in seen and not seen.add(str(x))]
unique_assigned_metrics = unique_noHash(assigned_metrics)
if ENABLE_BOUNDARY_DEBUG:
logger.info('debug :: unique_assigned_metrics - %s' % str(unique_assigned_metrics))
logger.info('debug :: printing unique_assigned_metrics:')
for unique_assigned_metric in unique_assigned_metrics:
logger.info('debug :: unique_assigned_metric - %s' % str(unique_assigned_metric))
# Check if this process is unnecessary
if len(unique_assigned_metrics) == 0:
return
# Multi get series
try:
raw_assigned = self.redis_conn.mget(unique_assigned_metrics)
except:
logger.error('error :: failed to mget assigned_metrics from redis')
return
# Make process-specific dicts
exceptions = defaultdict(int)
#.........这里部分代码省略.........
示例8: Notes
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import mget [as 别名]
#.........这里部分代码省略.........
'blue': 'blue:{user_id}'.format(user_id=g.user_id),
'green': 'green:{user_id}'.format(user_id=g.user_id)
}
self.check_keys = {
'red': 'red:{user_id}:lcase'.format(user_id=g.user_id),
'blue': 'blue:{user_id}:lcase'.format(user_id=g.user_id),
'green': 'green:{user_id}:lcase'.format(user_id=g.user_id)
}
self.todays_note_key = '{user_id}:notes:{date}'.format(
user_id=g.user_id, date=datetime.combine(
datetime.now(pytz.timezone('US/Pacific')).date(), time())
.strftime('%y%m%d'))
self.previous_notes_key = '{user_id}:notes:*'.format(
user_id=g.user_id)
def get_notes_count(self):
with self.redis.pipeline() as pipe:
pipe.scard(self.keys['red'])
pipe.scard(self.keys['blue'])
pipe.scard(self.keys['green'])
results = pipe.execute()
return {'red': results[0], 'blue': results[1], 'green': results[2]}
def get_color(self, type):
if type in self.keys:
notes = self.redis.smembers(self.keys[type])
return notes
else:
return []
def add_notes(self, notes):
with self.redis.pipeline() as pipe:
for type, note_list in notes.iteritems():
for note in note_list:
pipe.sadd(self.keys[type], note)
pipe.sadd(self.check_keys[type], note.lower())
pipe.execute()
def validate_notes(self, notes):
checked_notes = []
with self.redis.pipeline() as pipe:
for type, note_list in notes.iteritems():
for note in note_list:
if type in self.keys:
checked_notes.append(note)
pipe.sismember(self.check_keys[type], note.lower())
else:
raise HTTPError(403, 'invalid note type')
results = pipe.execute()
err_notes = []
for i, result in enumerate(results):
if result == 1:
err_notes.append(checked_notes[i])
if err_notes:
err_message = "Some notes already exist."
raise HTTPError(403, {'message': err_message, 'notes': err_notes})
def modify_note(self, type, old_note, new_note):
if type in self.keys:
if not self.redis.sismember(self.keys[type], new_note):
self.delete_note(type, old_note)
self.add_note(type, new_note)
else:
return {'result': 'error', 'err': 'message already in key'}
else:
raise HTTPError(403, 'invalid note type')
def delete_note(self, type, note):
if type in self.keys:
self.redis.srem(self.keys[type], note)
self.redis.srem(self.check_keys[type], note.lower())
return {'result': 'success'}
else:
raise HTTPError(403, 'invalid note type')
def get_todays_note(self, type=None):
if not self.redis.exists(self.todays_note_key):
if not type:
return None
note = self.redis.spop(self.keys[type])
if note:
note_value = {
'note': note,
'type': type
}
self.redis.set(self.todays_note_key, json.dumps(note_value))
self.delete_note(type, note)
return Note(**note_value)
else:
return None
else:
note = self.redis.get(self.todays_note_key)
return Note(**json.loads(note))
def get_previous_notes(self):
previous_keys = self.redis.keys(self.previous_notes_key)
previous_notes = self.redis.mget(previous_keys)
return previous_notes
示例9: DB
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import mget [as 别名]
db = DB()
# open the database
redis_conn = StrictRedis(unix_socket_path = '/tmp/redis.sock')
full_list = list(redis_conn.smembers('system.unique_metrics'))
if len(full_list) == 0:
print "No metrics"
exit()
count = 0
start = time.time()
for metric in full_list:
count += 1
if not db.open("/opt/skyline/src/cabinet/" + metric + ".kct", DB.OWRITER | DB.OCREATE):
print >>sys.stderr, metric + "open error: " + str(db.error())
raw_metric = redis_conn.mget(metric)
for i, metric_name in enumerate(raw_metric):
unpacker = Unpacker(use_list = False)
unpacker.feed(metric_name)
timeseries = list(unpacker)
for value in timeseries:
if db.check(value[0]) < 0:
db.set(value[0], value[1])
#db.set(value[0], value[1])
db.close()
if (count % 100) == 0:
print "%s keys. Rate: %s" % (count, (100/(time.time() - start)))
start = time.time()
示例10: Analyzer
# 需要导入模块: from redis import StrictRedis [as 别名]
# 或者: from redis.StrictRedis import mget [as 别名]
#.........这里部分代码省略.........
* Analyse each timeseries against `ALGORITHMS` to determine if it is\n
anomalous.\n
* If anomalous add it to the :obj:`self.anomalous_metrics` list\n
* Add what algorithms triggered to the :obj:`self.anomaly_breakdown_q` queue\n
Add keys and values to the queue so the parent process can collate for:\n
* :py:obj:`self.anomaly_breakdown_q`
* :py:obj:`self.exceptions_q`
"""
spin_start = time()
logger.info('spin_process started')
# Discover assigned metrics
keys_per_processor = int(ceil(float(len(unique_metrics)) / float(settings.ANALYZER_PROCESSES)))
if i == settings.ANALYZER_PROCESSES:
assigned_max = len(unique_metrics)
else:
assigned_max = min(len(unique_metrics), i * keys_per_processor)
# Fix analyzer worker metric assignment #94
# https://github.com/etsy/skyline/pull/94 @languitar:worker-fix
assigned_min = (i - 1) * keys_per_processor
assigned_keys = range(assigned_min, assigned_max)
# assigned_keys = range(300, 310)
# Compile assigned metrics
assigned_metrics = [unique_metrics[index] for index in assigned_keys]
# Check if this process is unnecessary
if len(assigned_metrics) == 0:
return
# Multi get series
raw_assigned = self.redis_conn.mget(assigned_metrics)
# Make process-specific dicts
exceptions = defaultdict(int)
anomaly_breakdown = defaultdict(int)
# Distill timeseries strings into lists
for i, metric_name in enumerate(assigned_metrics):
self.check_if_parent_is_alive()
# logger.info('analysing %s' % metric_name)
try:
raw_series = raw_assigned[i]
unpacker = Unpacker(use_list=False)
unpacker.feed(raw_series)
timeseries = list(unpacker)
anomalous, ensemble, datapoint = run_selected_algorithm(timeseries, metric_name)
# If it's anomalous, add it to list
if anomalous:
base_name = metric_name.replace(settings.FULL_NAMESPACE, '', 1)
metric = [datapoint, base_name]
self.anomalous_metrics.append(metric)
# Get the anomaly breakdown - who returned True?
triggered_algorithms = []
for index, value in enumerate(ensemble):
if value:
algorithm = settings.ALGORITHMS[index]
anomaly_breakdown[algorithm] += 1
triggered_algorithms.append(algorithm)