本文整理汇总了Python中pyelasticsearch.ElasticSearch.update_op方法的典型用法代码示例。如果您正苦于以下问题：Python ElasticSearch.update_op方法的具体用法？Python ElasticSearch.update_op怎么用？Python ElasticSearch.update_op使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类pyelasticsearch.ElasticSearch的用法示例。
在下文中一共展示了ElasticSearch.update_op方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: update_document
# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import update_op [as 别名]
def update_document(entries):
es_server = 'http://localhost:9200/'
if os.environ.get('ELASTICSEARCH_SERVER'):
es_server = os.environ['ELASTICSEARCH_SERVER']
es = ElasticSearch(es_server)
# es.update(index=os.environ['ELASTICSEARCH_INDEX'] if os.environ.get('ELASTICSEARCH_INDEX') else 'memex',
# doc_type=os.environ['ELASTICSEARCH_DOC_TYPE'] if os.environ.get('ELASTICSEARCH_DOC_TYPE') else 'page',
# id=url,
# script=doc,
# upsert=True
# )
es.bulk([es.update_op(doc, id=doc['url'], upsert=True) for doc in entries],
index=os.environ['ELASTICSEARCH_INDEX'] if os.environ.get('ELASTICSEARCH_INDEX') else 'memex',
doc_type=os.environ['ELASTICSEARCH_DOC_TYPE'] if os.environ.get('ELASTICSEARCH_DOC_TYPE') else 'page')示例2: MySync
# 需要导入模块: from pyelasticsearch import ElasticSearch [as 别名]
# 或者: from pyelasticsearch.ElasticSearch import update_op [as 别名]
class MySync(object):
ts = 0 #last chunk time
log_file = None
log_pos = None
def __init__(self):
self.config = yaml.load(open('./etc/config.yaml'))
self.mark_path = self.config['binlog']['mark']
self.bulk_size = self.config['es']['bulk_size']
self.excludes_fields = self.config['slave']['excludes_fields']
self.es = ElasticSearch('http://{host}:{port}/'.format(
host=self.config['es']['host'],
port=self.config['es']['port']
))
#set logger
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
filename=self.config['log']['run']
)
logging.getLogger('elasticsearch').setLevel(logging.INFO)
logging.getLogger('elasticsearch.trace').setLevel(logging.INFO)
logging.getLogger('elasticsearch.trace').addHandler(logging.StreamHandler())
#resume stream
if os.path.isfile(self.mark_path):
with open(self.mark_path, 'r') as y:
mark = yaml.load(y)
self.log_file = mark.get('log_file')
self.log_pos = mark.get('log_pos')
logging.info('resume stream >> file:%s, pos:%s' % (self.log_file, self.log_pos))
"""
mark binlog position
"""
def mark_binlog(self):
if self.log_file and self.log_pos:
with open(self.mark_path, 'w') as y:
logging.info('mark binlog >> file:%s, pos:%s' % (self.log_file, self.log_pos))
yaml.safe_dump({'log_file':self.log_file, 'log_pos':self.log_pos}, y, default_flow_style=False)
"""
format fields
"""
def _format(self, dat):
for k,v in dat.items():
if isinstance(v, datetime):
dat[k] = v.strftime('%Y-%m-%d %H:%M:%S')
elif isinstance(v, date):
dat[k] = v.strftime('%Y-%m-%d')
if k in self.excludes_fields:
del dat[k]
return dat
"""
mysql binlog event handle
"""
def proc_binlog(self):
stream = BinLogStreamReader(
connection_settings = self.config['mysql'],
server_id = self.config['slave']['server_id'],
log_file = self.log_file,
log_pos = self.log_pos,
only_schemas = self.config['slave']['schemas'],
blocking = True,
resume_stream = bool(self.log_file and self.log_pos),
only_events=[WriteRowsEvent, UpdateRowsEvent, DeleteRowsEvent]
)
for binlogevent in stream:
#binlogevent.dump()
self.log_file = stream.log_file
self.log_pos = stream.log_pos
for row in binlogevent.rows:
pk = binlogevent.primary_key
table = binlogevent.table
schema = binlogevent.schema
if isinstance(binlogevent, WriteRowsEvent):
yield self.es.index_op(self._format(row['values']), doc_type=table, index=schema, id=row['values'][pk])
elif isinstance(binlogevent, UpdateRowsEvent):
yield self.es.update_op(self._format(row['after_values']), doc_type=table, index=schema, id=row['after_values'][pk])
elif isinstance(binlogevent, DeleteRowsEvent):
yield self.es.delete_op(doc_type=table, index=schema, id=row['values'][pk])
else:
continue
stream.close()
"""
notify exception
"""
def send_email(self, msg):
import smtplib
from email.mime.text import MIMEText
msg = MIMEText(msg, 'plain', 'utf-8')
msg['From'] = self.config['email']['from']['user']
#.........这里部分代码省略.........