本文整理汇总了Python中utils.to_utf8函数的典型用法代码示例。如果您正苦于以下问题:Python to_utf8函数的具体用法?Python to_utf8怎么用?Python to_utf8使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了to_utf8函数的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: ensure_person
def ensure_person(person):
profiler_start("Ensuring person %s for repository %d",
(person.name, self.repo_id))
printdbg("DBContentHandler: ensure_person %s <%s>",
(person.name, person.email))
cursor = self.cursor
name = to_utf8(person.name)
email = person.email
if email is not None:
email = to_utf8(email).decode("utf-8")
cursor.execute(statement(
"SELECT id from people where name = ?", self.db.place_holder),
(to_utf8(name).decode("utf-8"),))
rs = cursor.fetchone()
if not rs:
p = DBPerson(None, person)
cursor.execute(statement(DBPerson.__insert__,
self.db.place_holder),
(p.id, to_utf8(p.name).decode("utf-8"),
email))
person_id = p.id
else:
person_id = rs[0]
profiler_stop("Ensuring person %s for repository %d",
(person.name, self.repo_id), True)
return person_id
示例2: __init__
def __init__(self, id, uri, name, type):
if id is None:
self.id = DBRepository.id_counter
DBRepository.id_counter += 1
else:
self.id = id
self.uri = to_utf8(uri)
self.name = to_utf8(name)
self.type = to_utf8(type)
示例3: parse
def parse(cls, selector):
with contextlib.closing(StringIO()) as result:
if type(selector) == dict:
for k, v in selector.items():
result.write('%s:(%s)' % (to_utf8(k), cls.parse(v)))
elif type(selector) in (list, tuple):
result.write(','.join(map(cls.parse, selector)))
else:
result.write(to_utf8(selector))
return result.getvalue()
示例4: write_headers
def write_headers(self, num_docs, num_terms, num_nnz):
self.fout.write(MmWriter.HEADER_LINE)
if num_nnz < 0:
# we don't know the matrix shape/density yet, so only log a general line
logger.info("saving sparse matrix to %s" % self.fname)
self.fout.write(utils.to_utf8(' ' * 50 + '\n')) # 48 digits must be enough for everybody
else:
logger.info("saving sparse %sx%s matrix with %i non-zero entries to %s" %
(num_docs, num_terms, num_nnz, self.fname))
self.fout.write(utils.to_utf8('%s %s %s\n' % (num_docs, num_terms, num_nnz)))
self.last_docno = -1
self.headers_written = True
示例5: save_cat2vec_format
def save_cat2vec_format(self, fname):
"""
Store cat vectors
"""
logger.info("storing %sx%s projection weights into %s" % (self.cat_len, self.layer1_size, fname))
assert (self.cat_len, self.layer1_size) == self.cats.shape
with utils.smart_open(fname, 'wb') as fout:
fout.write(utils.to_utf8("#cats_len: %d\n#size:%d\n" % self.cats.shape))
fout.write(utils.to_utf8("#sg:%d\n#hs:%d\n#negative:%d\n#cbow_mean:%d\n" % (self.sg,self.hs,self.negative,self.cbow_mean)))
for cat_id in self.cat_no_hash.keys():
row = self.cats[self.cat_no_hash[cat_id]]
fout.write(utils.to_utf8("%s\t%s\n" % (cat_id, ' '.join("%f" % val for val in row))))
示例6: modify
def modify(self, dn, mod_type=None, attrs=None):
""" Modify a record """
if self.read_only:
msg = 'Running in read-only mode, modification is disabled'
logger.info(msg)
return msg
utf8_dn = to_utf8(dn)
res = self.search(base=utf8_dn, scope=self.BASE)
attrs = attrs and attrs or {}
if res['exception']:
return res['exception']
if res['size'] == 0:
return 'LDAPDelegate.modify: Cannot find dn "%s"' % dn
cur_rec = res['results'][0]
mod_list = []
msg = ''
for key, values in attrs.items():
values = map(to_utf8, values)
if mod_type is None:
if cur_rec.get(key, ['']) != values and values != ['']:
mod_list.append((self.REPLACE, key, values))
elif cur_rec.has_key(key) and values == ['']:
mod_list.append((self.DELETE, key, None))
else:
mod_list.append((mod_type, key, values))
try:
connection = self.connect()
new_rdn = attrs.get(self.rdn_attr, [''])[0]
if new_rdn and new_rdn != cur_rec.get(self.rdn_attr)[0]:
new_utf8_rdn = to_utf8('%s=%s' % (self.rdn_attr, new_rdn))
connection.modrdn_s(utf8_dn, new_utf8_rdn)
old_dn_exploded = self.explode_dn(utf8_dn)
old_dn_exploded[0] = new_utf8_rdn
utf8_dn = ','.join(old_dn_exploded)
connection.modify_s(utf8_dn, mod_list)
except ldap.INVALID_CREDENTIALS, e:
e_name = e.__class__.__name__
msg = '%s No permission to modify "%s"' % (e_name, dn)
示例7: __insert_many
def __insert_many(self):
if not self.actions and not self.commits:
return
cursor = self.cursor
if self.actions:
actions = [(a.id, a.type, a.file_id, a.commit_id, a.branch_id) \
for a in self.actions]
profiler_start("Inserting actions for repository %d",
(self.repo_id,))
cursor.executemany(statement(DBAction.__insert__,
self.db.place_holder), actions)
self.actions = []
profiler_stop("Inserting actions for repository %d",
(self.repo_id,))
if self.commits:
commits = [(c.id, c.rev, c.committer, c.author, c.date, \
to_utf8(c.message).decode("utf-8"), c.composed_rev, \
c.repository_id) for c in self.commits]
profiler_start("Inserting commits for repository %d",
(self.repo_id,))
cursor.executemany(statement(DBLog.__insert__,
self.db.place_holder), commits)
self.commits = []
profiler_stop("Inserting commits for repository %d",
(self.repo_id,))
profiler_start("Committing inserts for repository %d",
(self.repo_id,))
self.cnn.commit()
profiler_stop("Committing inserts for repository %d",
(self.repo_id,))
示例8: create_dealer_index_xychart
def create_dealer_index_xychart(title,labels,score,mark_value=None,format='{value|1}',fontAngle=0,Scale=100):
new_labels = [truncate_hanzi(label,25) for label in labels]
colors = BASE_COLOR
chart_height = 60+20*len(new_labels)
c = XYChart(400, chart_height)
title = c.addTitle(utils.to_utf8(title), "simsun.ttc", 12)
title.setMargin2(20, 0, 10, 30)
c.setBackground(c.linearGradientColor(0, 0, 0, c.getHeight(), '0xFEFEFE', '0xFFFFFF'),'0X666666')
title_height = 0
c.addLine(20, title_height, c.getWidth() - 21, title_height, '0xffffff')
plot_height = chart_height-30
c.setPlotArea(70, 50, 270, plot_height, -1, -1, Transparent, '0xffffff')
layer = c.addBarLayer3(score, colors)
# layer.setBorderColor(Transparent, softLighting(Right))
layer.setAggregateLabelFormat(format)
layer.setAggregateLabelStyle("simsun.ttc", 8)
xAxis = c.xAxis()
xAxis.setLabels(new_labels)
c.yAxis().setColors(Transparent)
c.yAxis2().setColors(Transparent)
c.xAxis().setTickColor(Transparent)
c.xAxis().setLabelStyle("simsun.ttc", 9, 0x0, fontAngle)
c.yAxis().setLabelStyle("simsun.ttc", 9)
c.yAxis2().setLabelStyle("simsun.ttc", 9)
c.yAxis().setLinearScale(0,Scale)
c.packPlotArea(20, title_height + 15, c.getWidth() - 30, c.getHeight() - 15)
return c.makeChart2(PNG)
示例9: insert
def insert(self, base, rdn, attrs=None):
""" Insert a new record """
if self.read_only:
msg = 'Running in read-only mode, insertion is disabled'
logger.info(msg)
return msg
msg = ''
dn = to_utf8('%s,%s' % (rdn, base))
attribute_list = []
attrs = attrs and attrs or {}
for attr_key, attr_val in attrs.items():
if isinstance(attr_val, str) or isinstance(attr_val, unicode):
attr_val = [x.strip() for x in attr_val.split(';')]
if attr_val != ['']:
attr_val = map(to_utf8, attr_val)
attribute_list.append((attr_key, attr_val))
try:
connection = self.connect()
connection.add_s(dn, attribute_list)
except ldap.INVALID_CREDENTIALS, e:
e_name = e.__class__.__name__
msg = '%s No permission to insert "%s"' % (e_name, dn)
示例10: search
def search( self
, base
, scope
, filter='(objectClass=*)'
, attrs=[]
, bind_dn=''
, bind_pwd=''
):
""" The main search engine """
result = { 'exception' : ''
, 'size' : 0
, 'results' : []
}
filter = to_utf8(filter)
try:
connection = self.connect(bind_dn=bind_dn, bind_pwd=bind_pwd)
if connection is None:
result['exception'] = 'Cannot connect to LDAP server'
return result
try:
res = connection.search_s(base, scope, filter, attrs)
except ldap.PARTIAL_RESULTS:
res_type, res = connection.result(all=0)
except ldap.REFERRAL, e:
connection = self.handle_referral(e)
try:
res = connection.search_s(base, scope, filter, attrs)
except ldap.PARTIAL_RESULTS:
res_type, res = connection.result(all=0)
for rec_dn, rec_dict in res:
# When used against Active Directory, "rec_dict" may not be
# be a dictionary in some cases (instead, it can be a list)
# An example of a useless "res" entry that can be ignored
# from AD is
# (None, ['ldap://ForestDnsZones.PORTAL.LOCAL/DC=ForestDnsZones,DC=PORTAL,DC=LOCAL'])
# This appears to be some sort of internal referral, but
# we can't handle it, so we need to skip over it.
try:
items = rec_dict.items()
except AttributeError:
# 'items' not found on rec_dict
continue
for key, value in items:
if not isinstance(value, str):
try:
for i in range(len(value)):
value[i] = from_utf8(value[i])
except:
pass
rec_dict['dn'] = from_utf8(rec_dn)
result['results'].append(rec_dict)
result['size'] += 1
示例11: write_vector
def write_vector(self, docno, vector):
"""
Write a single sparse vector to the file.
Sparse vector is any iterable yielding (field id, field value) pairs.
"""
assert self.headers_written, "must write Matrix Market file headers before writing data!"
assert self.last_docno < docno, "documents %i and %i not in sequential order!" % (self.last_docno, docno)
vector = sorted((i, w) for i, w in vector if abs(w) > 1e-12) # ignore near-zero entries
for termid, weight in vector: # write term ids in sorted order
self.fout.write(utils.to_utf8("%i %i %s\n" % (docno + 1, termid + 1, weight))) # +1 because MM format starts counting from 1
self.last_docno = docno
return (vector[-1][0], len(vector)) if vector else (-1, 0)
示例12: save_word2vec_format
def save_word2vec_format(self, fname, fvocab=None, binary=False):
"""
Store the input-hidden weight matrix in the same format used by the original
C word2vec-tool, for compatibility.
"""
if fvocab is not None:
logger.info("Storing vocabulary in %s" % (fvocab))
with utils.smart_open(fvocab, 'wb') as vout:
for word, vocab in sorted(iteritems(self.vocab), key=lambda item: -item[1].count):
vout.write(utils.to_utf8("%s %s\n" % (word, vocab.count)))
logger.info("storing %sx%s projection weights into %s" % (len(self.vocab), self.layer1_size, fname))
assert (len(self.vocab), self.layer1_size) == self.syn0.shape
with utils.smart_open(fname, 'wb') as fout:
fout.write(utils.to_utf8("%s %s\n" % self.syn0.shape))
# store in sorted order: most frequent words at the top
for word, vocab in sorted(iteritems(self.vocab), key=lambda item: -item[1].count):
row = self.syn0[vocab.index]
if binary:
fout.write(utils.to_utf8(word) + b" " + row.tostring())
else:
fout.write(utils.to_utf8("%s %s\n" % (word, ' '.join("%f" % val for val in row))))
示例13: __get_person
def __get_person(self, person):
"""Get the person_id given a person struct
First, it tries to get it from cache and then from the database.
When a new person_id is gotten from the database, the cache must be
updated
"""
def ensure_person(person):
profiler_start("Ensuring person %s for repository %d",
(person.name, self.repo_id))
printdbg("DBContentHandler: ensure_person %s <%s>",
(person.name, person.email))
cursor = self.cursor
name = to_utf8(person.name)
email = person.email
if email is not None:
email = to_utf8(email).decode("utf-8")
cursor.execute(statement(
"SELECT id from people where name = ?", self.db.place_holder),
(to_utf8(name).decode("utf-8"),))
rs = cursor.fetchone()
if not rs:
p = DBPerson(None, person)
cursor.execute(statement(DBPerson.__insert__,
self.db.place_holder),
(p.id, to_utf8(p.name).decode("utf-8"),
email))
person_id = p.id
else:
person_id = rs[0]
profiler_stop("Ensuring person %s for repository %d",
(person.name, self.repo_id), True)
return person_id
if person is None:
return None
name = to_utf8(person.name)
if name in self.people_cache:
person_id = self.people_cache[name]
else:
person_id = ensure_person(person)
self.people_cache[name] = person_id
return person_id
示例14: save_word2vec_format
def save_word2vec_format(self, fname, binary=False):
"""
Store the input-hidden weight matrix in the same format used by the original
C word2vec-tool, for compatibility.
"""
logger.info("storing %sx%s projection weights into %s" % (len(self.vocab), self.layer1_size, fname))
assert (len(self.vocab), self.layer1_size) == self.syn0.shape
with open(fname, 'wb') as fout:
fout.write("%s %s\n" % self.syn0.shape)
# store in sorted order: most frequent words at the top
for word, vocab in sorted(self.vocab.iteritems(), key=lambda item: -item[1].count):
word = utils.to_utf8(word) # always store in utf8
row = self.syn0[vocab.index]
if binary:
fout.write("%s %s\n" % (word, row.tostring()))
else:
fout.write("%s %s\n" % (word, ' '.join("%f" % val for val in row)))
示例15: create_simple_xychart
def create_simple_xychart(title,labels,data,mark_value=None,format='{value|1}',fontAngle=0,x=560,y=220,swapxy=False,Scale=100):
colors = BASE_COLOR
c = XYChart(x, y)
c.setBackground(c.linearGradientColor(0, 0, 0, c.getHeight(), '0xFEFEFE', '0xFFFFFF'),'0X666666')
title_height = 0
c.addLine(20, title_height, c.getWidth() - 21, title_height, '0xffffff')
plot_width = 30+50*len(labels)
c.setPlotArea(70, 50, plot_width, 170, -1, -1, Transparent, '0xffffff')
if swapxy:
c.swapXY()
title = c.addTitle(utils.to_utf8(title), "simsun.ttc", 12)
title.setMargin2(20, 0, 10, 30)
layer = c.addBarLayer3(data, colors)
layer.setBorderColor(Transparent, softLighting(Right))
layer.setAggregateLabelFormat(format)
font_size = 8 if fontAngle == 0 else 7
layer.setAggregateLabelStyle("simsun.ttc", font_size)
layer.setBarWidth(x,15)
xAxis = c.xAxis()
xAxis.setLabels(labels)
c.yAxis().setLinearScale(0,Scale)
c.yAxis().setColors(Transparent)
c.yAxis2().setColors(Transparent)
c.xAxis().setTickColor(Transparent)
c.xAxis().setLabelStyle("simsun.ttc", 9, 0x0, fontAngle)
c.yAxis().setLabelStyle("simsun.ttc", 9)
c.yAxis2().setLabelStyle("simsun.ttc", 9)
# if mark_value:
# markData = [mark_value for i in range(len(data))]
# markLayer = c.addBoxWhiskerLayer(None, None, None, None, markData, -1, '0xff0000')
c.packPlotArea(20, title_height + 40, c.getWidth() - 30, c.getHeight() - 15)
return c.makeChart2(PNG)