本文整理汇总了Python中Preprocessor.Preprocessor类的典型用法代码示例。如果您正苦于以下问题：Python Preprocessor类的具体用法？Python Preprocessor怎么用？Python Preprocessor使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Preprocessor类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: TestPreprocessor
class TestPreprocessor(unittest.TestCase):
def setUp(self):
cur_dir = os.path.dirname(os.path.realpath(__file__))
self.tpl_dir = os.path.join(cur_dir,'..','UnitTests','test_templates')
self.preprocessor = Preprocessor(self.tpl_dir)
def process_content(self,content,dict,expected):
actual = self.preprocessor.process_content_with_dict(content,dict)
self.assertEqual(actual,expected)
def test_ifs(self):
dict = {'basic_key':'_trivial_', 'yes_key':True, 'no_key':False}
self.process_content("basic%basic_key%basic",dict,"basic_trivial_basic")
self.process_content("%if yes_key%%basic_key%%endif%",dict,"_trivial_")
self.process_content("%if no_key%%basic_key%%endif%",dict,"")
self.process_content("aa%if yes_key%%if no_key%%basic_key%%endif%%endif%bb",dict,"aabb")
self.process_content("start %if non_existant_key%non_existant%endif% finish",dict,"start finish")
self.process_content("start %if non_existant_key%%if yes_key%%basic_key%%endif%%endif% finish",dict,"start finish")
self.process_content("start %if yes_key%yes%else%no%endif%",dict,"start yes")
self.process_content("start %if no_key%yes%else%no%endif%",dict,"start no")
self.process_content("start %if non_existant_key%yes%else%non_exist%endif%",dict,"start non_exist")
self.process_content("%if yes_key%%if no_key%yes%else%no%endif%%endif%",dict,"no")
self.process_content("%if no_key% hello1 %else% %if yes_key% hello2 %else% hello3 %endif% %endif%",dict," hello2 ")
def test_includes(self):
content = self.preprocessor.process_tpl_name_with_dict('test_include',{'a':True,'aa':'a'})
self.assertEqual(content,'<html><body> a </body></html>')示例2: main
def main():
pp = Preprocessor()
print 'processing custom data, computing bows...'
tdpath = 'dataset/test/sms-data'
pp.process_custom_data(tdpath)
fm = FeatureModel()
print 'converting custom data to fvs...'
fm.compute_custom_fv_matrix('custom')
tdpath = 'bin_data/custom_fv.npy'
cpath = 'bin_data/mnb-classifier.npy'
data = np.load('bin_data/custom-data.npy').item()
tester = Tester(tdpath,cpath)
print 'predicting labels for custom data...'
results = tester.predict_labels_for_custom_data(data)
with open('output/results.txt','w') as textfile:
for msg in results:
line = '%s -> %s\n' % (msg,results[msg])
textfile.write(line)
textfile.close()
print 'Results written to results.txt'示例3: runModel
def runModel(self, testSize, debug):
self.trainVectorizer()
d = self.getXy('train.tsv')
if debug:
X_train, X_test, y_train, y_test = train_test_split(d['X'], d['y'], test_size=testSize, random_state=5)
else:
X_train = d['X']
y_train = d['y']
d_test = self.getXy('test.tsv')
X_test = d_test['X']
urlid = d_test['urlid']
self.fit(X_train, y_train)
print "20 Fold CV Score: ", np.mean(cross_val_score(self.model, d['X'], d['y'], cv=10, scoring='roc_auc'))
y_predicted = self.predict(X_test)
if debug:
print 'Topic Model AUC Score: %f' % roc_auc_score(y_test, y_predicted)
else:
Pre = Preprocessor()
Pre.generateSubmission('submission_12.csv', urlid, y_predicted)
P.figure()
P.hist(y_predicted, bins=100)
P.show()示例4: pictures_html_block
def pictures_html_block(self):
pictures_preprocessor = Preprocessor(tpl_dir)
dict = {
"img_size_l": "1836x2448",
"img_size_m": "1224x1632",
"img_size_s": "612x816",
"img_width_t": 150,
"img_height_t": 150,
}
text = ""
for i in xrange(1, 16):
text = text + pictures_preprocessor.process_tpl_name_with_dict("asset", dict)
return text示例5: album_list_html_block
def album_list_html_block(self):
albums_preprocessor = Preprocessor(tpl_dir)
dict = {
"number_of_pictures": 152,
"album_name": "My Photos",
"album_share_href": "/album.html",
"img_width_t": 150,
"img_height_t": 150,
"poster_image_src": "/test_image.jpeg",
}
text = ""
for i in xrange(1, 4):
text = text + albums_preprocessor.process_tpl_name_with_dict("album_list_item", dict)
return text示例6: get_preprocessor_names
def get_preprocessor_names():
result = []
for clazz in Preprocessor.__subclasses__():
result.append(clazz.get_name())
return result示例7: get_authors_and_title
def get_authors_and_title(text):
#print text.encode('utf8')
pattern = u'\x14(.*)\x15'
m = re.search(pattern, text.split('\n')[0])
all = m.group(1)
#print all.encode('utf8')
authors, title = Preprocessor.extract_authors(all)
return authors,title示例8: __init__
def __init__(self,
outputFormat='flat',
useJarfileManifest=True,
useChromeManifest=False):
self.outputFormat = outputFormat
self.useJarfileManifest = useJarfileManifest
self.useChromeManifest = useChromeManifest
self.pp = Preprocessor()示例9: __init__
def __init__(self, outputFormat = 'flat', useJarfileManifest = True,
useChromeManifest = False):
self.outputFormat = outputFormat
self.useJarfileManifest = useJarfileManifest
self.useChromeManifest = useChromeManifest
self.pp = Preprocessor()
self.topsourcedir = None
self.sourcedirs = []
self.localedirs = None示例10: get_new_instance
def get_new_instance(preprocessor_type):
result = None
for clazz in Preprocessor.__subclasses__():
if clazz.get_name() == preprocessor_type:
result = clazz()
break
if result is None:
raise ValueError("Cannot find retriever of type %s" % (preprocessor_type, ))
return result示例11: TestLineEndings
class TestLineEndings(unittest.TestCase):
"""
Unit tests for the Context class
"""
def setUp(self):
self.pp = Preprocessor()
self.pp.out = StringIO()
self.tempnam = os.tempnam('.')
def tearDown(self):
os.remove(self.tempnam)
def createFile(self, lineendings):
f = open(self.tempnam, 'wb')
for line, ending in zip(['a', '#literal b', 'c'], lineendings):
f.write(line+ending)
f.close()
def testMac(self):
self.createFile(['\x0D']*3)
self.pp.do_include(self.tempnam)
self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
def testUnix(self):
self.createFile(['\x0A']*3)
self.pp.do_include(self.tempnam)
self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
def testWindows(self):
self.createFile(['\x0D\x0A']*3)
self.pp.do_include(self.tempnam)
self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')示例12: __init__
def __init__(self, outputFormat = 'flat', useJarfileManifest = True,
useChromeManifest = False):
self.outputFormat = outputFormat
self.useJarfileManifest = useJarfileManifest
self.useChromeManifest = useChromeManifest
self.pp = Preprocessor()
self.topsourcedir = None
self.sourcedirs = []
self.localedirs = None
self.l10nbase = None
self.l10nmerge = None
self.relativesrcdir = None
self.rootManifestAppId = None示例13: trainClassifier
def trainClassifier(self, trainLetter, progress, progLab, maxSets):
# nacitanie a predspracovanie signalu
signalLoader = SignalLoader(self.chanNum,self.files)
prpr = Preprocessor(self.chanNum,[])
signal,stimCode,phaseInSequence = signalLoader.loadSignal()
self.signal = prpr.preprocess(240,1E-1,30E0,self.sf,signal,stimCode,phaseInSequence,0)
self.stimulusCode = prpr.stimulusCode
self.phaseInSequence = prpr.phaseInSequence
self.targetLetters = sum(trainLetter,[])
# najdenie prechodov medzi znakmi
charEnds = self.findCharEnds()
# rozdelenie dat do epoch
em = EpochManager(self.signal,self.stimulusCode,self.phaseInSequence)
isiList = em.createEpochs()
# trening jednotlivych znakov
for i in range(len(charEnds)):
progress["value"] = i
progLab["text"] = ("Trénujem znak: {}/{}").format(i+1, len(charEnds))
print "Averaging character:",i,"\n"
hi = charEnds[i]
if i == 0:
lo = 0
else:
lo = charEnds[i-1]
rowColBinList = em.getAveragedEpochs(hi,lo,isiList,maxSets)
finalDataArray = rowColBinList
classMarks = self.prepairTargetArray(self.getCharIndexes(self.targetLetters[i]))
if self.firsttrain == 1:
self.cl.learn(finalDataArray,classMarks,0)
self.firsttrain = 0
else:
self.cl.learn(finalDataArray,classMarks)示例14: preprocess
def preprocess(input, parser, defines={}):
'''
Preprocess the file-like input with the given defines, and send the
preprocessed output line by line to the given parser.
'''
pp = Preprocessor()
pp.context.update(defines)
pp.do_filter('substitution')
pp.out = PreprocessorOutputWrapper(pp, parser)
pp.do_include(input)示例15: make_preprocessor
def make_preprocessor(config_status):
pp = Preprocessor()
pp.setLineEndings("lf")
pp.setMarker("#")
pp.do_filter("substitution")
# Might need 'substs' too.
defines = {}
for k, v in config_status['defines']:
if v:
defines[k] = v
pp.context.update(defines)
return pp