Python Timer.show方法代码示例

# 需要导入模块: from Timer import Timer [as 别名]
# 或者: from Timer.Timer import show [as 别名]
def traverseFiles(dataFileNames, threadNr):
    print "starting ", threadNr 
    t = Timer()
    t.click()
    
    
    for fileName in dataFileNames:
        print "\nthread ", threadNr, " cleaning ", fileName
        dataFile = open(config.get("data-dir") + fileName)
        resultFile = open(config.get("clean-dir") + "clean_" + fileName, "w")
        
        for line in dataFile:
            firstChar = line[0]
            
            # line is URL
            if firstChar == "U":
                resultFile.write(line)
                #print line[3].split("http://en.wikipedia.org/wiki/")[1]
                
            # line is a MENTION
            elif firstChar =="M":
                resultFile.write(line)
                
            # line is a TOKEN, exclude them.
            elif firstChar == "T":
                continue
                
            # line is a separator between two documents
            elif firstChar == "\n":
                resultFile.write(line)
            else: 
                print "ERROR: strange line: ", line
    
            
        resultFile.close()  
            
    t.click()
    print "thread ", threadNr, " done in ", t.show()

# 需要导入模块: from Timer import Timer [as 别名]
# 或者: from Timer.Timer import show [as 别名]
			if phrase not in phraseDict:
				phraseDict.update({ phrase : phraseIndex })
				phraseIndex += 1
			#=========================================	
			
		else:
			print "===============", fileName, "======="
			print "	returned = None: bad Entity mention: ", entity


resultFile.close()

T.click()

print "=== SUMMARY ==="
print "dictionaries building done in ", T.show()
print "found ", entCount, " entity mentions in ", len(articles), " articles"
print " number of distinct entities: ", len(entDict)
print " referred to by ", len(phraseDict), " distinct phrases"

#print " ", len(articles), " ", len(entDict), " ", len(phraseDict), " ", entCount




print "docDict size [bytes]: ", sys.getsizeof(docDict)
print "entDict size [bytes]: ", sys.getsizeof(entDict)
print "phraseDict size [bytes]: ", sys.getsizeof(phraseDict)

# 需要导入模块: from Timer import Timer [as 别名]
# 或者: from Timer.Timer import show [as 别名]
for line in file:
	pageName = line.split(" ")[0]
	charCount = int(line.split(" ")[2])
	rawDict.update({pageName: charCount})

print charCount.__class__()

# sort dictionary
T = Timer()
T.click()
#sorts dict by second element of item - RETURNS A SORTED LIST of items
# if list is put into a dict, then ORDERING IS LOST !!!
pageCountList = sorted(rawDict.iteritems(), key = operator.itemgetter(1)) 
T.click()
print "time to sort 37501 items: " ,T.show()


# write the files
for item in pageCountList:
	resultFile.write(item[0] + " " + str(item[1]) + "\n")
	charCountStat.write(str(item[1]) + "\n")
		
resultFile.close()
charCountStat.close()



#############
## Remove junk
#############

# 需要导入模块: from Timer import Timer [as 别名]
# 或者: from Timer.Timer import show [as 别名]
		# this article contains too few entity mentions
		numThinArticles += 1
	else: 	
		# copy good article (corresponding entities and lang_links file) to target dir
		# shutil.copyfile("sourcefile", "destfile")
		copyfile(sourceDirEnts + fileName, resultDirEnts + fileName)
		copyfile(sourceDirLanglinks + fileName, resultDirLanglinks + fileName)
		
	



T.click()

print "numThinArticles: ", numThinArticles
print "done in ", T.show()
    
    
    
# minNumMentions = 3:
# numThinArticles:  3735
# number of good articles: 15725 (containing 161744 entity mentions)
    


# minNumMentions = 5:
# numThinArticles:  7205
# number of good articles: 12255
    
    
    

# 需要导入模块: from Timer import Timer [as 别名]
# 或者: from Timer.Timer import show [as 别名]
data = []
T.click()

for fileName in dataFiles:
    dataFile = open(config.get("clean-dir") + fileName)
    print "processing ", fileName
    for line in dataFile:
        data.append(line)
        
T.click()


size = sys.getsizeof(data) / 1000000.0
print "data array size: ", str(size), "MB"
print "filling data array done in ", T.show()





"""
>
>
URL     ftp://212.18.29.48/ftp/pub/allnet/nas/all60600/ALL60600_UM_EN.pdf
MENTION NetBIOS 186757  http://en.wikipedia.org/wiki/NetBIOS
>
>
URL     ftp://38.107.129.5/Training/Traininga.pdf
MENTION Microsoft       80679   http://en.wikipedia.org/wiki/Microsoft
>

# 需要导入模块: from Timer import Timer [as 别名]
# 或者: from Timer.Timer import show [as 别名]
'''
Created on 18.12.2013

@author: pilatus
'''
import os
from Timer import Timer
import xml.etree.ElementTree as ET

defaultNamespace = "{http://www.mediawiki.org/xml/export-0.8/}"

targetDir = "/dev/shm/wikinews/wikiAPITest/"

sourceDir = "/dev/shm/wikinews/articles_cleaned/"

articles = os.listdir(sourceDir)

i = 0
T = Timer()
T.click()
for art in articles:
    root = ET.parse(sourceDir + art).getroot()
    title = root.find(".//" + defaultNamespace + "title").text.replace(" ", "_")
    """print title
    i += 1
    if i>5:
        break"""
T.click()
print T.show()

# 需要导入模块: from Timer import Timer [as 别名]
# 或者: from Timer.Timer import show [as 别名]
	sys.exit()




# 1 
# parse dump and get the root of xml tree
#
L.debug("==============================================")
L.debug("==============================================")
L.debug("parsing started")
T.click()
dump = ET.parse(dumpFileName)
wikinews = dump.getroot()
T.click()
parsingTime = T.show()
L.debug("done parsing in: " + parsingTime)


# 2 
# extract the namespace string from the root element tag name
# should be "{http://www.mediawiki.org/xml/export-0.8/}"
#
namespace = wikinews.tag.split("}")[0].strip() + "}"


# 3
# collect all target elements from the xml tree
#
L.debug("xpathing all elements of interest")
T.click()

# 需要导入模块: from Timer import Timer [as 别名]
# 或者: from Timer.Timer import show [as 别名]
	if len(list(textEls))>0:
		print ".",
		i += 1
		textLength = len(textEls[0].text)
		ctl += textLength
		resultFile.write(fName + " 1 " + str(textLength) + "\n")
	else:
		print "-",
		j += 1
		resultFile.write(fName + " 0 0 \n")

T.click()

resultFile.write("pages with text: " + str(i) + "\n")
resultFile.write("mean text length: " + str(ctl/i)+ "\n")
resultFile.write("pages without text: " + str(j)+ "\n")
resultFile.write("processed in : " + T.show() + "\n")
resultFile.close()



"""for f in files:
	anf = f.split("-")[0]
	try: 
		i = int(anf)
		print f
	except:
		print "bad file:  " , f

"""