本文整理匯總了Python中retriever.lib.models.Table.cleanup方法的典型用法代碼示例。如果您正苦於以下問題：Python Table.cleanup方法的具體用法？Python Table.cleanup怎麽用？Python Table.cleanup使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類retriever.lib.models.Table的用法示例。
在下文中一共展示了Table.cleanup方法的2個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: download
# 需要導入模塊: from retriever.lib.models import Table [as 別名]
# 或者: from retriever.lib.models.Table import cleanup [as 別名]
def download(self, engine=None, debug=False):
try:
Script.download(self, engine, debug)
engine = self.engine
# Species table
table = Table("species", cleanup=Cleanup(), contains_pk=True,
header_rows=6)
table.columns=[("species_id", ("pk-int",) ),
("AOU", ("int",) ),
("english_common_name", ("char",50) ),
("french_common_name", ("char",50) ),
("spanish_common_name", ("char",50) ),
("sporder", ("char",30) ),
("family", ("char",30) ),
("genus", ("char",30) ),
("species", ("char",50) ),
]
table.fixed_width = [7,6,51,51,51,51,51,51,50]
engine.table = table
engine.create_table()
engine.insert_data_from_url(self.urls["species"])
# Routes table
if not os.path.isfile(engine.format_filename("routes_new.csv")):
engine.download_files_from_archive(self.urls["routes"],
["routes.csv"])
read = open(engine.format_filename("routes.csv"), "rb")
write = open(engine.format_filename("routes_new.csv"), "wb")
print "Cleaning routes data..."
write.write(read.readline())
for line in read:
values = line.split(',')
v = Decimal(values[5])
if v > 0:
values[5] = str(v * Decimal("-1"))
write.write(','.join(str(value) for value in values))
write.close()
read.close()
engine.auto_create_table(Table("routes", cleanup=Cleanup()),
filename="routes_new.csv")
engine.insert_data_from_file(engine.format_filename("routes_new.csv"))
# Weather table
if not os.path.isfile(engine.format_filename("weather_new.csv")):
engine.download_files_from_archive(self.urls["weather"],
["weather.csv"])
read = open(engine.format_filename("weather.csv"), "rb")
write = open(engine.format_filename("weather_new.csv"), "wb")
print "Cleaning weather data..."
for line in read:
values = line.split(',')
newvalues = []
for value in values:
if ':' in value:
newvalues.append(value.replace(':', ''))
elif value == "N":
newvalues.append(None)
else:
newvalues.append(value)
write.write(','.join(str(value) for value in newvalues))
write.close()
read.close()
engine.auto_create_table(Table("weather", pk="RouteDataId", cleanup=Cleanup()),
filename="weather_new.csv")
engine.insert_data_from_file(engine.format_filename("weather_new.csv"))
# Region_codes table
table = Table("region_codes", pk=False, header_rows=11,
fixed_width=[11, 11, 30])
def regioncodes_cleanup(value, engine):
replace = {chr(225):"a", chr(233):"e", chr(237):"i", chr(243):"o"}
newvalue = str(value)
for key in replace.keys():
if key in newvalue:
newvalue = newvalue.replace(key, replace[key])
return newvalue
table.cleanup = Cleanup(regioncodes_cleanup)
table.columns=[("countrynum" , ("int",) ),
("regioncode" , ("int",) ),
("regionname" , ("char",30) )]
engine.table = table
engine.create_table()
engine.insert_data_from_url(self.urls["region_codes"])
# Counts table
table = Table("counts", pk=False, delimiter=',')
table.columns=[("RouteDataID" , ("int",) ),
#.........這裏部分代碼省略.........
示例2: download
# 需要導入模塊: from retriever.lib.models import Table [as 別名]
# 或者: from retriever.lib.models.Table import cleanup [as 別名]
def download(self, engine=None, debug=False):
try:
Script.download(self, engine, debug)
engine = self.engine
# Routes table
if not os.path.isfile(engine.format_filename("routes_new.csv")):
engine.download_files_from_archive(self.urls["routes"],
["routes.csv"])
read = open(engine.format_filename("routes.csv"), "rb")
write = open(engine.format_filename("routes_new.csv"), "wb")
print "Cleaning routes data..."
write.write(read.readline())
for line in read:
values = line.split(',')
v = Decimal(values[5])
if v > 0:
values[5] = str(v * Decimal("-1"))
write.write(','.join(str(value) for value in values))
write.close()
read.close()
engine.auto_create_table(Table("routes", cleanup=Cleanup()),
filename="routes_new.csv")
engine.insert_data_from_file(engine.format_filename("routes_new.csv"))
# Weather table
if not os.path.isfile(engine.format_filename("weather_new.csv")):
engine.download_files_from_archive(self.urls["weather"],
["weather.csv"])
read = open(engine.format_filename("weather.csv"), "rb")
write = open(engine.format_filename("weather_new.csv"), "wb")
print "Cleaning weather data..."
for line in read:
values = line.split(',')
newvalues = []
for value in values:
if ':' in value:
newvalues.append(value.replace(':', ''))
elif value == "N":
newvalues.append(None)
else:
newvalues.append(value)
write.write(','.join(str(value) for value in newvalues))
write.close()
read.close()
engine.auto_create_table(Table("weather", pk="RouteDataId", cleanup=Cleanup()),
filename="weather_new.csv")
engine.insert_data_from_file(engine.format_filename("weather_new.csv"))
# Species table
table = Table("species", pk=False, delimiter=',')
table.columns=[("species_id" , ("pk-auto",) ),
("AOU" , ("int",) ),
("genus" , ("char",30) ),
("species" , ("char",50) ),
("subspecies" , ("char",30) ),
("id_to_species" , ("bool",) )]
engine.table = table
engine.create_table()
engine.download_file(self.urls["species"], "SpeciesList.txt")
species_list = open(engine.format_filename("SpeciesList.txt"), "rb")
for n in range(8):
species_list.readline()
rows = []
for line in species_list:
if line and len(line) > 273:
latin_name = line[273:].split()
if len(latin_name) < 2:
# If there's no species given, add "None" value
latin_name.append("None")
subspecies = ' '.join(latin_name[2:]) if len(latin_name) > 2 else "None"
id_to_species = "1" if latin_name[1] != "None" else "0"
if latin_name[1] == "sp.":
latin_name[1] = "None"
id_to_species = "0"
if ("x" in latin_name or "/" in latin_name
or "/" in subspecies or "or" in latin_name):
# Hybrid species or only identified to a group of species
latin_name[1] = ' '.join(latin_name[1:])
subspecies = "None"
id_to_species = "0"
rows.append(','.join([
line.split()[1],
latin_name[0],
latin_name[1],
subspecies,
id_to_species
]))
#.........這裏部分代碼省略.........