本文整理汇总了Python中Tools.Tools.write_log方法的典型用法代码示例。如果您正苦于以下问题：Python Tools.write_log方法的具体用法？Python Tools.write_log怎么用？Python Tools.write_log使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类Tools.Tools的用法示例。
在下文中一共展示了Tools.write_log方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: Options_Spider
# 需要导入模块: from Tools import Tools [as 别名]
# 或者: from Tools.Tools import write_log [as 别名]
class Options_Spider(BaseSpider):
db = None
debug = True
name = "options"
attributes = []
allowed_domains = ["autohome.com.cn"]
start_urls = [
"http://www.autohome.com.cn/199/options.html"
]
# parse start
def parse(self, response):
urls = []
handle = open("./files/option_urls.log", 'r')
while True:
url = handle.readline()
if len(url) == 0:
break
urls.append(url[0:-1])
handle.close()
self.tools = Tools()
self.attributes = ConfigObj("./spiders/attributes.ini", encoding='UTF8')
for url in self.start_urls:
yield scrapy.Request(url, callback=self.main_parse)
def main_parse(self, response):
if self.debug:
print "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Parse Page xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
self.db = MysqlDB()
self.db.connection()
self.tools.write_log('Start Parse', response.url)
#self.parse_option(response)
self.parse_config(response)
self.tools.write_log('Finish All')
self.db.commit()
self.db.close()
#parse page
def parse_option(self, response):
self.tools.write_log('Parse Option...')
option_js = re.findall(r"var option = (.*);", response.body)
option_str = option_js[0].decode("GB2312")
option_data = json.loads(option_str, encoding="UTF-8")
option_configs = option_data["result"]["configtypeitems"]
car_option = self.analysis_json("configitems", option_configs)
self.save_option(car_option)
self.tools.write_log('<Finish Option>')
def parse_config(self, response):
self.tools.write_log('Parse Config...')
config_js = re.findall(r"var config = (.*);", response.body)
config_str = config_js[0].decode("GB2312")
config_data = json.loads(config_str, encoding="UTF-8")
config_configs = config_data["result"]["paramtypeitems"]
car_info = self.analysis_json("paramitems", config_configs)
self.save_option(car_info)
self.tools.write_log('<Finish Config>')
#parse option
def analysis_json(self, _key, _configs):
car_option = dict()
first_item = _configs[0][_key][0]["valueitems"]
# init car_option keys
for i in range(0, len(first_item)):
_id = int(first_item[i]["specid"])
car_option[_id] = dict()
# set car_option data
for i in range(0, len(_configs)):
_configis = _configs[i][_key]
for j in range(0, len(_configis)):
_values = _configis[j]["valueitems"]
for k in range(0, len(_values)):
_id = _values[k]["specid"]
if i not in car_option[_id] :
car_option[_id][i] = dict()
car_option[_id][i]['items'] = dict()
if j not in car_option[_id][i]['items'] :
#.........这里部分代码省略.........