本文整理匯總了Python中Tools.Tools.write_log方法的典型用法代碼示例。如果您正苦於以下問題：Python Tools.write_log方法的具體用法？Python Tools.write_log怎麽用？Python Tools.write_log使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類Tools.Tools的用法示例。
在下文中一共展示了Tools.write_log方法的1個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: Options_Spider
# 需要導入模塊: from Tools import Tools [as 別名]
# 或者: from Tools.Tools import write_log [as 別名]
class Options_Spider(BaseSpider):
db = None
debug = True
name = "options"
attributes = []
allowed_domains = ["autohome.com.cn"]
start_urls = [
"http://www.autohome.com.cn/199/options.html"
]
# parse start
def parse(self, response):
urls = []
handle = open("./files/option_urls.log", 'r')
while True:
url = handle.readline()
if len(url) == 0:
break
urls.append(url[0:-1])
handle.close()
self.tools = Tools()
self.attributes = ConfigObj("./spiders/attributes.ini", encoding='UTF8')
for url in self.start_urls:
yield scrapy.Request(url, callback=self.main_parse)
def main_parse(self, response):
if self.debug:
print "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Parse Page xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
self.db = MysqlDB()
self.db.connection()
self.tools.write_log('Start Parse', response.url)
#self.parse_option(response)
self.parse_config(response)
self.tools.write_log('Finish All')
self.db.commit()
self.db.close()
#parse page
def parse_option(self, response):
self.tools.write_log('Parse Option...')
option_js = re.findall(r"var option = (.*);", response.body)
option_str = option_js[0].decode("GB2312")
option_data = json.loads(option_str, encoding="UTF-8")
option_configs = option_data["result"]["configtypeitems"]
car_option = self.analysis_json("configitems", option_configs)
self.save_option(car_option)
self.tools.write_log('<Finish Option>')
def parse_config(self, response):
self.tools.write_log('Parse Config...')
config_js = re.findall(r"var config = (.*);", response.body)
config_str = config_js[0].decode("GB2312")
config_data = json.loads(config_str, encoding="UTF-8")
config_configs = config_data["result"]["paramtypeitems"]
car_info = self.analysis_json("paramitems", config_configs)
self.save_option(car_info)
self.tools.write_log('<Finish Config>')
#parse option
def analysis_json(self, _key, _configs):
car_option = dict()
first_item = _configs[0][_key][0]["valueitems"]
# init car_option keys
for i in range(0, len(first_item)):
_id = int(first_item[i]["specid"])
car_option[_id] = dict()
# set car_option data
for i in range(0, len(_configs)):
_configis = _configs[i][_key]
for j in range(0, len(_configis)):
_values = _configis[j]["valueitems"]
for k in range(0, len(_values)):
_id = _values[k]["specid"]
if i not in car_option[_id] :
car_option[_id][i] = dict()
car_option[_id][i]['items'] = dict()
if j not in car_option[_id][i]['items'] :
#.........這裏部分代碼省略.........