Python FileManager.get_locations方法代碼示例

# 需要導入模塊: from FileManager import FileManager [as 別名]
# 或者: from FileManager.FileManager import get_locations [as 別名]
def main():
	captor = AqiCaptor(logger)
	file_manager = FileManager()

	try:
		try:
			locations = file_manager.get_locations(source)
		except IOError:
			raise InitError("File %s is missing"% file_manager.get_locations_path(source))
		except ValueError:
			raise InitError("The %s file does not contain any correct JSON object"% file_manager.get_locations_path(source))
			
		for city, location in locations.iteritems():
			logger.warning("- Capting for %s"% city)
			my_date = datetime.now().strftime('%y-%m-%d-%H')
			loaded_json = captor.get_data(location)
			try:
				file_manager.write_json(loaded_json, source, city, my_date)
			except IOError:
				raise InitError("Folder %s is missing"% file_manager.get_folder_path(source, city))
		
		return 0

	except InitError as e:
		logger.critical("%s: %s"% (type(e).__name__, e))
		return 1
	except requests.exceptions.RequestException as e:
		logger.critical("%s: %s"% (type(e).__name__, e))
		return 2
	except Exception as e:
		logger.critical(e, exc_info=True)
		return 3

# 需要導入模塊: from FileManager import FileManager [as 別名]
# 或者: from FileManager.FileManager import get_locations [as 別名]
def main():
	captor = YahooWeatherCaptor()
	file_manager = FileManager()

	try:
		try:
			woe_ids = file_manager.get_locations(source)
		except IOError:
			raise InitError("File %s is missing"% file_manager.get_locations_path(source))
		except ValueError:
			raise InitError("The %s file does not contain any correct JSON object"% file_manager.get_locations_path(source))
		
		for city, woe_id in woe_ids.iteritems():
			logger.warning("Capting for %s" % city)
			loaded_json = captor.get_data(woe_id)
			
			last_build_date = dateutil.parser.parse(loaded_json['lastBuildDate'])
			date_for_path = last_build_date.strftime('%y-%m-%d-%H-%M')
			
			if os.path.isfile(file_manager.get_path(source, city, date_for_path)):
				logger.warning("Same build date for %s (%s)" % (city, last_build_date))
			else:
				try:
					file_manager.write_json(loaded_json, source, city, date_for_path)
				except IOError:
					raise InitError("Folder %s is missing"% file_manager.get_folder_path(source, city))
				logger.warning("New JSON written for %s %s" % (city, last_build_date))
		return 0

	except InitError as e:
		logger.critical("%s: %s"% (type(e).__name__, e))
		return 1
	except requests.exceptions.RequestException as e:
		logger.critical("%s: %s"% (type(e).__name__, e))
		return 2
	except Exception as e:
		logger.critical(e, exc_info=True)
		return 3

# 需要導入模塊: from FileManager import FileManager [as 別名]
# 或者: from FileManager.FileManager import get_locations [as 別名]
def main():
	return_value = 0
	
	captor = YelpCaptor(logger)
	file_manager = FileManager()
	
	db_conn = None
	try:
		db_conn = sqlite3.connect(file_manager.get_path(source, None, yelp_db, ".db"))
		db_cursor = db_conn.cursor()
		db_manager = YelpDatabaseManager(logger)

		try:
			cities = file_manager.get_locations(source)
		except IOError:
			raise InitError("File %s is missing"% file_manager.get_locations_path(source))
		except ValueError:
			raise InitError("The %s file does not contain any correct JSON object"% file_manager.get_locations_path(source))
				
		my_cities = exit_logger.read_back_file()
		if len(my_cities) == 0:
			my_cities = cities
		remaining_locations = my_cities.copy()
		
		now = datetime.now()
		for city, locations in my_cities.iteritems():
			nb_total_locations = len(cities[city])
			logger.warning("Capting for %s" % city)
			
			for location in locations:
				loaded_json = captor.get_data(location)
				if len(loaded_json) > 0:
					db_manager.insert_venues(db_cursor, loaded_json)
					db_conn.commit()
					
				remaining_locations[city].remove(location)
				nb_locations = nb_total_locations - len(remaining_locations[city])
				logger.warning("%d location(s) done over %d locations for %s"% (nb_locations, nb_total_locations, city))

			venues = db_manager.get_venues(db_cursor)
			nb_venues = len(venues)
			if nb_venues > 0:
				try:
					file_manager.write_json(venues, source, city, now.strftime('%y-%m-%d'))
				except IOError:
					raise InitError("Folder %s is missing"% file_manager.get_folder_path(source, city))
				logger.warning("New JSON written for %s: %d venues"% (city, nb_venues))
				
				db_manager.delete_all_venues(db_cursor)
				db_conn.commit()

			remaining_locations.pop(city)
		
		exit_logger.write_back_file(None)
		
	except InitError as e:
		logger.critical("%s: %s"% (type(e).__name__, e))
		return_value = 1	
	except requests.exceptions.RequestException as e:
		logger.critical("%s: %s"% (type(e).__name__, e))
		exit_logger.write_back_file(remaining_locations)
		return_value = 2
	except YelpCaptor.YelpApiError as e:
		logger.critical("%s: %s"% (type(e).__name__, e))
		exit_logger.write_back_file(remaining_locations)
		return_value = 3
	except sqlite3.OperationalError as e:
		logger.critical("SQLite3 %s: %s"% (type(e).__name__, e))
		return_value = 4
	except Exception as e:
		logger.critical(e, exc_info=True)
		exit_logger.write_back_file(remaining_locations)
		return_value = 5
	finally:
		if db_conn<> None:
			db_conn.close()
		return return_value

# 需要導入模塊: from FileManager import FileManager [as 別名]
# 或者: from FileManager.FileManager import get_locations [as 別名]
def main():	
	last_date = datetime.now() - timedelta(days=2)

	captor = FlickrCaptor(logger)
	file_manager = FileManager()
	
	try:
		try:
			woe_ids = file_manager.get_locations(source)
		except IOError:
			raise InitError("File %s is missing"% file_manager.get_locations_path(source))
		except ValueError:
			raise InitError("The %s file does not contain any correct JSON object"% file_manager.get_locations_path(source))
		
		min_date_file_path = file_manager.get_path(source, None, min_date_file)
		try:
			min_date_json = file_manager.read_json(source, min_date_file)
			min_date = datetime.strptime(min_date_json['min_date'], min_date_format)
		except IOError:
			raise InitError("File %s is missing. You should create this file and set {'min_date':YYYY-MM-DD} in it."% min_date_file_path)
		except (ValueError, KeyError):
			raise InitError("You need to set {\"min_date\":\"YYYY-MM-DD\"} in file %s"% min_date_file_path)
		
		zero_day = timedelta(days=0) 
		one_day = timedelta(days=1)

		if (last_date - min_date) < zero_day:
			raise InitError("The date set as min_date in %s is after 2 days ago."% min_date_file_path)
		
		while (last_date - min_date) >= zero_day:
			logger.warning("---- Capting for %s" % min_date.strftime('%y-%m-%d'))
			for city, woe_id in woe_ids.iteritems():
				max_date = min_date + one_day
				logger.warning("Capting for %s" % city)
				loaded_json = captor.get_data(min_date, max_date, woe_id)
				
				try:
					file_manager.write_json(loaded_json, source, city, min_date.strftime('%y-%m-%d'))
				except IOError:
					raise InitError("Folder %s is missing"% file_manager.get_folder_path(source, city))

				logger.warning("New JSON written for %s " % city)
			
			min_date = max_date
			f = open(min_date_file_path, 'w+')
			json.dump({'min_date':min_date.strftime(min_date_format)}, f)
			f.close()

			logger.warning("... Sleeping for 5 s")
			sleep(5)
		return 0
		
	except InitError as e:
		logger.critical("%s: %s"% (type(e).__name__, e))
		return 1	
	except requests.exceptions.RequestException as e:
		logger.critical("%s: %s"% (type(e).__name__, e))
		return 2
	except FlickrCaptor.FlickrApiError as e:
		logger.critical("%s: %s"% (type(e).__name__, e))
		return 3
	except Exception as e:
		logger.critical(e, exc_info=True)
		return 4