Java FileIO.getResource方法代码示例

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public static void getAllUsers() throws Exception {
	String filePath = FileIO.getResource("dvd.ciao.txt");
	List<String> urls = FileIO.readAsList(filePath);

	String dir = Systems.getDesktop() + "dvd.ciao.co.uk\\";
	String userFile = dir + "users.txt";
	Map<String, String> userMap = new HashMap<>();
	for (String url : urls) {
		// each category
		String[] data = url.split(": ");
		String category = data[0];
		String dirPath = FileIO.makeDirPath(dir, category);

		// users
		String userPath = dirPath + "users.txt";
		List<String> users = FileIO.readAsList(userPath);
		for (String line : users) {
			String[] d = line.split(",");
			userMap.put(d[0], d[1]);
		}
	}

	FileIO.writeMap(userFile, userMap);
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
protected void load_proxy() throws Exception
{
	proxy = new HashMap<>();

	String path = FileIO.getResource("proxy.txt");

	List<String> lines = FileIO.readAsList(path);
	for (String line : lines)
	{
		String[] data = line.split(":");
		String host = data[0];
		int port = Integer.parseInt(data[1]);

		proxy.put(host, port);
	}

	hosts = new ArrayList<>(proxy.keySet());
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public static void getAllRatings() throws Exception {
	String filePath = FileIO.getResource("dvd.ciao.txt");
	List<String> urls = FileIO.readAsList(filePath);

	String dir = Systems.getDesktop() + "dvd.ciao.co.uk\\";
	String ratingFile = dir + "ratings.txt";
	String reviewFile = dir + "review-ratings.txt";
	FileIO.deleteFile(ratingFile);
	FileIO.deleteFile(reviewFile);

	for (String url : urls) {
		// each category
		String[] data = url.split(": ");
		String category = data[0];
		String dirPath = FileIO.makeDirPath(dir, category);

		// ratings
		String ratingPath = dirPath + "ratings.txt";
		List<String> ratings = FileIO.readAsList(ratingPath);

		FileIO.writeList(ratingFile, ratings, null, true);

		// reviews
		String reviewPath = dirPath + "review-ratings.txt";
		List<String> reviews = FileIO.readAsList(reviewPath);

		FileIO.writeList(reviewFile, reviews, null, true);
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public static void crawl_data() throws Exception
{
	String filePath = FileIO.getResource("gewara.txt");

	List<String> urls = FileIO.readAsList(filePath);
	String[] tasks = { "comments" };

	int nd = 4;
	for (String task : tasks)
	{
		Logs.info("Current task: " + task);
		for (int i = 0; i < urls.size(); i += nd)
		{
			Thread[] tds = new Thread[nd];

			boolean flag = false;
			for (int j = 0; j < nd; j++)
			{
				if (i + j >= urls.size())
				{
					flag = true;
					break;
				}

				String url = urls.get(i + j).trim();
				tds[j] = new Thread(new GewaraCrawler(url, task, i + j + 1));
				tds[j].start();
			}

			for (Thread td : tds)
			{
				if (td != null) td.join();
			}

			if (flag) break;
		}
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public static void crawl_data() throws Exception
{
	String filePath = FileIO.getResource("douban.txt");

	List<String> urls = FileIO.readAsList(filePath);
	String[] tasks = { "reviews" };

	int nd = 4;
	for (String task : tasks)
	{
		Logs.info("Current task: " + task);
		for (int i = 0; i < urls.size(); i += nd)
		{
			Thread[] tds = new Thread[nd];

			boolean flag = false;
			for (int j = 0; j < nd; j++)
			{
				if (i + j >= urls.size())
				{
					flag = true;
					break;
				}

				String url = urls.get(i + j).trim();
				tds[j] = new Thread(new DoubanCrawler(url, task, i + j + 1));
				tds[j].start();
			}

			for (Thread td : tds)
			{
				if (td != null) td.join();
			}

			if (flag) break;
		}
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public static void crawl_data() throws Exception {
	if (Debug.OFF) {
		// home page
		CiaoCrawler cc = new CiaoCrawler();
		cc.run_home_page();
		return;
	}

	String sourceFile = "users.txt"; // "dvd.ciao.txt"
	String filePath = FileIO.getResource(sourceFile);
	List<String> urls = FileIO.readAsList(filePath);

	String[] tasks = { "users" };

	int nd = 8;
	for (String task : tasks) {
		Logs.info("Current task: " + task);
		for (int i = 0; i < urls.size(); i += nd) {
			Thread[] tds = new Thread[nd];

			boolean flag = false;
			for (int j = 0; j < nd; j++) {
				if (i + j >= urls.size()) {
					flag = true;
					break;
				}

				String url = urls.get(i + j).trim();
				tds[j] = new Thread(new CiaoCrawler(url, task, i + j + 1));
				tds[j].start();
			}

			for (Thread td : tds) {
				if (td != null)
					td.join();
			}

			if (flag)
				break;
		}
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public static void parseCategoryPages() throws Exception {
	String filePath = FileIO.getResource("dvd.ciao.txt");
	List<String> urls = FileIO.readAsList(filePath);

	String dir = Systems.getDesktop() + "dvd.ciao.co.uk\\";
	for (String url : urls) {
		// each category
		String[] data = url.split(": ");
		String category = data[0];
		String dirCate = FileIO.makeDirPath(dir, category);
		String dirPath = FileIO.makeDirPath(dir, category, "webPages");

		// clear
		FileIO.deleteFile(dirCate + "movies.txt");

		File dirs = new File(dirPath);
		for (File f : dirs.listFiles()) {
			// each web page
			String html = FileIO.readAsString(dirPath + f.getName());
			Document doc = Jsoup.parse(html);

			Logs.debug(category + ": " + f.getName());

			List<String> movies = new ArrayList<>();
			Elements products = doc.select("td.prodInfo");
			for (Element product : products) {
				// each product
				Element prod = product.select("p.prodName").first();
				String name = prod.text();
				String link = prod.select("a").first().attr("href");
				String id = link.substring(link.lastIndexOf("_") + 1);

				// number of user reviews
				prod = product.select("p.prodRating").first();
				String cnt = prod.select(".userReviewsCount").text()
						.replace("(", "").replace(")", "");
				int count = 0;
				if (!cnt.isEmpty())
					count = Integer.parseInt(cnt);

				// do not consider movies without any reviews
				if (count > 0) {
					String movie = id + "::" + name + "::" + link;
					movies.add(movie);
				}
			}

			FileIO.writeList(dirCate + "movies.txt", movies, null, true);
		}
	}

}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public static void parseReviewPages() throws Exception {
	String filePath = FileIO.getResource("dvd.ciao.txt");
	List<String> urls = FileIO.readAsList(filePath);

	String dir = Systems.getDesktop() + "dvd.ciao.co.uk\\";
	for (String url : urls) {
		// each category
		String[] data = url.split(": ");
		String category = data[0];
		String dirCate = FileIO.makeDirPath(dir, category);

		File dirs = new File(dirCate);
		for (File f : dirs.listFiles()) {
			// each product folder
			if (f.getName().equals("webPages"))
				continue;
			if (!f.isDirectory())
				continue;

			String prodPath = FileIO.makeDirPath(dirCate, f.getName());
			String revwPath = FileIO.makeDirPath(prodPath, "Reviews");

			List<String> reviews = new ArrayList<>();
			File reviewDirs = new File(revwPath);

			for (File rf : reviewDirs.listFiles()) {
				// each review page
				String html = FileIO.readAsString(rf.getPath());
				Document doc = Jsoup.parse(html);

				Elements es = doc.select("div.m-shortReviewSnippet");
				for (Element e : es) {
					Element a = e.select(
							"p.m-shet-review-title a.ReviewTitle").first();
					if (a == null)
						continue; // some reviews do not have specific link
									// to detailed contents

					// url
					String link = a.attr("href");

					int idx = link.lastIndexOf("_");
					String id = link.substring(idx + 1);

					reviews.add(id + "::" + link);
				}
			}

			FileIO.writeList(prodPath + "reviews.txt", reviews, null, false);
		}

	}

}