Java FileIO.deleteFile方法代码示例

import happy.coding.io.FileIO; //导入方法依赖的package包/类
@Test
public void ratings() throws Exception {
	String dirPath = "D:\\Research\\Datasets\\Ciao\\";
	String dest = dirPath + "formatted\\" + "ratings.txt";

	FileIO.deleteFile(dest);

	BufferedReader br = FileIO.getReader(dirPath
			+ "rating_with_timestamp.txt");
	String line = null;
	List<String> lines = new ArrayList<>();
	while ((line = br.readLine()) != null) {
		String[] data = line.trim().split("  ");

		int user = (int) Double.parseDouble(data[0]);
		int item = (int) Double.parseDouble(data[1]);
		double rate = Double.parseDouble(data[3]);

		line = user + " " + item + " " + rate;
		lines.add(line);

		if (lines.size() >= 1000) {
			FileIO.writeList(dest, lines, true);
			lines.clear();
		}
	}

	if (lines.size() > 0)
		FileIO.writeList(dest, lines, true);

	br.close();
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
@Test
public void trust() throws Exception {
	String dirPath = "D:\\Research\\Datasets\\Ciao\\";
	String dest = dirPath + "formatted\\" + "trust.txt";

	FileIO.deleteFile(dest);

	BufferedReader br = FileIO.getReader(dirPath + "trust.txt");
	String line = null;
	List<String> lines = new ArrayList<>();
	while ((line = br.readLine()) != null) {
		String[] data = line.trim().split("  ");

		int trustor = (int) Double.parseDouble(data[0]);
		int trustee = (int) Double.parseDouble(data[1]);

		line = trustor + " " + trustee + " " + 1.0;
		lines.add(line);

		if (lines.size() >= 1000) {
			FileIO.writeList(dest, lines, true);
			lines.clear();
		}
	}

	if (lines.size() > 0)
		FileIO.writeList(dest, lines, true);

	br.close();
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void run_category_reviews(String url) throws Exception {
	String[] data = url.split(": ");
	String category = data[0];
	String catPath = FileIO.makeDirPath(desktop, domain, category);
	File Dir = new File(catPath);
	File[] prodDirs = Dir.listFiles();
	int tk = prodDirs.length;
	String movie_reviews = catPath + "movie-review-ratings.txt";
	FileIO.deleteFile(movie_reviews);

	for (int k = 0; k < tk; k++) {
		File prodDir = prodDirs[k];
		// for each product
		String productID = prodDir.getName();
		if (productID.equals("webPages"))
			continue;
		if (!prodDir.isDirectory())
			continue;

		String prodPath = FileIO.makeDirPath(catPath, productID);
		String dvdPath = prodPath + "review-ratings.txt";
		if (!FileIO.exist(dvdPath))
			continue; // no review ratings

		// read review ratings from each product, and remove the duplicated
		// review ratings
		Set<String> review_ratings = FileIO.readAsSet(dvdPath);
		FileIO.writeList(movie_reviews, review_ratings, null, true);
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
/**
 * Concate all the dvd ratings about the products in a specific category
 * 
 * @param url
 * @throws Exception
 */
public void run_category_ratings(String url) throws Exception {
	String[] data = url.split(": ");
	String category = data[0];

	String catPath = FileIO.makeDirPath(desktop, domain, category);
	File Dir = new File(catPath);
	File[] prodDirs = Dir.listFiles();
	int tk = prodDirs.length;

	String ratingFile = catPath + "movie-ratings.txt";
	FileIO.deleteFile(ratingFile);

	for (int k = 0; k < tk; k++) {
		File prodDir = prodDirs[k];
		// for each product
		String productID = prodDir.getName();
		if (productID.equals("webPages"))
			continue;
		if (!prodDir.isDirectory())
			continue;

		Logs.debug("{}: {} ({}/{})", new Object[] { category, productID,
				(k + 1), tk });

		String prodPath = FileIO.makeDirPath(catPath, productID);
		String dvdPath = prodPath + "dvd-ratings.txt";
		if (!FileIO.exist(dvdPath))
			continue;

		List<String> dvd_ratings = FileIO.readAsList(dvdPath);

		FileIO.writeList(ratingFile, dvd_ratings, null, true);
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public static void getAllRatings() throws Exception {
	String filePath = FileIO.getResource("dvd.ciao.txt");
	List<String> urls = FileIO.readAsList(filePath);

	String dir = Systems.getDesktop() + "dvd.ciao.co.uk\\";
	String ratingFile = dir + "ratings.txt";
	String reviewFile = dir + "review-ratings.txt";
	FileIO.deleteFile(ratingFile);
	FileIO.deleteFile(reviewFile);

	for (String url : urls) {
		// each category
		String[] data = url.split(": ");
		String category = data[0];
		String dirPath = FileIO.makeDirPath(dir, category);

		// ratings
		String ratingPath = dirPath + "ratings.txt";
		List<String> ratings = FileIO.readAsList(ratingPath);

		FileIO.writeList(ratingFile, ratings, null, true);

		// reviews
		String reviewPath = dirPath + "review-ratings.txt";
		List<String> reviews = FileIO.readAsList(reviewPath);

		FileIO.writeList(reviewFile, reviews, null, true);
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
@Test
public void samplingNetflix() throws Exception
{
	ConfigParams.defaultInstance();

	String source = Dataset.DIRECTORY + Dataset.RATING_SET;
	String dest = Dataset.DIRECTORY + "Sample" + Systems.FILE_SEPARATOR + Dataset.RATING_SET;
	FileIO.deleteFile(dest);

	BufferedReader br = new BufferedReader(new FileReader(new File(source)));
	String line = null;
	List<String> lines = new ArrayList<>(3000);
	int numUser = 1000;
	while ((line = br.readLine()) != null)
	{
		String[] data = line.split(" ");
		int userId = Integer.parseInt(data[0]);

		if (userId > numUser) continue;
		lines.add(line);

		if (lines.size() > 1024)
		{
			FileIO.writeList(dest, lines, null, true);
			lines.clear();
		}
	}
	br.close();

	if (lines.size() > 0) FileIO.writeList(dest, lines, null, true);
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void run_dvd_ratings(String url) throws Exception {
	String[] data = url.split(": ");
	String category = data[0];
	String category_url = data[1];
	category_url = category_url.substring(0, category_url.lastIndexOf('_'));
	String categoryID = category_url.substring(category_url
			.lastIndexOf('_') + 1);

	String catPath = FileIO.makeDirPath(desktop, domain, category);
	File Dir = new File(catPath);
	File[] prodDirs = Dir.listFiles();
	int tk = prodDirs.length;
	for (int k = 0; k < tk; k++) {
		File prodDir = prodDirs[k];
		// for each product
		String productID = prodDir.getName();
		if (productID.equals("webPages"))
			continue;
		if (!prodDir.isDirectory())
			continue;

		String prodPath = FileIO.makeDirPath(catPath, productID);
		String reviewPath = FileIO
				.makeDirPath(prodPath, "Detailed_Reviews");
		String dvdPath = prodPath + "dvd-ratings.txt";
		FileIO.deleteFile(dvdPath);

		File reviewDir = new File(reviewPath);
		File[] reviewDirs = reviewDir.listFiles();
		List<String> reviews = new ArrayList<>();
		for (int i = 0; i < reviewDirs.length; i++) {
			// for each review
			File reviewFile = reviewDirs[i];
			String reviewID = reviewFile.getName();
			Logs.debug("{}: {} ({}/{}): {} ({}/{})", category, productID,
					(k + 1), tk, reviewID, (i + 1), reviewDirs.length);

			String dirPath = FileIO.makeDirPath(reviewPath, reviewID);
			String ratingPath = dirPath + reviewID + ".html";
			if (!FileIO.exist(ratingPath))
				continue; // review page is not existing
			String html = FileIO.readAsString(ratingPath);
			Document doc = Jsoup.parse(html);
			Element div = doc.select("div#OH_BingUserInfo").first();
			if (div == null)
				continue; // no user review exists

			// user-info
			Element a = div.select("p.m-reer-usertab.clearfix a.black")
					.first();
			String raw = a.attr("onmousedown");
			raw = raw.substring(raw.indexOf("(") + 1, raw.lastIndexOf(")"));
			String userUrl = raw.replace(",", "").replace("'", "");
			String userID = userUrl.substring(userUrl.lastIndexOf('_') + 1);

			// user-rating value
			Element r = div.select(
					"p.m-reer-usertab.clearfix img.ratingStars").first();
			String rating = r.attr("alt");

			// user-rating date
			div = doc.select("div#OH_BingUserOpinion").first();
			Element date = div
					.select("div.m-reer-opheader.reviewTitle span.m-reer-ddwrap span[property]")
					.first();
			String datetime = date.attr("content");

			// content
			String review = userID + "," + productID + "," + categoryID
					+ "," + reviewID + "," + rating + "," + datetime + ","
					+ userUrl;
			reviews.add(review);
		}
		FileIO.writeList(dvdPath, reviews);
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void run_products(String url) throws Exception {
	String[] data = url.split(": ");
	String category = data[0];
	// String link = data[1];

	String dirPath = FileIO.makeDirPath(desktop, domain, category);
	List<String> links = FileIO.readAsList(dirPath + "movies.txt");
	int tk = links.size();
	for (int k = 0; k < tk; k++) {
		String link = links.get(k);
		String[] d = link.split("::");
		String id = d[0];
		String name = d[1];
		String productLink = d[2];
		int idx = productLink.lastIndexOf("/");
		String p1 = productLink.substring(0, idx) + "/Reviews";
		String reviewLink = p1 + productLink.substring(idx);

		// create folder
		String path = FileIO.makeDirectory(dirPath, id);

		// product page
		String html = null;

		String pagePath = path + id + ".html";
		if (!FileIO.exist(pagePath)) {
			html = read_url(productLink);
			FileIO.deleteFile(path + name + ".html");
			FileIO.writeString(pagePath, html);
		}

		// product reviews
		// get first page anyway to identify the maximum pages
		path = FileIO.makeDirectory(path, "Reviews");
		String reviewPath = path + "page_1.html";
		if (FileIO.exist(reviewPath)) {
			html = FileIO.readAsString(reviewPath);
		} else {
			html = read_url(reviewLink);
			FileIO.writeString(reviewPath, html);
		}
		Logs.debug(category + ": " + id + " (" + (k + 1) + "/" + tk + ")"
				+ ": page " + 1);

		Document doc = Jsoup.parse(html);
		Elements nav = doc.select("div#Pagination");

		if (!nav.isEmpty()) {
			int maxPage = 1;

			Elements last = nav.select("li.last");
			if (!last.isEmpty())
				maxPage = Integer.parseInt(last.first().text()); // more
																	// than
																	// 11
																	// pages
			else
				maxPage = Integer.parseInt(nav.select("li").last().text()); // less
																			// or
																			// equal
																			// 11
																			// pages

			for (int i = 2; i <= maxPage; i++) {
				String filePath = path + "page_" + i + ".html";
				if (FileIO.exist(filePath))
					continue;

				reviewLink = reviewLink + "/Start/" + ((i - 1) * 15);
				html = read_url(reviewLink);
				FileIO.writeString(filePath, html);

				Logs.debug(category + ": " + id + " (" + (k + 1) + "/" + tk
						+ ")" + ": page " + i + "/" + maxPage);
			}
		}
	}

}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public static void parseCategoryPages() throws Exception {
	String filePath = FileIO.getResource("dvd.ciao.txt");
	List<String> urls = FileIO.readAsList(filePath);

	String dir = Systems.getDesktop() + "dvd.ciao.co.uk\\";
	for (String url : urls) {
		// each category
		String[] data = url.split(": ");
		String category = data[0];
		String dirCate = FileIO.makeDirPath(dir, category);
		String dirPath = FileIO.makeDirPath(dir, category, "webPages");

		// clear
		FileIO.deleteFile(dirCate + "movies.txt");

		File dirs = new File(dirPath);
		for (File f : dirs.listFiles()) {
			// each web page
			String html = FileIO.readAsString(dirPath + f.getName());
			Document doc = Jsoup.parse(html);

			Logs.debug(category + ": " + f.getName());

			List<String> movies = new ArrayList<>();
			Elements products = doc.select("td.prodInfo");
			for (Element product : products) {
				// each product
				Element prod = product.select("p.prodName").first();
				String name = prod.text();
				String link = prod.select("a").first().attr("href");
				String id = link.substring(link.lastIndexOf("_") + 1);

				// number of user reviews
				prod = product.select("p.prodRating").first();
				String cnt = prod.select(".userReviewsCount").text()
						.replace("(", "").replace(")", "");
				int count = 0;
				if (!cnt.isEmpty())
					count = Integer.parseInt(cnt);

				// do not consider movies without any reviews
				if (count > 0) {
					String movie = id + "::" + name + "::" + link;
					movies.add(movie);
				}
			}

			FileIO.writeList(dirCate + "movies.txt", movies, null, true);
		}
	}

}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public static void getAllTrust() throws Exception {

		Map<String, String> users = new HashMap<>();
		List<String> userLines = FileIO.readAsList(FileIO
				.getResource("users.txt"));
		for (String line : userLines) {
			String[] data = line.split(",");
			users.put(data[1], data[0]);
		}

		String usersPath = FileIO.makeDirPath(desktop, domain,
				"users.ciao.co.uk");
		FileIO.deleteFile(usersPath + "trust.txt");

		File dir = new File(usersPath);
		File[] files = dir.listFiles();

		for (int i = 0, n = files.length; i < n; i++) {
			// for each user
			File userFile = files[i];
			final String userID = userFile.getName();

			String html = null;
			Document doc = null;
			List<String> friends = new ArrayList<>();
			File[] pages = userFile.listFiles();
			for (int j = 0, m = pages.length; j < m; j++) {
				// for each trust page
				File file = pages[j];
				String name = file.getName();
				if (name.startsWith("friends")) {
					html = FileIO.readAsString(file.getPath());
					doc = Jsoup.parse(html);
					Element trustTable = doc.select("form table.trust").first();
					Elements trs = trustTable.select("tbody tr");
					for (Element tr : trs) {
						Element td = tr.select("td").get(1);
						Element a = td.select("a").first();
						String link = a.attr("href");
						if (users.containsKey(link)) {
							friends.add(users.get(link));
						}
					}
				}
			}

			FileIO.writeList(usersPath + "trust.txt", friends,
					new Converter<String, String>() {

						@Override
						public String transform(String friend) {
							return userID + "," + friend + ",1";
						}
					}, true);
		}

	}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception
{
	String dir_path = "D:/Dropbox/PhD/My Work/Ongoing/Data Crawl/gewara.com/";
	File dir = new File(dir_path);
	String sep = ",\t";

	for (File movie : dir.listFiles())
	{
		if (!movie.isDirectory()) continue;
		String movie_path = movie.getPath();

		File comments = new File(movie_path + "/comments/");

		String file = movie_path + "/comments.csv";
		FileIO.deleteFile(file);

		int total = comments.listFiles().length;
		for (File page : comments.listFiles())
		{
			Logs.debug("Current page: " + page.getName() + "/" + total);

			Document doc = Jsoup.parse(FileIO.readAsString(page.getPath()));

			Elements es = doc.select("div.ui_wala_comment dl");

			StringBuilder sb = new StringBuilder();
			for (int i = 0; i < es.size(); i++)
			{
				Element e = es.get(i);
				Element li = e.select("div.page_wala p").first();

				String user = li.select("a").first().text();
				String user_url = li.select("a").first().attr("href");

				Element eli = li.select("span.ui_grades").first();
				String rate = "";
				String comment = null;
				if (eli != null)
				{
					rate = eli.text();
					comment = eli.nextSibling().toString().substring(1);
				} else
				{
					comment = li.select("a").first().nextSibling().toString().substring(1);
				}

				li = e.select("div.page_replay.page_replay_my.clear").first();
				String time = e.select("span.left").first().text();
				String forward = li.select("a.page_ico.forwards").first().text();
				String reply = li.select("a.page_ico.comment").first().text();

				String line = user + sep + user_url + sep + rate + sep + time + sep + forward + sep + reply + sep
						+ comment;
				if (i < es.size() - 1) line += "\n";
				sb.append(line);
			}
			FileIO.writeString(file, sb.toString(), true);
		}
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void parse_ratings() throws Exception
{
	File directory = new File(dir);
	File[] movies = directory.listFiles();

	// different movies
	for (File movie : movies)
	{
		if (!movie.isDirectory()) continue;
		String movie_path = movie.getPath();
		File ratings = new File(movie_path + "/ratings/");

		// different pages
		String rating_file = movie_path + "/ratings.csv";
		FileIO.deleteFile(rating_file);

		for (File page : ratings.listFiles())
		{
			Document doc = Jsoup.parse(FileIO.readAsString(page.getPath()));
			Elements es = doc.select("div#collections_tab .sub_ins table");

			StringBuilder sb = new StringBuilder();
			for (int i = 0; i < es.size(); i++)
			{
				Element e = es.get(i);
				String line = "";
				Element user = e.select("div.pl2 a").first();
				Element rate = e.select("p.pl").first();

				line += user.text() + sep; // user
				if (rate.select("span").size() == 0) continue; // if no rating, ignore it
				line += rate.select("span").first().attr("class").substring(7, 8) + sep; // rating
				line += rate.text().substring(0, 10) + sep; // time
				line += user.attr("href"); // user url

				if (i < es.size() - 1) line += "\n";

				sb.append(line);
			}

			FileIO.writeString(rating_file, sb.toString(), true);
		}
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void parse_reviews() throws Exception
{
	File directory = new File(dir);
	File[] movies = directory.listFiles();

	// different movies
	for (File movie : movies)
	{
		if (!movie.isDirectory()) continue;
		String movie_path = movie.getPath();
		File ratings = new File(movie_path + "/reviews/");

		// different pages
		String rating_file = movie_path + "/reviews.csv";
		FileIO.deleteFile(rating_file);

		for (File page : ratings.listFiles())
		{
			Document doc = Jsoup.parse(FileIO.readAsString(page.getPath()));
			Elements es = doc.select("div.ctsh");

			StringBuilder sb = new StringBuilder();
			for (int i = 0; i < es.size(); i++)
			{
				Element e = es.get(i);

				Element li = e.select("li.nlst").first();
				String detail_url = li.select("a.j.a_unfolder").first().attr("href");
				String title = li.select("a[title]").first().attr("title");

				li = e.select("li.ilst").first();
				String user = li.select("a").first().attr("title");
				String user_url = li.select("a").first().attr("href");

				li = e.select("li.clst.report-link").first();
				String rate = li.select("span.pl.ll.obss").first().child(1).attr("class").substring(7, 8);
				String review = li.select("div.review-short").first().childNode(0).toString().substring(1);

				String str = li.select("span.fleft").first().text();
				String[] val = str.replace("&nbsp", " ").split(" ");
				String datetime = val[0] + " " + val[1];
				String helpful = val[4].substring(0, val[4].indexOf("有用"));

				String line = title + sep + rate + sep + user + sep + user_url + sep + datetime + sep + helpful
						+ sep + detail_url + sep + review;

				if (i < es.size() - 1) line += "\n";

				sb.append(line);
			}

			FileIO.writeString(rating_file, sb.toString(), true);
		}
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void parse_comments() throws Exception
{
	File directory = new File(dir);
	File[] movies = directory.listFiles();

	// different movies
	for (File movie : movies)
	{
		if (!movie.isDirectory()) continue;
		String movie_path = movie.getPath();
		File ratings = new File(movie_path + "/comments/");

		// different pages
		String rating_file = movie_path + "/comments.csv";
		FileIO.deleteFile(rating_file);

		for (File page : ratings.listFiles())
		{
			Logs.debug("current page: " + page.getName() + "/" + ratings.listFiles().length);

			Document doc = Jsoup.parse(FileIO.readAsString(page.getPath()));
			Elements es = doc.select("div.comment-item");

			StringBuilder sb = new StringBuilder();
			for (int i = 0; i < es.size(); i++)
			{
				Element e = es.get(i).select("div.comment").first();

				Element li = e.select("span.comment-vote").first();
				String helpful = li.select("span.votes.pr5").first().text();

				li = e.select("span.comment-info").first();

				String user = li.select("a").first().text();
				String user_url = li.select("a").first().attr("href");

				Element eli = li.select("span[title]").first();
				String rate = "";
				String date = null;
				if (eli != null)
				{
					rate = eli.attr("class").substring(7, 8);

					date = li.childNode(4).toString();
				} else
				{
					date = li.childNode(2).toString();
				}
				String comment = e.select("p").first().text();

				String line = user + sep + user_url + sep + rate + sep + date + sep + helpful + sep + comment;

				if (i < es.size() - 1) line += "\n";

				sb.append(line);
			}

			FileIO.writeString(rating_file, sb.toString(), true);
		}
	}
}