Java FileIO.writeString方法代码示例

import happy.coding.io.FileIO; //导入方法依赖的package包/类
private void run_home_page() throws Exception {
	String url = "http://dvd.ciao.co.uk/";
	String html = read_url(url);
	FileIO.writeString(dir + "dvd.ciao.html", html);

	Document doc = Jsoup.parse(html);
	Element categories = doc.getElementById("category_tree_table");
	Elements cs = categories.select("dl");

	List<String> cls = new ArrayList<>();
	for (Element c : cs) {
		Element cat = c.select("dt").first().select("a").first();
		String category = cat.text();
		String link = cat.attr("href");

		cls.add(category + ": " + link);
	}

	FileIO.writeList(dir + "dvd.ciao.txt", cls);
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void run_web_pages(String url) throws Exception {
	String[] data = url.split(": ");
	String category = data[0];
	String link = data[1];
	String dirPath = FileIO.makeDirectory(dir, category, "webPages");

	int pageSize = 15;
	String html = read_url(link);
	FileIO.writeString(dirPath + "page_" + 1 + ".html", html);

	Document doc = Jsoup.parse(html);
	int maxPage = Integer.parseInt(doc.select(
			"div.CWCiaoKievPagination.clearfix li.last").text());
	Logs.debug(category + ": progress [" + 1 + "/" + maxPage + "]");

	for (int i = 2; i <= maxPage; i++) {
		String pageLink = link + "~s" + (i - 1) * pageSize;
		String content = read_url(pageLink);
		FileIO.writeString(dirPath + "page_" + i + ".html", content);
		Logs.debug(category + ": progress [" + i + "/" + maxPage + "]");
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void crawl_web_pages() throws Exception
{
	String filePath = "./src/main/resources/mtime.txt";
	List<String> urls = FileIO.readAsList(filePath);

	for (String url : urls)
	{
		String html = URLReader.read(url);
		Document doc = Jsoup.parse(html);
		String name = doc.select("span[property=v:itemreviewed]").text();
		name = Strings.filterWebString(name, '_');

		String dirPath = dir + name + "/";
		FileIO.makeDirectory(dirPath);
		FileIO.writeString(dirPath + name + ".html", html);
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public static void retrieveTrustData(String dirPath, String trustPath) throws Exception
{
	ConfigParams.defaultInstance();
	String ratingSet = dirPath + Dataset.RATING_SET;
	Map<String, Map<String, Rating>> userMap = Dataset.loadRatingSet(ratingSet);

	BufferedReader br = new BufferedReader(new FileReader(trustPath));
	StringBuilder sb = new StringBuilder();
	String line = null;

	while ((line = br.readLine()) != null)
	{
		if (line.isEmpty()) continue;

		String[] data = line.split(Dataset.REGMX);
		String trustor = data[0];
		String trustee = data[1];

		if (userMap.containsKey(trustor) && userMap.containsKey(trustee)) sb.append(line + "\n");
	}
	br.close();

	String filePath = dirPath + Dataset.TRUST_SET;
	FileIO.writeString(filePath, sb.toString());
	Logs.debug("Saved the trust sample to: " + filePath);
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void crawl_web_pages(String url) throws Exception
{
	String html = read_url(url);
	Document doc = Jsoup.parse(html);
	String name = doc.select("div.detail_head_name h1").first().text();
	name = Strings.filterWebString(name, '_');

	String dirPath = dir + name + "/";
	FileIO.makeDirectory(dirPath);
	FileIO.writeString(dirPath + name + ".html", html);
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void run_web_pages(String url) throws Exception
{
	String html = read_url(url);
	Document doc = Jsoup.parse(html);
	String name = doc.select("span[property=v:itemreviewed]").text();
	name = Strings.filterWebString(name, '_');

	String dirPath = dir + name + "/";
	FileIO.makeDirectory(dirPath);
	FileIO.writeString(dirPath + name + ".html", html);
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void run_ratings(String url) throws Exception
{
	String html = read_url(url);
	Document doc = Jsoup.parse(html);
	String name = doc.select("span[property=v:itemreviewed]").text();
	name = Strings.filterWebString(name, '_');

	String dirPath = dir + name + "/ratings/";
	FileIO.makeDirectory(dirPath);

	// save rating pages
	int k = 0;
	while (true)
	{
		String link = url + "collections?start=" + (k * 20);
		String page = read_url(link);

		k++;
		FileIO.writeString(dirPath + "page_" + k + ".html", page);
		Logs.debug("Current processing page: " + k);

		// if finished;
		Document doc2 = Jsoup.parse(page);
		Elements es = doc2.select("div#collections_tab span.next");
		if (es == null || es.size() == 0)
		{
			break;
		}
	}

}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public static void ReadFolder(String dirPath) throws Exception
{
	dirPath = FileIO.makeDirPath(dirPath);
	File dir = new File(dirPath);
	if (!dir.isDirectory()) throw new Exception(dirPath + " is not a directory");

	File[] files = dir.listFiles();
	for (File file : files)
	{
		String results = file.getName() + "\r\n";
		results += FileIO.readAsString(file.getPath(), new String[] { "MAE", "MAUE" });
		FileIO.writeString(dirPath + "Results.txt", results, true);
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
private void output_trust(String line, int... indexes) throws Exception {
	String dir = Dataset.DIRECTORY + "Trust/";
	for (int i = 0; i < indexes.length; i++) {
		int index = indexes[i];
		if (i > 0)
			dir += "_";
		dir += index;
	}
	dir += "/";
	FileIO.makeDirectory(dir);
	String file = dir + "trust.txt";

	FileIO.writeString(file, line, true);
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
@Test
public void usage() throws Exception
{
	String url = "http://www.80stees.com/products/DC_Comics_Superman_Classic_t-shirt.asp";
	String html = URLReader.read(url);
	String filePath = Systems.getDesktop() + "html.html";

	FileIO.writeString(filePath, html);

	FileIO.writeString(Systems.getDesktop() + "html2.html", URLReader.read(url, "46.231.14.177", 8080));
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void run_user(String url) throws Exception {
	String[] data = url.split(",");
	String userID = data[0];
	String userUrl = data[1];

	String userPath = FileIO.makeDirPath(desktop, domain,
			"users.ciao.co.uk");
	String html = read_url(userUrl);

	// check if user exists now
	Document doc = Jsoup.parse(html);
	Elements tabs = doc.select("table.tabs");
	if (tabs == null || tabs.size() == 0)
		return; // no such user profile

	userPath = FileIO.makeDirectory(userPath, userID);
	FileIO.writeString(userPath + userID + ".html", html);

	// trusted neighbors
	String link = "http://www.ciao.co.uk/member_view.php/MemberId/"
			+ userID + "/TabId/5/subTabId/1";
	html = read_url(link);

	// find the max pages
	doc = Jsoup.parse(html);
	Element page = doc
			.select("table#comparePricesShowAllTop td.rangepages").first();
	if (page == null)
		return; // no friends at all

	FileIO.writeString(userPath + "friends-1.html", html);

	if (page.text().length() <= 1)
		return; // no more pages

	Element a = page.select("a").last();
	int maxPage = Integer.parseInt(a.text());

	for (int i = 2; i <= maxPage; i++) {
		String nextPage = link + "/Start/" + (i - 1) * 15;
		html = read_url(nextPage);
		FileIO.writeString(userPath + "friends-" + i + ".html", html);
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void run_products(String url) throws Exception {
	String[] data = url.split(": ");
	String category = data[0];
	// String link = data[1];

	String dirPath = FileIO.makeDirPath(desktop, domain, category);
	List<String> links = FileIO.readAsList(dirPath + "movies.txt");
	int tk = links.size();
	for (int k = 0; k < tk; k++) {
		String link = links.get(k);
		String[] d = link.split("::");
		String id = d[0];
		String name = d[1];
		String productLink = d[2];
		int idx = productLink.lastIndexOf("/");
		String p1 = productLink.substring(0, idx) + "/Reviews";
		String reviewLink = p1 + productLink.substring(idx);

		// create folder
		String path = FileIO.makeDirectory(dirPath, id);

		// product page
		String html = null;

		String pagePath = path + id + ".html";
		if (!FileIO.exist(pagePath)) {
			html = read_url(productLink);
			FileIO.deleteFile(path + name + ".html");
			FileIO.writeString(pagePath, html);
		}

		// product reviews
		// get first page anyway to identify the maximum pages
		path = FileIO.makeDirectory(path, "Reviews");
		String reviewPath = path + "page_1.html";
		if (FileIO.exist(reviewPath)) {
			html = FileIO.readAsString(reviewPath);
		} else {
			html = read_url(reviewLink);
			FileIO.writeString(reviewPath, html);
		}
		Logs.debug(category + ": " + id + " (" + (k + 1) + "/" + tk + ")"
				+ ": page " + 1);

		Document doc = Jsoup.parse(html);
		Elements nav = doc.select("div#Pagination");

		if (!nav.isEmpty()) {
			int maxPage = 1;

			Elements last = nav.select("li.last");
			if (!last.isEmpty())
				maxPage = Integer.parseInt(last.first().text()); // more
																	// than
																	// 11
																	// pages
			else
				maxPage = Integer.parseInt(nav.select("li").last().text()); // less
																			// or
																			// equal
																			// 11
																			// pages

			for (int i = 2; i <= maxPage; i++) {
				String filePath = path + "page_" + i + ".html";
				if (FileIO.exist(filePath))
					continue;

				reviewLink = reviewLink + "/Start/" + ((i - 1) * 15);
				html = read_url(reviewLink);
				FileIO.writeString(filePath, html);

				Logs.debug(category + ": " + id + " (" + (k + 1) + "/" + tk
						+ ")" + ": page " + i + "/" + maxPage);
			}
		}
	}

}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void crawl_comments(String url) throws Exception
{
	String html = read_url(url);
	Document doc = Jsoup.parse(html);
	String name = doc.select("div.detail_head_name h1").first().text();
	name = Strings.filterWebString(name, '_');

	String val = doc.select("#detail_nav li a").first().attr("href");
	String id = val.substring(val.lastIndexOf("/") + 1);

	String dirPath = dir + name + "/comments/";
	FileIO.makeDirectory(dirPath);

	// save rating pages
	int max = 1;
	boolean maxSet = false;
	url = url + "/commentlist";

	for (int k = 0; k <= max; k++)
	{
		String page_file = dirPath + "page_" + (k + 1) + ".html";
		Logs.debug(name + " comments with page: " + (k + 1) + "/" + (max + 1));

		String contents = null;
		if (!FileIO.exist(page_file))
		{

			String link = "http://www.gewara.com/ajax/common/qryComment.xhtml?pageNumber="
					+ k
					+ "&relatedid="
					+ id
					+ "&title=&issue=false&hasMarks=true&tag=movie&isPic=true&isVideo=false&pages=true&maxCount=20&userLogo=";

			contents = read_url(link);
			FileIO.writeString(page_file, contents);// new String(contents.getBytes("utf-8"), "utf-8"));
		} else
		{
			contents = FileIO.readAsString(page_file);
		}

		// find the maximum page num;
		if (!maxSet)
		{
			Document doc2 = Jsoup.parse(contents);
			Elements es = doc2.select("div#page a");
			Element e = es.get(es.size() - 2);
			max = Integer.parseInt(e.attr("lang"));
			maxSet = true;
		}

	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception
{
	String dir_path = "D:/Dropbox/PhD/My Work/Ongoing/Data Crawl/gewara.com/";
	File dir = new File(dir_path);
	String sep = ",\t";

	for (File movie : dir.listFiles())
	{
		if (!movie.isDirectory()) continue;
		String movie_path = movie.getPath();

		File comments = new File(movie_path + "/comments/");

		String file = movie_path + "/comments.csv";
		FileIO.deleteFile(file);

		int total = comments.listFiles().length;
		for (File page : comments.listFiles())
		{
			Logs.debug("Current page: " + page.getName() + "/" + total);

			Document doc = Jsoup.parse(FileIO.readAsString(page.getPath()));

			Elements es = doc.select("div.ui_wala_comment dl");

			StringBuilder sb = new StringBuilder();
			for (int i = 0; i < es.size(); i++)
			{
				Element e = es.get(i);
				Element li = e.select("div.page_wala p").first();

				String user = li.select("a").first().text();
				String user_url = li.select("a").first().attr("href");

				Element eli = li.select("span.ui_grades").first();
				String rate = "";
				String comment = null;
				if (eli != null)
				{
					rate = eli.text();
					comment = eli.nextSibling().toString().substring(1);
				} else
				{
					comment = li.select("a").first().nextSibling().toString().substring(1);
				}

				li = e.select("div.page_replay.page_replay_my.clear").first();
				String time = e.select("span.left").first().text();
				String forward = li.select("a.page_ico.forwards").first().text();
				String reply = li.select("a.page_ico.comment").first().text();

				String line = user + sep + user_url + sep + rate + sep + time + sep + forward + sep + reply + sep
						+ comment;
				if (i < es.size() - 1) line += "\n";
				sb.append(line);
			}
			FileIO.writeString(file, sb.toString(), true);
		}
	}
}
 

import happy.coding.io.FileIO; //导入方法依赖的package包/类
public void run_reviews(String url) throws Exception
{
	url = url.trim();
	String html = read_url(url);
	Document doc = Jsoup.parse(html);
	String name = doc.select("span[property=v:itemreviewed]").text();
	name = Strings.filterWebString(name, '_');

	String dirPath = dir + name + "/reviews/";
	FileIO.makeDirectory(dirPath);

	// save rating pages
	int k = 0;
	url = url + "reviews";
	String link = url;
	while (true)
	{
		k++;
		String page = null;
		String path = dirPath + "page_" + k + ".html";
		if (!FileIO.exist(path))
		{
			page = read_url(link);
			FileIO.writeString(path, page);
			Logs.debug(name + " reviews with page: " + k);
		} else
		{
			page = FileIO.readAsString(path);
		}

		// find the next page link;
		Document doc2 = Jsoup.parse(page);
		Elements es = doc2.select("div#paginator a.next");
		if (es == null || es.size() == 0)
		{
			break;
		} else
		{
			link = url + es.first().attr("href");
		}
	}
}