Java Element.absUrl方法代码示例

import org.jsoup.nodes.Element; //导入方法依赖的package包/类
@Override
protected Void doInBackground(final Void... voids) {

    final Document document;
    try {
        document = Jsoup.connect(PUBG_NEWS_LINK).get();
        for (final Element row : document.select(NEWS_DOCUMENT_SELECTOR)) {
            final String title = row.select(NEWS_TITLE_SELECTOR).text();
            final String type = row.select(NEWS_TYPE_SELECTOR).text();
            final String date = row.select(NEWS_DATE_SELECTOR).text();
            final String description = row.select(NEWS_DESCRIPTION_SELECTOR).text();
            final Element link = row.select(NEWS_LINK_SELECTOR).first();
            final String linkSrc = link.absUrl(NEWS_LINKSRC_SELECTOR);
            final Element img = row.select(NEWS_IMG_SELECTOR).first();
            final String imgSrc = img.absUrl(NEWS_IMGSRC_SELECTOR);

            publishProgress(new NewsItem(title, type, date, description, linkSrc, imgSrc));
        }
    } catch(IOException e) {
        e.printStackTrace();
    }
    return null;
}
 

import org.jsoup.nodes.Element; //导入方法依赖的package包/类
@Override
public void handle(boolean allAttr, String attrKey, Element element, List<SIPNode> ret) {
    if (allAttr) {
        for (Attribute attribute : element.attributes()) {
            ret.add(SIPNode.t(element.absUrl(attribute.getKey())));
        }
    } else {
        String value = element.absUrl(attrKey);
        if (StringUtils.isNotBlank(value)) {
            ret.add(SIPNode.t(value));
        }
    }
}
 

import org.jsoup.nodes.Element; //导入方法依赖的package包/类
private static List<String> getURLsFromPageStatic(Document doc) {
     List<String> imageURLs = new ArrayList<>();
     Elements els = doc.select("#ContentPlaceHolder1_divContent");
     Elements imgs = els.select("img");
     for (Element img : imgs) {
         String src = img.absUrl("src");
         src = src.replaceAll("_[a-zA-Z]{3,5}", "");

         if (!src.equals("")) {
             imageURLs.add(src);
         }
     }
     return imageURLs;
}
 

import org.jsoup.nodes.Element; //导入方法依赖的package包/类
@Override
public void action(String[] args, GuildMessageReceivedEvent event) throws ParseException, IOException {
    if (args.length < 1) {event.getChannel().sendMessage(help()); return;}
    String query = "";
    for(int i = 0; i < args.length; i++){
        query += " " + args[i];
    }
    String google = "http://www.google.com/search?q=";
    String search = query;
    String charset = "UTF-8";
    String userAgent = "Amme 7.5 (+http://lordlee.de)";

    Elements links = Jsoup.connect(google + URLEncoder.encode(search, charset)).userAgent(userAgent).get().select(".g>.r>a");

    for (Element link : links) {
        String title = link.text();
        String url = link.absUrl("href"); // Google returns URLs in format "http://www.google.com/url?q=<url>&sa=U&ei=<someKey>".
        url = URLDecoder.decode(url.substring(url.indexOf('=') + 1, url.indexOf('&')), "UTF-8");

        if (!url.startsWith("http")) {
            continue; // Ads/news/etc.
        }

        ttemp = ttemp + link.text().toString() + link.baseUri() +  "\n";

    }
    util.embedSender.sendEmbed("Search Results for **" + query + "**:\n" + ttemp,event.getChannel(), Color.GREEN);
}
 

import org.jsoup.nodes.Element; //导入方法依赖的package包/类
/**
 * 将页面转为{@link org.jsoup.nodes.Document}对象，xhtml 格式
 *
 * @param url
 * @return
 * @throws Exception
 */
protected Document url2xhtml(String url) throws Exception {
    Document doc = Jsoup.connect(url).get(); //获得

    if (logger.isDebugEnabled()) {
        logger.debug("baseUri: {}", doc.baseUri());
    }

    for (Element script : doc.getElementsByTag("script")) { //除去所有 script
        script.remove();
    }

    for (Element a : doc.getElementsByTag("a")) { //除去 a 的 onclick，href 属性
        a.removeAttr("onclick");
        a.removeAttr("href");
    }

    Elements links = doc.getElementsByTag("link"); //将link中的地址替换为绝对地址
    for (Element element : links) {
        String href = element.absUrl("href");

        if (logger.isDebugEnabled()) {
            logger.debug("href: {} -> {}", element.attr("href"), href);
        }

        element.attr("href", href);
    }

    doc.outputSettings()
            .syntax(Document.OutputSettings.Syntax.xml)
            .escapeMode(Entities.EscapeMode.xhtml);  //转为 xhtml 格式

    if (logger.isDebugEnabled()) {
        String[] split = doc.html().split("\n");
        for (int c = 0; c < split.length; c++) {
            logger.debug("line {}:\t{}", c + 1, split[c]);
        }
    }
    return doc;
}
 

import org.jsoup.nodes.Element; //导入方法依赖的package包/类
public Link parse() throws IOException {
    Link parent = null;

    do {
        visited.add(startUrl);

        Connection con = Jsoup.connect(startUrl).userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0");
        Document doc = con.get();

        Element head = doc.getElementById("firstHeading");
        Link current = new Link(startUrl, head.text());

        current.parent = parent;
        parent = current;

        Elements links = doc.getElementById("bodyContent").select("a[href]");

        for(Element link : links) {
            String url = link.absUrl("href");

            if(url.startsWith("https://en.wikipedia.org/wiki/") && url.lastIndexOf(":") == 5 && !url.contains("#")) {
                if(url.equals("https://en.wikipedia.org/wiki/Philosophy")) {
                    Link phil = new Link("https://en.wikipedia.org/wiki/Philosophy", "Philosophy");
                    phil.parent = parent;

                    return phil;
                }

                if(!visited.contains(url))
                    toVisit.add(url);
            }
        }

        startUrl = toVisit.poll();
    } while(!toVisit.isEmpty());

    return null;
}
 

import org.jsoup.nodes.Element; //导入方法依赖的package包/类
private void crawlQuoraPage(String url, boolean crawlRelated) {
	// log("crawling for Quora url : " + url);
	uniqueQuoraLinks.add(url);
	Document doc = getDocument(url);
	if (doc == null) {
		// log("document is null for quora url: " + url);
		return;
	}

	Elements statsElements = doc.select("div.QuestionStats");
	for (Element statsElement : statsElements) {
		System.out.print(count++ + "\t" + url);
		Elements statsElementValues = statsElement.getElementsByTag("strong");
		int i = 1;
		for (Element statsElementValue : statsElementValues) {
			if (i > 2) {
				break;
			}
			System.out.print("\t" + statsElementValue.getElementsByTag("strong").text());
			i++;
		}
	}

	Elements activityElements = doc.getElementsByClass("QuestionLastActivityTime");
	if (activityElements.size() > 0) {
		for (Element activityElement : activityElements) {
			String lastAsked = StringUtils.remove(activityElement.text(), "Last asked: ");
			if (!lastAsked.contains("201")) {
				lastAsked = lastAsked + ", 2016";
			}
			System.out.println("\t" + lastAsked);
		}
	} else {
		System.out.println();
	}

	if (crawlRelated) {
		// log("crawlRelated is true for : " + url);
		Elements relatedQuestions = doc.getElementsByClass("question_related");
		List<Element> relatedQuestionList = new ArrayList<>();
		for (Element relatedQuestion : relatedQuestions) {
			Elements relatedQuestionHrefs = relatedQuestion.select("a[href]");
			for (Element element : relatedQuestionHrefs) {
				relatedQuestionList.add(element);
			}
		}

		relatedQuestionList = Utils.getSublist(relatedQuestionList, QuoraConstants.numReLatedQuestion);
		for (Element relatedQuestionHref : relatedQuestionList) {
			String relatedUrl = relatedQuestionHref.absUrl("href");
			if (!uniqueQuoraLinks.contains(relatedUrl)) {
				crawlQuoraPage(relatedUrl, false);
			}
		}

	}
}