Java Canonicalizer类代码示例

import org.netpreserve.urlcanon.Canonicalizer; //导入依赖的package包/类
static String fixupUrl(String raw) {
    ParsedUrl url = ParsedUrl.parseUrl(raw);
    Canonicalizer.WHATWG.canonicalize(url);

    // early versions of httrack wrote the URL without a scheme
    if (url.getScheme().isEmpty()) {
        url.setScheme(new ByteString("http"));
        url.setColonAfterScheme(new ByteString(":"));
        url.setSlashes(new ByteString("//"));
    }

    // httrack incorrectly makes requests including the fragment. Should we fix clear them?
    //url.setHashSign(ByteString.EMPTY);
    //url.setFragment(ByteString.EMPTY);

    return url.toString();
}
 

import org.netpreserve.urlcanon.Canonicalizer; //导入依赖的package包/类
/**
 * Canonicalize and return the SURT form.
 *
 * - perform WHATWG canonicalization
 * - lowercase the path
 * - remove the fragment
 * - remove www. prefix from hostname
 * - replace https scheme with http
 *
 * These rules are a little aggressive to make defining rules less error prone.
 *
 * TODO: query string?
 *
 * TODO: reconcile this with UrlCanonicalizer. We should probably switch over to urlcanon as its a more robust
 * canonicalizer but a change will require rebuilding the index. Maybe keep both implementations and allow an
 * offline upgrade to be run?
 */
static String canonSsurt(String url) {
    if (FeatureFlags.pandoraHacks()) {
        /*
         * Strip PANDORA prefix from URLs so rules so a single rule can match both PANDORA and non-PANDORA
         * content.
         */
        Matcher m = PANDORA_REGEX.matcher(url);
        if (m.matches()) {
            String hackedOffUrl = "http://" + m.group(1);
            url = hackedOffUrl;
        }
    }

    ParsedUrl parsed = ParsedUrl.parseUrl(url);
    Canonicalizer.AGGRESSIVE.canonicalize(parsed);
    return parsed.ssurt().toString();
}
 

import org.netpreserve.urlcanon.Canonicalizer; //导入依赖的package包/类
public static void setUrls(Document doc, String url) throws TextExtractionException {
    String deliveryUrl = url;
    Matcher m = PANDORA_REGEX.matcher(url);
    if (m.matches()) {
        // TODO: consult url.map
        String hackedOffUrl = "http://" + m.group(1);
        url = hackedOffUrl;
    }
    doc.setUrl(url);
    ParsedUrl parse = ParsedUrl.parseUrl(deliveryUrl);
    Canonicalizer.AGGRESSIVE.canonicalize(parse);
    doc.setDeliveryUrl(parse.toString());

    try {
        doc.setHost(new URL(url).getHost());
        doc.setSite(topPrivateDomain(url));
    } catch (MalformedURLException e) {
        throw new TextExtractionException(e);
    }
}
 

import org.netpreserve.urlcanon.Canonicalizer; //导入依赖的package包/类
/**
 * For a url convert into a search url that should match with the way it is normalised for delivery.
 * @param url
 * @return
 */
public static String getSearchUrl(String url){
	ParsedUrl parsed = ParsedUrl.parseUrl(url);
	if(parsed.getScheme().isEmpty()){
		// default to http as this is needed to force the host to be detected
		parsed = ParsedUrl.parseUrl("http://"+url);
	}
	Canonicalizer.WHATWG.canonicalize(parsed);
	parsed.setPath(parsed.getPath().asciiLowerCase());
	parsed.setHost(parsed.getHost().replaceAll(WWW_PREFIX, ""));
	String ret = parsed.getHost().toString() + parsed.getPath().toString();
	return ret;
}
 

import org.netpreserve.urlcanon.Canonicalizer; //导入依赖的package包/类
private static String canonSsurt(String url) {
  ParsedUrl parsed = ParsedUrl.parseUrl(url);
  Canonicalizer.WHATWG.canonicalize(parsed);
  parsed.setPath(parsed.getPath().asciiLowerCase());
  parsed.setFragment(ByteString.EMPTY);
  parsed.setHashSign(ByteString.EMPTY);
  parsed.setHost(parsed.getHost().replaceAll(WWW_PREFIX, ""));
  if (parsed.getScheme().toString().equals("https")) {
    parsed.setScheme(new ByteString("http"));
  }
  return parsed.ssurt().toString();
}
 

import org.netpreserve.urlcanon.Canonicalizer; //导入依赖的package包/类
private String canonUrl(String input) {
  ParsedUrl url = ParsedUrl.parseUrl(input);
  Canonicalizer.AGGRESSIVE.canonicalize(url);
  return url.toString();
}
 

import org.netpreserve.urlcanon.Canonicalizer; //导入依赖的package包/类
static String cleanUrl(String url) {
    ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
    Canonicalizer.WHATWG.canonicalize(parsedUrl);
    return parsedUrl.toString().replace(" ", "%20").replace("\r", "%0a").replace("\n", "%0d");
}