Java Normalizer.normalize方法代码示例

import java.text.Normalizer; //导入方法依赖的package包/类
public void setWaarde(final String waarde) {
    this.waarde = waarde;
    if (waarde == null) {
        this.slimZoekenWaarde = null;
    } else if (waarde.startsWith("\\")) {
        this.exact = true;
        this.slimZoekenWaarde = waarde.substring(1);
    } else if (waarde.endsWith("*")) {
        this.wildcard = true;
        this.slimZoekenWaarde = waarde.substring(0, waarde.length() - 1);
    } else {
        this.slimZoekenWaarde = waarde;
    }

    if (waarde != null && !this.exact) {
        if (!waarde.matches(".*[A-Z].*") && attribuut.isString()) {
            this.caseInsensitive = true;
        }
        String normalizedWaarde = Normalizer.normalize(waarde, Normalizer.Form.NFD);
        Pattern pattern = Pattern.compile("\\p{InCombiningDiacriticalMarks}+");
        if (pattern.matcher(normalizedWaarde).find()) {
            this.diakriet = true;
        }
    }
}
 

import java.text.Normalizer; //导入方法依赖的package包/类
/**
 * Encodes a string containing non ASCII characters using an UTF-8 encoder.
 * 
 * @param s
 *            The string the encode (assuming ASCII characters only)
 */
private static String minimalEncode_UTF8(String s) {
	// TODO: Normalizer requires Java 6!
	String n = (Normalizer.isNormalized(s, Form.NFKC)) ? s : Normalizer.normalize(s, Form.NFKC);
	// convert String to UTF-8
	ByteBuffer bb = UTF8.encode(n);
	// URI encode
	StringBuffer sb = new StringBuffer();

	while (bb.hasRemaining()) {
		int b = bb.get() & 0xff;

		if (isLegal(b)) {
			sb.append((char) b);
		} else {
			appendEscape(sb, (byte) b);
		}
	}
	return sb.toString();
}
 

import java.text.Normalizer; //导入方法依赖的package包/类
/**
 * The pattern is converted to normalizedD form and then a pure group
 * is constructed to match canonical equivalences of the characters.
 */
private void normalize() {
    boolean inCharClass = false;
    int lastCodePoint = -1;

    // Convert pattern into normalizedD form
    normalizedPattern = Normalizer.normalize(pattern, Normalizer.Form.NFD);
    patternLength = normalizedPattern.length();

    // Modify pattern to match canonical equivalences
    StringBuilder newPattern = new StringBuilder(patternLength);
    for(int i=0; i<patternLength; ) {
        int c = normalizedPattern.codePointAt(i);
        StringBuilder sequenceBuffer;
        if ((Character.getType(c) == Character.NON_SPACING_MARK)
            && (lastCodePoint != -1)) {
            sequenceBuffer = new StringBuilder();
            sequenceBuffer.appendCodePoint(lastCodePoint);
            sequenceBuffer.appendCodePoint(c);
            while(Character.getType(c) == Character.NON_SPACING_MARK) {
                i += Character.charCount(c);
                if (i >= patternLength)
                    break;
                c = normalizedPattern.codePointAt(i);
                sequenceBuffer.appendCodePoint(c);
            }
            String ea = produceEquivalentAlternation(
                                           sequenceBuffer.toString());
            newPattern.setLength(newPattern.length()-Character.charCount(lastCodePoint));
            newPattern.append("(?:").append(ea).append(")");
        } else if (c == '[' && lastCodePoint != '\\') {
            i = normalizeCharClass(newPattern, i);
        } else {
            newPattern.appendCodePoint(c);
        }
        lastCodePoint = c;
        i += Character.charCount(c);
    }
    normalizedPattern = newPattern.toString();
}
 

import java.text.Normalizer; //导入方法依赖的package包/类
public int getTweetLength(String text) {
  text = Normalizer.normalize(text, Normalizer.Form.NFC);
  int length = text.codePointCount(0, text.length());

  for (Extractor.Entity urlEntity : extractor.extractURLsWithIndices(text)) {
    length += urlEntity.start - urlEntity.end;
    length += urlEntity.value.toLowerCase().startsWith("https://") ? shortUrlLengthHttps : shortUrlLength;
  }

  return length;
}
 

import java.text.Normalizer; //导入方法依赖的package包/类
public void setFullTag(String fullTag) {
    if (fullTag != null) {
        fullTag = fullTag.toLowerCase();
        fullTag = Normalizer.normalize(fullTag, Normalizer.Form.NFD);
        fullTag = fullTag.replaceAll("[\\p{InCombiningDiacriticalMarks}]", "");
        fullTag = fullTag.replaceAll("[^a-z0-9-]", "");
    }
    this.fullTag = fullTag;
}
 

import java.text.Normalizer; //导入方法依赖的package包/类
/**
 * Attempts to compose input by combining the first character
 * with the first combining mark following it. Returns a String
 * that is the composition of the leading character with its first
 * combining mark followed by the remaining combining marks. Returns
 * null if the first two characters cannot be further composed.
 */
private static String composeOneStep(String input) {
    int len = countChars(input, 0, 2);
    String firstTwoCharacters = input.substring(0, len);
    String result = Normalizer.normalize(firstTwoCharacters, Normalizer.Form.NFC);
    if (result.equals(firstTwoCharacters))
        return null;
    else {
        String remainder = input.substring(len);
        return result + remainder;
    }
}
 

import java.text.Normalizer; //导入方法依赖的package包/类
protected boolean isPathNameCompatible(Path p, String simpleName, Kind kind) {
    Objects.requireNonNull(simpleName);
    Objects.requireNonNull(kind);

    if (kind == Kind.OTHER && BaseFileManager.getKind(p) != kind) {
        return false;
    }

    String sn = simpleName + kind.extension;
    String pn = p.getFileName().toString();
    if (pn.equals(sn)) {
        return true;
    }

    if (p.getFileSystem() == defaultFileSystem) {
        if (isMacOS) {
            if (Normalizer.isNormalized(pn, Normalizer.Form.NFD)
                    && Normalizer.isNormalized(sn, Normalizer.Form.NFC)) {
                // On Mac OS X it is quite possible to have the file name and the
                // given simple name normalized in different ways.
                // In that case we have to normalize file name to the
                // Normal Form Composed (NFC).
                String normName = Normalizer.normalize(pn, Normalizer.Form.NFC);
                if (normName.equals(sn)) {
                    return true;
                }
            }
        }

        if (pn.equalsIgnoreCase(sn)) {
            try {
                // allow for Windows
                return p.toRealPath(LinkOption.NOFOLLOW_LINKS).getFileName().toString().equals(sn);
            } catch (IOException e) {
            }
        }
    }

    return false;
}
 

import java.text.Normalizer; //导入方法依赖的package包/类
/**
 * Attempts to compose input by combining the first character
 * with the first combining mark following it. Returns a String
 * that is the composition of the leading character with its first
 * combining mark followed by the remaining combining marks. Returns
 * null if the first two characters cannot be further composed.
 */
private String composeOneStep(String input) {
    int len = countChars(input, 0, 2);
    String firstTwoCharacters = input.substring(0, len);
    String result = Normalizer.normalize(firstTwoCharacters, Normalizer.Form.NFC);

    if (result.equals(firstTwoCharacters))
        return null;
    else {
        String remainder = input.substring(len);
        return result + remainder;
    }
}
 

import java.text.Normalizer; //导入方法依赖的package包/类
public static String makeSlug(String input, boolean transliterate) {
  String origInput = input;
  // Validate the input
  if (input == null) {
    ProjectLogger.log("Provided input value is null");
    return input;
  }
  // Remove extra spaces
  input = input.trim();
  // Remove URL encoding
  input = urlDecode(input);
  // If transliterate is required
  if (transliterate) {
    // Tranlisterate & cleanup
    String transliterated = transliterate(input);
    // transliterated = removeDuplicateChars(transliterated);
    input = transliterated;
  }
  // Replace all whitespace with dashes
  input = WHITESPACE.matcher(input).replaceAll("-");
  // Remove all accent chars
  input = Normalizer.normalize(input, Form.NFD);
  // Remove all non-latin special characters
  input = NONLATIN.matcher(input).replaceAll("");
  // Remove any consecutive dashes
  input = normalizeDashes(input);
  // Validate before returning
  validateResult(input, origInput);
  // Slug is always lowercase
  return input.toLowerCase(Locale.ENGLISH);
}
 

import java.text.Normalizer; //导入方法依赖的package包/类
public static String stripAccents(String s) {
    if (s != null) {
        s = Normalizer.normalize(s, Normalizer.Form.NFD);
        s = s.replaceAll("[^\\p{ASCII}]", "");
        return s;
    } else {
        return null;
    }
}
 

import java.text.Normalizer; //导入方法依赖的package包/类
/** package private, use {@link SearchPreprocessor} */
TvShowSearchInfo(Uri file, String showName, int season, int episode) {
    super(file);
    mShowName = Normalizer.normalize(showName, Normalizer.Form.NFC);
    mSeason = season;
    mEpisode = episode;
}
 

import java.text.Normalizer; //导入方法依赖的package包/类
/**
 * Loads the main class from the class path (LM_CLASS or LM_JAR).
 */
private static Class<?> loadMainClass(int mode, String what) {
    // get the class name
    String cn;
    switch (mode) {
        case LM_CLASS:
            cn = what;
            break;
        case LM_JAR:
            cn = getMainClassFromJar(what);
            break;
        default:
            // should never happen
            throw new InternalError("" + mode + ": Unknown launch mode");
    }

    // load the main class
    cn = cn.replace('/', '.');
    Class<?> mainClass = null;
    ClassLoader scl = ClassLoader.getSystemClassLoader();
    try {
        try {
            mainClass = Class.forName(cn, false, scl);
        } catch (NoClassDefFoundError | ClassNotFoundException cnfe) {
            if (System.getProperty("os.name", "").contains("OS X")
                    && Normalizer.isNormalized(cn, Normalizer.Form.NFD)) {
                try {
                    // On Mac OS X since all names with diacritical marks are
                    // given as decomposed it is possible that main class name
                    // comes incorrectly from the command line and we have
                    // to re-compose it
                    String ncn = Normalizer.normalize(cn, Normalizer.Form.NFC);
                    mainClass = Class.forName(ncn, false, scl);
                } catch (NoClassDefFoundError | ClassNotFoundException cnfe1) {
                    abort(cnfe1, "java.launcher.cls.error1", cn,
                            cnfe1.getClass().getCanonicalName(), cnfe1.getMessage());
                }
            } else {
                abort(cnfe, "java.launcher.cls.error1", cn,
                        cnfe.getClass().getCanonicalName(), cnfe.getMessage());
            }
        }
    } catch (LinkageError le) {
        abort(le, "java.launcher.cls.error6", cn,
                le.getClass().getName() + ": " + le.getLocalizedMessage());
    }
    return mainClass;
}
 

import java.text.Normalizer; //导入方法依赖的package包/类
/**
 * Returns the main class for a module. The query is either a module name
 * or module-name/main-class. For the former then the module's main class
 * is obtained from the module descriptor (MainClass attribute).
 */
private static Class<?> loadModuleMainClass(String what) {
    int i = what.indexOf('/');
    String mainModule;
    String mainClass;
    if (i == -1) {
        mainModule = what;
        mainClass = null;
    } else {
        mainModule = what.substring(0, i);
        mainClass = what.substring(i+1);
    }

    // main module is in the boot layer
    ModuleLayer layer = ModuleLayer.boot();
    Optional<Module> om = layer.findModule(mainModule);
    if (!om.isPresent()) {
        // should not happen
        throw new InternalError("Module " + mainModule + " not in boot Layer");
    }
    Module m = om.get();

    // get main class
    if (mainClass == null) {
        Optional<String> omc = m.getDescriptor().mainClass();
        if (!omc.isPresent()) {
            abort(null, "java.launcher.module.error1", mainModule);
        }
        mainClass = omc.get();
    }

    // load the class from the module
    Class<?> c = null;
    try {
        c = Class.forName(m, mainClass);
        if (c == null && System.getProperty("os.name", "").contains("OS X")
                && Normalizer.isNormalized(mainClass, Normalizer.Form.NFD)) {

            String cn = Normalizer.normalize(mainClass, Normalizer.Form.NFC);
            c = Class.forName(m, cn);
        }
    } catch (LinkageError le) {
        abort(null, "java.launcher.module.error3", mainClass, m.getName(),
                le.getClass().getName() + ": " + le.getLocalizedMessage());
    }
    if (c == null) {
        abort(null, "java.launcher.module.error2", mainClass, mainModule);
    }

    System.setProperty("jdk.module.main.class", c.getName());
    return c;
}
 

import java.text.Normalizer; //导入方法依赖的package包/类
/**
 * 过滤XSS注入
 *
 * @param value
 * @return
 */
public static String filterXSS(String value) {
    String cleanValue = null;
    if (value != null) {
        cleanValue = Normalizer.normalize(value, Normalizer.Form.NFD);
        // Avoid null characters
        cleanValue = cleanValue.replaceAll("\0", "");

        // Avoid anything between script tags
        Pattern scriptPattern = Pattern.compile("<script>(.*?)</script>", Pattern.CASE_INSENSITIVE);
        cleanValue = scriptPattern.matcher(cleanValue).replaceAll("");

        // Avoid anything in a src='...' type of expression
        scriptPattern = Pattern.compile("src[\r\n]*=[\r\n]*\\\'(.*?)\\\'", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL);
        cleanValue = scriptPattern.matcher(cleanValue).replaceAll("");

        scriptPattern = Pattern.compile("src[\r\n]*=[\r\n]*\\\"(.*?)\\\"", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL);
        cleanValue = scriptPattern.matcher(cleanValue).replaceAll("");

        // Remove any lonesome </script> tag
        scriptPattern = Pattern.compile("</script>", Pattern.CASE_INSENSITIVE);
        cleanValue = scriptPattern.matcher(cleanValue).replaceAll("");

        // Remove any lonesome <script ...> tag
        scriptPattern = Pattern.compile("<script(.*?)>", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL);
        cleanValue = scriptPattern.matcher(cleanValue).replaceAll("");

        // Avoid eval(...) expressions
        scriptPattern = Pattern.compile("eval\\((.*?)\\)", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL);
        cleanValue = scriptPattern.matcher(cleanValue).replaceAll("");

        // Avoid expression(...) expressions
        scriptPattern = Pattern.compile("expression\\((.*?)\\)", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL);
        cleanValue = scriptPattern.matcher(cleanValue).replaceAll("");

        // Avoid javascript:... expressions
        scriptPattern = Pattern.compile("javascript:", Pattern.CASE_INSENSITIVE);
        cleanValue = scriptPattern.matcher(cleanValue).replaceAll("");

        // Avoid vbscript:... expressions
        scriptPattern = Pattern.compile("vbscript:", Pattern.CASE_INSENSITIVE);
        cleanValue = scriptPattern.matcher(cleanValue).replaceAll("");

        // Avoid onload= expressions
        scriptPattern = Pattern.compile("onload(.*?)=", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL);
        cleanValue = scriptPattern.matcher(cleanValue).replaceAll("");
    }
    return cleanValue;
}
 

import java.text.Normalizer; //导入方法依赖的package包/类
private static void normalizeSlice(String src, int off, int limit,
                                   StringBuilder dst)
{
    int len = src.length();
    int off0 = off;
    while (off < limit && ASCII.isAscii(src.charAt(off))) {
        off++;
    }
    if (off == limit) {
        dst.append(src, off0, limit);
        return;
    }
    off--;
    if (off < off0)
        off = off0;
    else
        dst.append(src, off0, off);
    while (off < limit) {
        int ch0 = src.codePointAt(off);
        if (".$|()[]{}^?*+\\".indexOf(ch0) != -1) {
            dst.append((char)ch0);
            off++;
            continue;
        }
        int j = off + Character.charCount(ch0);
        int ch1;
        while (j < limit) {
            ch1 = src.codePointAt(j);
            if (Grapheme.isBoundary(ch0, ch1))
                break;
            ch0 = ch1;
            j += Character.charCount(ch1);
        }
        String seq = src.substring(off, j);
        String nfd = Normalizer.normalize(seq, Normalizer.Form.NFD);
        off = j;
        if (nfd.length() > 1) {
            ch0 = nfd.codePointAt(0);
            ch1 = nfd.codePointAt(Character.charCount(ch0));
            if (Character.getType(ch1) == Character.NON_SPACING_MARK) {
                Set<String> altns = new LinkedHashSet<>();
                altns.add(seq);
                produceEquivalentAlternation(nfd, altns);
                dst.append("(?:");
                altns.forEach( s -> dst.append(s).append('|'));
                dst.delete(dst.length() - 1, dst.length());
                dst.append(")");
                continue;
            }
        }
        String nfc = Normalizer.normalize(seq, Normalizer.Form.NFC);
        if (!seq.equals(nfc) && !nfd.equals(nfc))
            dst.append("(?:" + seq + "|" + nfd  + "|" + nfc + ")");
        else if (!seq.equals(nfd))
            dst.append("(?:" + seq + "|" + nfd + ")");
        else
            dst.append(seq);
    }
}