Java CharsetDetector.detect方法代码示例

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
public static CharsetMatch checkCharset(InputStream input) {
	//		BufferedInputStream bis = new BufferedInputStream(input);
	CharsetDetector cd = new CharsetDetector();
	try {
		cd.setText(input);
	} catch (IOException e) {
		try {
			input.close();
		} catch (IOException e1) {
			e1.printStackTrace();
		}
		e.printStackTrace();
	}
	CharsetMatch cm = cd.detect();

	//		if (cm != null) {
	//			//reader = cm.getReader();
	//			return cm.getName();
	//		} else {
	//			throw new UnsupportedCharsetException(null);
	//		}
	return cm;
}
 

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
/**
     * 利用 icu4j 探测输入流编码，只能探测文本类型的输入流
     * -
     * 抛弃 juniversalchardet
     *
     * @param in
     * @return
     * @throws IOException
     */
    public static Charset detectEncoding(InputStream in) throws IOException {
        final CharsetDetector detector = new CharsetDetector();
        detector.setText(in);

        final CharsetMatch charsetMatch = detector.detect();
        if (charsetMatch == null) {
            log.info("Cannot detect source charset.");
            return null;
        }
        //This is an integer from 0 to 100. The higher the value, the more confidence
        //探测的相似度在 1~100 之间，相似度越高结果越准确。
        int confidence = charsetMatch.getConfidence();
        final String name = charsetMatch.getName();
        log.info("CharsetMatch: {} ({}% 相似度，相似度小于 50% 时，可能编码无法判断。)", name, confidence);
        //打印该文本编码，所有可能性
//        CharsetMatch[] matches = detector.detectAll();
//        System.out.println("All possibilities : " + Arrays.asList(matches));
        return Charset.forName(name);
    }
 

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
/**
 * Extract text to be indexed
 */
public static String getText(String mimeType, String encoding, InputStream isContent) throws IOException {
	BufferedInputStream bis = new BufferedInputStream(isContent);
	TextExtractor te = engine.get(mimeType);
	String text = null;

	if (te != null) {
		if (mimeType.startsWith("text/") && encoding == null) {
			CharsetDetector detector = new CharsetDetector();
			detector.setText(bis);
			CharsetMatch cm = detector.detect();
			encoding = cm.getName();
		}

		text = te.extractText(bis, mimeType, encoding);
	} else {
		throw new IOException("Full text indexing of '" + mimeType + "' is not supported");
	}


	IOUtils.closeQuietly(bis);
	return text;
}
 

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
/**
 * Read a text file detecting encoding using http://userguide.icu-project.org/conversion/detection
 * Return the file contents as a String.
 */
public static String fileAnyEncodingToString(File f) throws IOException {

  byte[] byteData = IOUtils.toByteArray(new FileInputStream(f));

  CharsetDetector detector = new CharsetDetector();

  String unicodeData = detector.getString(byteData, null);
  // Add to newline at the end of the file otherwise the subtitle parser library can get confused by EOF
  unicodeData += System.getProperty("line.separator") + System.getProperty("line.separator");
  CharsetMatch match = detector.detect();
  if (match != null && match.getConfidence() > 60) {
    LOGGER.debug("{} has a detected encoding: {}", f.getName(), match.getName());
    if (match.getLanguage() != null) {
      LOGGER.debug("{} has a detected language: {}", f.getName(), match.getLanguage());
    }
  }
  return unicodeData;
}
 

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
private Charset guessCharset(Path file, Charset charset) throws IOException {

        CharsetDetector detector = new CharsetDetector();
        byte[] data;

        try (SeekableByteChannel byteChannel = Files.newByteChannel(file, StandardOpenOption.READ)) {
            long size = byteChannel.size();

            if (size >= Integer.MAX_VALUE) {
                return guessCharsetChardet(file, charset);
            }

            int smallsize = (int) size;
            ByteBuffer buffer = ByteBuffer.allocate(smallsize);
            byteChannel.read(buffer);
            data = buffer.array();
        }

        detector.setText(data);
        CharsetMatch match = detector.detect();

        return Charset.forName(match.getName());
    }
 

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
public Encoding sniff() throws IOException {
    try {
        CharsetDetector detector = new CharsetDetector();
        detector.setText(this);
        CharsetMatch match = detector.detect();
        Encoding enc = Encoding.forName(match.getName());
        Encoding actual = enc.getActualHtmlEncoding();
        if (actual != null) {
            enc = actual;
        }
        if (enc != Encoding.WINDOWS1252 && enc.isAsciiSuperset()) {
            return enc;
        } else {
            return null;
        }
    } catch (Exception e) {
        return null;
    }
}
 

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
@Override
protected void parseContent(StreamLimiter streamLimiter, LanguageEnum lang)
		throws IOException {
	CharsetDetector detector = new CharsetDetector();
	BufferedInputStream bis = null;
	try {
		bis = new BufferedInputStream(streamLimiter.getNewInputStream());
		detector.setText(bis);
		CharsetMatch match = detector.detect();
		String content;
		if (match != null)
			content = match.getString();
		else
			content = IOUtils.toString(streamLimiter.getNewInputStream(), "UTF-8");
		ParserResultItem result = getNewParserResultItem();
		result.addField(ParserFieldEnum.content, content);
		result.langDetection(10000, ParserFieldEnum.content);
	} finally {
		IOUtils.close(bis);
	}
}
 

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
/**
 * Use a third party library as last resort to guess the charset from the
 * bytes.
 */
private static String getCharsetFromText(byte[] content,
        String declaredCharset, int maxLengthCharsetDetection) {
    String charset = null;
    // filter HTML tags
    CharsetDetector charsetDetector = new CharsetDetector();
    charsetDetector.enableInputFilter(true);
    // give it a hint
    if (declaredCharset != null)
        charsetDetector.setDeclaredEncoding(declaredCharset);
    // trim the content of the text for the detection
    byte[] subContent = content;
    if (maxLengthCharsetDetection != -1
            && content.length > maxLengthCharsetDetection) {
        subContent = Arrays.copyOfRange(content, 0,
                maxLengthCharsetDetection);
    }
    charsetDetector.setText(subContent);
    try {
        CharsetMatch charsetMatch = charsetDetector.detect();
        charset = validateCharset(charsetMatch.getName());
    } catch (Exception e) {
        charset = null;
    }
    return charset;
}
 

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
public static Reader toReader(InputStream input) throws IOException {
	if (!input.markSupported())
		input = new BufferedInputStream(input);
	
	CharsetDetector charsetDetector = new CharsetDetector();
	charsetDetector.setText(input);
	
	CharsetMatch m = charsetDetector.detect();
	
	Reader reader;
	if (m.getConfidence() > 50) {
		reader = m.getReader();
	} else {
		reader = new InputStreamReader(input);
	}
	return reader;
}
 

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
/**
 * Returns the detected encoding of the given byte array.
 *
 * @param input The data to detect the encoding for.
 * @param assume88591IfNotUtf8 True to assume that the encoding is ISO-8859-1 (the standard
 *     encoding for HTTP) if the bytes are not valid UTF-8. Only recommended if you can reasonably
 *     expect that other encodings are going to be specified. Full encoding detection is very
 *     expensive!
 * @return The detected encoding.
 */
public static Charset detectEncoding(byte[] input, boolean assume88591IfNotUtf8) {
  if (looksLikeValidUtf8(input)) {
    return UTF_8;
  }

  if (assume88591IfNotUtf8) {
    return ISO_8859_1;
  }

  // Fall back to the incredibly slow ICU. It might be better to just skip this entirely.
  CharsetDetector detector = new CharsetDetector();
  detector.setText(input);
  CharsetMatch match = detector.detect();
  return Charset.forName(match.getName().toUpperCase());
}
 

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
/**
 * Uses ICU4J to determine the charset of the given InputStream.
 * 
 * @param input
 * @return Detected charset name; null if not detected.
 * @throws IOException
 * @should detect charset correctly
 */
public static String getCharset(InputStream input) throws IOException {
    CharsetDetector cd = new CharsetDetector();
    try (BufferedInputStream bis = new BufferedInputStream(input)) {
        cd.setText(bis);
        CharsetMatch cm = cd.detect();
        if (cm != null) {
            return cm.getName();
        }
    }

    return null;
}
 

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
private String ibmICU4j(byte[] bytes) {
	CharsetDetector charsetDetector = new CharsetDetector();
	charsetDetector.setText(bytes);
	CharsetMatch charsetMatch = charsetDetector.detect();
	String charset = charsetMatch.getName();
	return charset;
}
 

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
protected String suggestEncoding(final byte[] bytes) {
    final CharsetDetector cd = new CharsetDetector();
    cd.setText(bytes);

    final CharsetMatch charsetMatch = cd.detect();
    final String charSet = charsetMatch.getName();

    final int confidence = charsetMatch.getConfidence();
    logger.info("CharsetMatch: {} ({}% confidence)", charSet, confidence);
    return charSet;
}
 

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
public String autoDetectEncoding(final byte[] bytes) {
    final CharsetDetector cd = new CharsetDetector();
    cd.setText(bytes);
    final CharsetMatch charsetMatch = cd.detect();
    final String charSet = charsetMatch.getName();

    final int confidence = charsetMatch.getConfidence();
    logger.info("CharsetMatch: {} ({}% confidence)", charSet, confidence);
    setSelectedItem(charSet);
    return charSet;
}
 

import com.ibm.icu.text.CharsetDetector; //导入方法依赖的package包/类
private BufferedReader readFile(File file)
		throws UnsupportedEncodingException, FileNotFoundException {
	// InputStreamReader isr = new InputStreamReader(new
	// FileInputStream(file));
	// System.out.println(isr.getEncoding());
	// System.exit(0);
	String charSet = "UTF-8";

	final BufferedInputStream bis = new BufferedInputStream(
			new FileInputStream(file));
	final CharsetDetector cd = new CharsetDetector();
	try {
		cd.setText(bis);
		final CharsetMatch cm = cd.detect();

		if (cm != null && cm.getName() == "ISO-8859-1") {
			charSet = cm.getName();
		}
	} catch (final IOException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	// System.out.println(charSet);
	// String charSet = "UTF-8"; //"ISO-8859-1";
	final BufferedReader bf = new BufferedReader(new InputStreamReader(
			new FileInputStream(file), charSet));

	return bf;

}