Java LanguageIdentifier.getLanguage方法代码示例

import org.apache.tika.language.LanguageIdentifier; //导入方法依赖的package包/类
@Override
protected List<DetectedLanguage> detectLanguage(SolrInputDocument doc) {
  List<DetectedLanguage> languages = new ArrayList<>();
  String content = concatFields(doc);
  if (content.length() != 0) {
    LanguageIdentifier identifier = new LanguageIdentifier(content);
    // FIXME: Hack - we get the distance from toString and calculate our own certainty score
    Double distance = Double.parseDouble(tikaSimilarityPattern.matcher(identifier.toString()).replaceFirst("$1"));
    // This formula gives: 0.02 => 0.8, 0.1 => 0.5 which is a better sweetspot than isReasonablyCertain()
    Double certainty = 1 - (5 * distance); 
    certainty = (certainty < 0) ? 0 : certainty;
    DetectedLanguage language = new DetectedLanguage(identifier.getLanguage(), certainty);
    languages.add(language);
    log.debug("Language detected as "+language+" with a certainty of "+language.getCertainty()+" (Tika distance="+identifier.toString()+")");
  } else {
    log.debug("No input text to detect language from, returning empty list");
  }
  return languages;
}
 

import org.apache.tika.language.LanguageIdentifier; //导入方法依赖的package包/类
@Override
protected List<DetectedLanguage> detectLanguage(String content) {
  List<DetectedLanguage> languages = new ArrayList<DetectedLanguage>();
  if(content.trim().length() != 0) { 
    LanguageIdentifier identifier = new LanguageIdentifier(content);
    // FIXME: Hack - we get the distance from toString and calculate our own certainty score
    Double distance = Double.parseDouble(tikaSimilarityPattern.matcher(identifier.toString()).replaceFirst("$1"));
    // This formula gives: 0.02 => 0.8, 0.1 => 0.5 which is a better sweetspot than isReasonablyCertain()
    Double certainty = 1 - (5 * distance); 
    certainty = (certainty < 0) ? 0 : certainty;
    DetectedLanguage language = new DetectedLanguage(identifier.getLanguage(), certainty);
    languages.add(language);
    log.debug("Language detected as "+language+" with a certainty of "+language.getCertainty()+" (Tika distance="+identifier.toString()+")");
  } else {
    log.debug("No input text to detect language from, returning empty list");
  }
  return languages;
}
 

import org.apache.tika.language.LanguageIdentifier; //导入方法依赖的package包/类
/** Use statistical language identification to extract page language */
private String identifyLanguage(Parse parse) {
  StringBuilder text = new StringBuilder();
  if (parse == null)
    return null;

  String title = parse.getData().getTitle();
  if (title != null) {
    text.append(title.toString());
  }

  String content = parse.getText();
  if (content != null) {
    text.append(" ").append(content.toString());
  }

  // trim content?
  String titleandcontent = text.toString();

  if (this.contentMaxlength != -1
      && titleandcontent.length() > this.contentMaxlength)
    titleandcontent = titleandcontent.substring(0, contentMaxlength);

  LanguageIdentifier identifier = new LanguageIdentifier(titleandcontent);

  if (onlyCertain) {
    if (identifier.isReasonablyCertain())
      return identifier.getLanguage();
    else
      return null;
  }
  return identifier.getLanguage();
}
 

import org.apache.tika.language.LanguageIdentifier; //导入方法依赖的package包/类
/**
 * Cerca di identificare la lingua in cui è scritto un testo (attraverso
 * TIKA). In caso di indecisione ritorna Italiano
 *
 * @param text testo di cui identificare la lingua
 * @return lingua del documento
 */
public String getLanguageFromText(String text) {
    LanguageIdentifier identifier = new LanguageIdentifier(text);
    String lang = identifier.getLanguage();
    if (!MyAnalyzer.languagesSet.contains(lang)) {
        lang = "it";
    }
    return (String) intern.intern(lang);
}
 

import org.apache.tika.language.LanguageIdentifier; //导入方法依赖的package包/类
/** Use statistical language identification to extract page language */
private String identifyLanguage(Parse parse) {
    StringBuilder text = new StringBuilder();
    if (parse == null)
        return null;

    String title = parse.getData().getTitle();
    if (title != null) {
        text.append(title.toString());
    }

    String content = parse.getText();
    if (content != null) {
        text.append(" ").append(content.toString());
    }

    // trim content?
    String titleandcontent = text.toString();

    if (this.contentMaxlength != -1
            && titleandcontent.length() > this.contentMaxlength)
        titleandcontent = titleandcontent.substring(0, contentMaxlength);

    LanguageIdentifier identifier = new LanguageIdentifier(titleandcontent);

    if (onlyCertain) {
        if (identifier.isReasonablyCertain())
            return identifier.getLanguage();
        else
            return null;
    }
    return identifier.getLanguage();
}
 

import org.apache.tika.language.LanguageIdentifier; //导入方法依赖的package包/类
public static void main(String[] args) {
    String content = "This is an example string of text that will be detected as English.";

    LanguageIdentifier identifier = new LanguageIdentifier(content);
    String language = identifier.getLanguage();

    System.out.println("The String is in " + language + ".");
}
 

import org.apache.tika.language.LanguageIdentifier; //导入方法依赖的package包/类
/**
 * <p>Can return the wrong language; defaults to "en"</p>
 * <p>NOTE: we may want to simply truncate the string to some
 * short max length.</p>
 * @param string
 * @return
 */
public static String identifyLanguage(String string) {
	if (string == null)
		return "en";
	LanguageIdentifier identifier = new LanguageIdentifier(trimString(string));
   	String language = identifier.getLanguage();
   	boolean isReasonablyCertain = identifier.isReasonablyCertain();
   	if (language.equals("") || !isReasonablyCertain)
   		language = "en"; //defaults
   	return language;
}
 

import org.apache.tika.language.LanguageIdentifier; //导入方法依赖的package包/类
@Override
public void execute(IngestDocument ingestDocument) {
    Map<String, Object> additionalFields = new HashMap<>();

    byte[] input = ingestDocument.getFieldValueAsBytes(field, ignoreMissing);

    if (input == null && ignoreMissing) {
        return;
    } else if (input == null) {
        throw new IllegalArgumentException("field [" + field + "] is null, cannot parse.");
    }

    try {
        Metadata metadata = new Metadata();
        String parsedContent = TikaImpl.parse(input, metadata, indexedChars);

        if (properties.contains(Property.CONTENT) && Strings.hasLength(parsedContent)) {
            // somehow tika seems to append a newline at the end automatically, lets remove that again
            additionalFields.put(Property.CONTENT.toLowerCase(), parsedContent.trim());
        }

        if (properties.contains(Property.LANGUAGE) && Strings.hasLength(parsedContent)) {
            LanguageIdentifier identifier = new LanguageIdentifier(parsedContent);
            String language = identifier.getLanguage();
            additionalFields.put(Property.LANGUAGE.toLowerCase(), language);
        }

        if (properties.contains(Property.DATE)) {
            String createdDate = metadata.get(TikaCoreProperties.CREATED);
            if (createdDate != null) {
                additionalFields.put(Property.DATE.toLowerCase(), createdDate);
            }
        }

        if (properties.contains(Property.TITLE)) {
            String title = metadata.get(TikaCoreProperties.TITLE);
            if (Strings.hasLength(title)) {
                additionalFields.put(Property.TITLE.toLowerCase(), title);
            }
        }

        if (properties.contains(Property.AUTHOR)) {
            String author = metadata.get("Author");
            if (Strings.hasLength(author)) {
                additionalFields.put(Property.AUTHOR.toLowerCase(), author);
            }
        }

        if (properties.contains(Property.KEYWORDS)) {
            String keywords = metadata.get("Keywords");
            if (Strings.hasLength(keywords)) {
                additionalFields.put(Property.KEYWORDS.toLowerCase(), keywords);
            }
        }

        if (properties.contains(Property.CONTENT_TYPE)) {
            String contentType = metadata.get(Metadata.CONTENT_TYPE);
            if (Strings.hasLength(contentType)) {
                additionalFields.put(Property.CONTENT_TYPE.toLowerCase(), contentType);
            }
        }

        if (properties.contains(Property.CONTENT_LENGTH)) {
            String contentLength = metadata.get(Metadata.CONTENT_LENGTH);
            long length;
            if (Strings.hasLength(contentLength)) {
                length = Long.parseLong(contentLength);
            } else {
                length = parsedContent.length();
            }
            additionalFields.put(Property.CONTENT_LENGTH.toLowerCase(), length);
        }
    } catch (Exception e) {
        throw new ElasticsearchParseException("Error parsing document in field [{}]", e, field);
    }

    ingestDocument.setFieldValue(targetField, additionalFields);
}
 

import org.apache.tika.language.LanguageIdentifier; //导入方法依赖的package包/类
/**
 * identify language of text
 * @param text to check
 * @return language detected
 */
protected String identifyLanguage(String text) {
	LanguageIdentifier identifier = new LanguageIdentifier(text);
	return identifier.getLanguage();
}