Java CommonTextObjectFactories.forDetectingOnLargeText方法代码示例

import com.optimaize.langdetect.text.CommonTextObjectFactories; //导入方法依赖的package包/类
private void initLanguageDetector() throws IOException {
    List<LanguageProfile> languageProfiles = new LanguageProfileReader().readAllBuiltIn();
    languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
            .withProfiles(languageProfiles)
            .build();
    textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
}
 

import com.optimaize.langdetect.text.CommonTextObjectFactories; //导入方法依赖的package包/类
public QALanguageDetector() throws IOException {
	languageProfiles = new LanguageProfileReader().readAllBuiltIn();

	//build language detector:
	languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
		.withProfiles(languageProfiles)
		.build();

	//create a text object factory
	textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
}
 

import com.optimaize.langdetect.text.CommonTextObjectFactories; //导入方法依赖的package包/类
@Override
public void doInitialize(UimaContext aContext) throws ResourceInitializationException {
	try{
		List<LanguageProfile> languageProfiles = new LanguageProfileReader().readAllBuiltIn();
		languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
				.withProfiles(languageProfiles)
				.build();
		
		textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
	}catch(IOException ioe){
		throw new ResourceInitializationException(ioe);
	}
}
 

import com.optimaize.langdetect.text.CommonTextObjectFactories; //导入方法依赖的package包/类
/**
 * Language detection test for each file (--detectlang option)
 * 
 * <pre>
 * usage: --detectlang -d [profile directory] -a [alpha] -s [seed] [test file(s)]
 * </pre>
 * 
 */
public void detectLang() throws IOException {
    LanguageDetector languageDetector = makeDetector();
    TextObjectFactory textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();

    for (String filename: arglist) {
        try (BufferedReader is = new BufferedReader(new InputStreamReader(new FileInputStream(filename), "utf-8"))) {
            TextObject textObject = textObjectFactory.create().append(is);
            List<DetectedLanguage> probabilities = languageDetector.getProbabilities(textObject);
            System.out.println(filename + ":" + probabilities);
        }
    }
}
 

import com.optimaize.langdetect.text.CommonTextObjectFactories; //导入方法依赖的package包/类
public LanguageIdentifier() {
  try {
    List<LanguageProfile> profiles = loadProfiles(getLanguageCodes());
    languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
            .minimalConfidence(MINIMAL_CONFIDENCE)
            .withProfiles(profiles)
            .build();
    textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
  } catch (IOException e) {
    throw new RuntimeException("Could not set up language identifier", e);
  }
}
 

import com.optimaize.langdetect.text.CommonTextObjectFactories; //导入方法依赖的package包/类
private void lazyInitialize() {
    // lazy initialization
    if (languageDetector == null) {
        logger.debug("Initializing language detector...");
        //load all languages:
        try {
            // read built-in profiles
            this.languageProfiles = new LanguageProfileReader().readAllBuiltIn();

            // read custom profiles

            List<String> profileFileNames = new ArrayList<>();

            // prevent FileSystemNotFoundException...
            final Map<String, String> env = new HashMap<>();
            final String[] array = this.getClass().getResource("/languageProfiles").toURI().toString().split("!");
            Path path;
            FileSystem fs = null;
            if (array.length > 1) {
                fs = FileSystems.newFileSystem(URI.create(array[0]), env);
                path = fs.getPath(array[1]);
            } else {
                path = Paths.get(this.getClass().getResource("/languageProfiles").toURI());
            }

            Files.walk(path)
                    .forEach(file -> {
                        if (!Files.isDirectory(file))
                            profileFileNames.add(file.getFileName().toString());
                    });

            if (fs != null)
                fs.close();
            // according to the documentation LanguageProfileReader#readAll should not be used for files within the .jar.
            this.languageProfiles.addAll(new LanguageProfileReader().read("languageProfiles", profileFileNames));
        } catch (IOException | URISyntaxException e) {
            logger.error("Error loading language profiles", e);
        }

        this.languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard()).withProfiles(languageProfiles).build();

        this.textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
        logger.debug("... language detector initialized");
    }
}
 

import com.optimaize.langdetect.text.CommonTextObjectFactories; //导入方法依赖的package包/类
public static String checkIfNonEnglish(Post post){


        List<LanguageProfile> languageProfiles;
        com.optimaize.langdetect.LanguageDetector optimaizeDetector;
        org.apache.tika.language.detect.LanguageDetector tikaDetector;
        TextObjectFactory textObjectFactory;

        String dataToCheck = stripTags(stripBody(post)).replaceAll("\\p{Punct}+", "");
        try {

            languageProfiles = new LanguageProfileReader().readAllBuiltIn();
            optimaizeDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
                    .withProfiles(languageProfiles)
                    .build();
            textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
            TextObject textObject = textObjectFactory.forText(dataToCheck);
            Optional<LdLocale> lang = optimaizeDetector.detect(textObject);
            if (!lang.isPresent()) {
                if(dataToCheck.length()>50) {
                   tikaDetector = new OptimaizeLangDetector().loadModels();
                   LanguageWriter writer = new LanguageWriter(tikaDetector);
                   writer.append(dataToCheck);
                   LanguageResult result = writer.getLanguage();
                   String tikaLang = result.getLanguage();
                   writer.close();

                   if (!tikaLang.toLowerCase().equals("")) {
                       return tikaLang;
                   }
                   else{
                       return null;
                   }
                }
                else if(dataToCheck.length()<50){
                    return null;
                }
                if(checkIfNoCodeBlock(post)){
                    return "Gibberish";
                }
                return null;
            }
            return lang.get().getLanguage();
        }
        catch (IOException e){
            e.printStackTrace();
        }
        return null;
    }