Java Processor类代码示例

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
@Override
public Processor create(Map<String, Processor.Factory> processorFactories, String tag,
                        Map<String, Object> config) throws Exception {
    String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
    if (field.contains(".") == false) {
        throw ConfigurationUtils.newConfigurationException(ConfigurationUtils.TAG_KEY, tag, "field",
                "field does not contain a dot");
    }
    if (field.indexOf('.') == 0 || field.lastIndexOf('.') == field.length() - 1) {
        throw ConfigurationUtils.newConfigurationException(ConfigurationUtils.TAG_KEY, tag, "field",
                "Field can't start or end with a dot");
    }
    int firstIndex = -1;
    for (int index = field.indexOf('.'); index != -1; index = field.indexOf('.', index + 1)) {
        if (index - firstIndex == 1) {
            throw ConfigurationUtils.newConfigurationException(ConfigurationUtils.TAG_KEY, tag, "field",
                    "No space between dots");
        }
        firstIndex = index;
    }

    String path = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "path");
    return new DotExpanderProcessor(tag, path, field);
}
 

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
@Override
public GrokProcessor create(Map<String, Processor.Factory> registry, String processorTag,
                            Map<String, Object> config) throws Exception {
    String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
    List<String> matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns");
    boolean traceMatch = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "trace_match", false);
    boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);

    if (matchPatterns.isEmpty()) {
        throw newConfigurationException(TYPE, processorTag, "patterns", "List of patterns must not be empty");
    }
    Map<String, String> customPatternBank = ConfigurationUtils.readOptionalMap(TYPE, processorTag, config, "pattern_definitions");
    Map<String, String> patternBank = new HashMap<>(builtinPatterns);
    if (customPatternBank != null) {
        patternBank.putAll(customPatternBank);
    }

    try {
        return new GrokProcessor(processorTag, patternBank, matchPatterns, matchField, traceMatch, ignoreMissing);
    } catch (Exception e) {
        throw newConfigurationException(TYPE, processorTag, "patterns",
            "Invalid regex pattern found in: " + matchPatterns + ". " + e.getMessage());
    }

}
 

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
@Override
public JsonProcessor create(Map<String, Processor.Factory> registry, String processorTag,
                            Map<String, Object> config) throws Exception {
    String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
    String targetField = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "target_field");
    boolean addToRoot = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "add_to_root", false);

    if (addToRoot && targetField != null) {
        throw newConfigurationException(TYPE, processorTag, "target_field",
            "Cannot set a target field while also setting `add_to_root` to true");
    }

    if (targetField == null) {
        targetField = field;
    }

    return new JsonProcessor(processorTag, field, targetField, addToRoot);
}
 

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public DateProcessor create(Map<String, Processor.Factory> registry, String processorTag,
                            Map<String, Object> config) throws Exception {
    String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
    String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET_FIELD);
    String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "timezone");
    DateTimeZone timezone = timezoneString == null ? DateTimeZone.UTC : DateTimeZone.forID(timezoneString);
    String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "locale");
    Locale locale = Locale.ENGLISH;
    if (localeString != null) {
        try {
            locale = (new Locale.Builder()).setLanguageTag(localeString).build();
        } catch (IllformedLocaleException e) {
            throw new IllegalArgumentException("Invalid language tag specified: " + localeString);
        }
    }
    List<String> formats = ConfigurationUtils.readList(TYPE, processorTag, config, "formats");
    return new DateProcessor(processorTag, timezone, locale, field, formats, targetField);
}
 

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
@Override
public SortProcessor create(Map<String, Processor.Factory> registry, String processorTag,
                            Map<String, Object> config) throws Exception {
    String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, FIELD);
    try {
        SortOrder direction = SortOrder.fromString(
            ConfigurationUtils.readStringProperty(
                TYPE,
                processorTag,
                config,
                ORDER,
                DEFAULT_ORDER));
        return new SortProcessor(processorTag, field, direction);
    } catch (IllegalArgumentException e) {
        throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, ORDER, e.getMessage());
    }
}
 

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
public void testJoinStrings() throws Exception {
    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
    int numItems = randomIntBetween(1, 10);
    String separator = randomFrom(SEPARATORS);
    List<String> fieldValue = new ArrayList<>(numItems);
    String expectedResult = "";
    for (int j = 0; j < numItems; j++) {
        String value = randomAsciiOfLengthBetween(1, 10);
        fieldValue.add(value);
        expectedResult += value;
        if (j < numItems - 1) {
            expectedResult += separator;
        }
    }
    String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
    Processor processor = new JoinProcessor(randomAsciiOfLength(10), fieldName, separator);
    processor.execute(ingestDocument);
    assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedResult));
}
 

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
public void testJoinIntegers() throws Exception {
    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
    int numItems = randomIntBetween(1, 10);
    String separator = randomFrom(SEPARATORS);
    List<Integer> fieldValue = new ArrayList<>(numItems);
    String expectedResult = "";
    for (int j = 0; j < numItems; j++) {
        int value = randomInt();
        fieldValue.add(value);
        expectedResult += value;
        if (j < numItems - 1) {
            expectedResult += separator;
        }
    }
    String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
    Processor processor = new JoinProcessor(randomAsciiOfLength(10), fieldName, separator);
    processor.execute(ingestDocument);
    assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(expectedResult));
}
 

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
public void testSplitAppendable() throws Exception {
    Map<String, Object> splitConfig = new HashMap<>();
    splitConfig.put("field", "flags");
    splitConfig.put("separator", "\\|");
    Processor splitProcessor = (new SplitProcessor.Factory()).create(null, null, splitConfig);
    Map<String, Object> source = new HashMap<>();
    source.put("flags", "new|hot|super|fun|interesting");
    IngestDocument ingestDocument = new IngestDocument(source, new HashMap<>());
    splitProcessor.execute(ingestDocument);
    @SuppressWarnings("unchecked")
    List<String> flags = (List<String>)ingestDocument.getFieldValue("flags", List.class);
    assertThat(flags, equalTo(Arrays.asList("new", "hot", "super", "fun", "interesting")));
    ingestDocument.appendFieldValue("flags", "additional_flag");
    assertThat(ingestDocument.getFieldValue("flags", List.class), equalTo(Arrays.asList("new", "hot", "super",
            "fun", "interesting", "additional_flag")));
}
 

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
public void testCreateWithTooManyProcessorTypes() throws Exception {
    Processor processor = new TestProcessor(ingestDocument -> { });
    Map<String, Processor.Factory> registry = new HashMap<>();
    registry.put("_first", (r, t, c) -> processor);
    registry.put("_second", (r, t, c) -> processor);
    ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory();

    Map<String, Object> config = new HashMap<>();
    config.put("field", "_field");
    Map<String, Object> processorTypes = new HashMap<>();
    processorTypes.put("_first", Collections.emptyMap());
    processorTypes.put("_second", Collections.emptyMap());
    config.put("processor", processorTypes);
    Exception exception = expectThrows(ElasticsearchParseException.class, () -> forEachFactory.create(registry, null, config));
    assertThat(exception.getMessage(), equalTo("[processor] Must specify exactly one processor type"));
}
 

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
public void testConvertBooleanList() throws Exception {
    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
    int numItems = randomIntBetween(1, 10);
    List<String> fieldValue = new ArrayList<>();
    List<Boolean> expectedList = new ArrayList<>();
    for (int j = 0; j < numItems; j++) {
        boolean randomBoolean = randomBoolean();
        String booleanString = Boolean.toString(randomBoolean);
        if (randomBoolean) {
            booleanString = booleanString.toUpperCase(Locale.ROOT);
        }
        fieldValue.add(booleanString);
        expectedList.add(randomBoolean);
    }
    String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
    Processor processor = new ConvertProcessor(randomAsciiOfLength(10), fieldName, fieldName, Type.BOOLEAN, false);
    processor.execute(ingestDocument);
    assertThat(ingestDocument.getFieldValue(fieldName, List.class), equalTo(expectedList));
}
 

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
@Override
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
    Path userAgentConfigDirectory = parameters.env.configFile().resolve("ingest-user-agent");

    if (Files.exists(userAgentConfigDirectory) == false && Files.isDirectory(userAgentConfigDirectory)) {
        throw new IllegalStateException(
            "the user agent directory [" + userAgentConfigDirectory + "] containing the regex file doesn't exist");
    }

    long cacheSize = CACHE_SIZE_SETTING.get(parameters.env.settings());
    Map<String, UserAgentParser> userAgentParsers;
    try {
        userAgentParsers = createUserAgentParsers(userAgentConfigDirectory, new UserAgentCache(cacheSize));
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return Collections.singletonMap(UserAgentProcessor.TYPE, new UserAgentProcessor.Factory(userAgentParsers));
}
 

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
@Override
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
    if (databaseReaders != null) {
        throw new IllegalStateException("getProcessors called twice for geoip plugin!!");
    }
    Path geoIpConfigDirectory = parameters.env.configFile().resolve("ingest-geoip");
    NodeCache cache;
    long cacheSize = CACHE_SIZE.get(parameters.env.settings());
    if (cacheSize > 0) {
        cache = new GeoIpCache(cacheSize);
    } else {
        cache = NoCache.getInstance();
    }
    try {
        databaseReaders = loadDatabaseReaders(geoIpConfigDirectory, cache);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return Collections.singletonMap(GeoIpProcessor.TYPE, new GeoIpProcessor.Factory(databaseReaders));
}
 

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
public void testRenameAtomicOperationRemoveFails() throws Exception {
    Map<String, Object> source = new HashMap<String, Object>() {
        @Override
        public Object remove(Object key) {
            if (key.equals("list")) {
                throw new UnsupportedOperationException();
            }
            return super.remove(key);
        }
    };
    source.put("list", Collections.singletonList("item"));

    IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
    Processor processor = new RenameProcessor(randomAsciiOfLength(10), "list", "new_field", false);
    try {
        processor.execute(ingestDocument);
        fail("processor execute should have failed");
    } catch (UnsupportedOperationException e) {
        //the set failed, the old field has not been removed
        assertThat(ingestDocument.getSourceAndMetadata().containsKey("list"), equalTo(true));
        assertThat(ingestDocument.getSourceAndMetadata().containsKey("new_field"), equalTo(false));
    }
}
 

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
@Override
public CsvProcessor create(Map<String, Processor.Factory> factories, String tag, Map<String, Object> config) 
    throws Exception {
    String field = readStringProperty(TYPE, tag, config, "field");
    List<String> columns = readList(TYPE, tag, config, "columns");
    // FIXME should test duplicate name
    if (columns.size() == 0) {
        throw new IllegalArgumentException("columns is missing");
    }
    String quoteChar = readStringProperty(TYPE, tag, config, "quote_char", "\"");
    if (Strings.isEmpty(quoteChar) || quoteChar.length() != 1) {
        throw new IllegalArgumentException("quote_char must be a character, like \" or \'");
    }
    String separator = readStringProperty(TYPE, tag, config, "separator", ",");
    if (Strings.isEmpty(separator) || separator.length() != 1) {
        throw new IllegalArgumentException("separator must be a character, like , or TAB");
    }

    return new CsvProcessor(tag, field, columns, quoteChar.charAt(0), separator.charAt(0));
}
 

import org.elasticsearch.ingest.Processor; //导入依赖的package包/类
public void testSortStrings() throws Exception {
    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
    int numItems = randomIntBetween(1, 10);
    List<String> fieldValue = new ArrayList<>(numItems);
    List<String> expectedResult = new ArrayList<>(numItems);
    for (int j = 0; j < numItems; j++) {
        String value = randomAsciiOfLengthBetween(1, 10);
        fieldValue.add(value);
        expectedResult.add(value);
    }
    Collections.sort(expectedResult);

    SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
    if (order.equals(SortOrder.DESCENDING)) {
        Collections.reverse(expectedResult);
    }

    String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
    Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
    processor.execute(ingestDocument);
    assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
}