Java Command类代码示例

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
@Test
public void executePipeline(
    final @Mocked MorphlineUtils.Pipeline pipeline,
    final @Mocked Command morphline
) throws Exception {

  final Record inputRecord = new Record();

  final Record outputRecord = new Record();
  outputRecord.put("field1", "value1");

  new Expectations() {{
    morphline.process(inputRecord); result = true;
    pipeline.getCollector().getRecords(); result = Lists.newArrayList(outputRecord);
  }};

  List<Record> outputList = MorphlineUtils.executePipeline(pipeline, inputRecord);
  assertEquals(Lists.newArrayList(outputRecord), outputList);
}
 

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
@Test (expected = MorphlineCompilationException.class)
public void morphlineCompilationError(
    final @Mocked Compiler compiler
    ) throws Exception {

  new Expectations() {{
    config.getString(MorphlineTranslator.MORPHLINE); result = getResourcePath(MORPHLINE_FILE);
    config.getStringList(MorphlineTranslator.FIELD_NAMES); result = Lists.newArrayList("bar");
    config.getStringList(MorphlineTranslator.FIELD_TYPES); result = Lists.newArrayList("int");

    compiler.compile((File) any, anyString, (MorphlineContext) any, (Command) any); result = new Exception("Compilation exception");
  }};

  stringMorphline.configure(config);
  stringMorphline.translate("The Key", "The Message");
}
 

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
public TokenizeText(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.inputFieldName = getConfigs().getString(config, "inputField");
  this.outputFieldName = getConfigs().getString(config, "outputField");      
  String solrFieldType = getConfigs().getString(config, "solrFieldType");      
  Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator");
  SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
  LOG.debug("solrLocator: {}", locator);
  IndexSchema schema = locator.getIndexSchema();
  FieldType fieldType = schema.getFieldTypeByName(solrFieldType);
  if (fieldType == null) {
    throw new MorphlineCompilationException("Missing Solr field type in schema.xml for name: " + solrFieldType, config);
  }
  this.analyzer = fieldType.getIndexAnalyzer();
  Preconditions.checkNotNull(analyzer);
  try { // register CharTermAttribute for later (implicit) reuse
    this.token = analyzer.tokenStream("content", reader).addAttribute(CharTermAttribute.class);
  } catch (IOException e) {
    throw new MorphlineCompilationException("Cannot create token stream", config, e);
  }
  Preconditions.checkNotNull(token);
  validateArguments();
}
 

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
public GenerateSolrSequenceKey(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.baseIdFieldName = getConfigs().getString(config, "baseIdField", Fields.BASE_ID);
  this.preserveExisting = getConfigs().getBoolean(config, "preserveExisting", true);      
  
  Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator");
  SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
  LOG.debug("solrLocator: {}", locator);
  IndexSchema schema = locator.getIndexSchema();
  SchemaField uniqueKey = schema.getUniqueKeyField();
  uniqueKeyName = uniqueKey == null ? null : uniqueKey.getName();
  
  String tmpIdPrefix = getConfigs().getString(config, "idPrefix", null);  // for load testing only
  Random tmpRandomIdPrefx = null;
  if ("random".equals(tmpIdPrefix)) { // for load testing only
    tmpRandomIdPrefx = new Random(new SecureRandom().nextLong());    
    tmpIdPrefix = null;
  }
  idPrefix = tmpIdPrefix;
  randomIdPrefix = tmpRandomIdPrefx;
  validateArguments();
}
 

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
LoadElasticsearch(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  elapsedTime = getTimer(Metrics.ELAPSED_TIME);
  Config elasticsearchConfig = getConfigs().getConfig(config, ELASTICSEARCH_CONFIGURATION);
  String loaderType = getConfigs().getString(config, DOCUMENT_LOADER_TYPE);
  indexName = new FieldExpression(getConfigs().getString(config, INDEX_NAME), getConfig());
  indexType = new FieldExpression(getConfigs().getString(config, TYPE), getConfig());
  ignoreFields = new LinkedHashSet<String>(getConfigs().getStringList(config, IGNORE_FIELDS, new ArrayList<String>()));
  ttl = getConfigs().getInt(config, TTL);
  validateArguments();

  DocumentLoaderFactory documentLoaderFactory = new DocumentLoaderFactory();
  try {
    loader = documentLoaderFactory.getClient(loaderType, elasticsearchConfig);
  } catch (IllegalArgumentException e) {
    throw new MorphlineRuntimeException(e);
  }
}
 

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
public TokenizeText(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.inputFieldName = getConfigs().getString(config, "inputField");
  this.outputFieldName = getConfigs().getString(config, "outputField");      
  String solrFieldType = getConfigs().getString(config, "solrFieldType");      
  Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator");
  SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
  LOG.debug("solrLocator: {}", locator);
  IndexSchema schema = locator.getIndexSchema();
  FieldType fieldType = schema.getFieldTypeByName(solrFieldType);
  if (fieldType == null) {
    throw new MorphlineCompilationException("Missing Solr field type in schema.xml for name: " + solrFieldType, config);
  }
  this.analyzer = fieldType.getAnalyzer();
  Preconditions.checkNotNull(analyzer);
  try { // register CharTermAttribute for later (implicit) reuse
    this.token = analyzer.tokenStream("content", reader).addAttribute(CharTermAttribute.class);
  } catch (IOException e) {
    throw new MorphlineCompilationException("Cannot create token stream", config, e);
  }
  Preconditions.checkNotNull(token);
  validateArguments();
}
 

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
/**
 *
 * @param morphlineFile
 * @param morphlineId
 * @param collector
 * @param isProduction
 * @return
 */
public static Pipeline setPipeline(String morphlineFile, String morphlineId, Collector collector, boolean isProduction) {
  LOG.debug("Constructing Pipeline[{}#{}]", morphlineFile, morphlineId);

  // Set up the Morphline context and handler
  MorphlineContext context = new MorphlineContext.Builder()
      .setExceptionHandler(new FaultTolerance(isProduction, false))
      .build();

  // Compile the Morphline process
  Command morphline;
  try {
    morphline = new Compiler().compile(
        new File(morphlineFile),
        morphlineId,
        context,
        collector);
  } catch (Exception e) {
    throw new MorphlineCompilationException("Morphline compilation error", null, e);
  }

  // Create the pipeline wrapper
  Pipeline pipeline = new Pipeline(morphline, collector);

  // Ensure shutdown notification to Morphline commands esp in streaming environments
  JVMUtils.closeAtShutdown(pipeline);

  // Prep the pipeline
  Notifications.notifyBeginTransaction(pipeline.getMorphline());

  // Register the pipeline into the cache
  if (null == pipelineCache.get()) {
    pipelineCache.set(new HashMap<String, Pipeline>());
  }
  pipelineCache.get().put(morphlineFile + SEPARATOR + morphlineId, pipeline);

  LOG.trace("Pipeline[{}#{}] prepared", morphlineFile, morphlineId);
  return pipeline;
}
 

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
public static List<Record> executePipeline(Pipeline pipeline, Record inputRecord) {
  Command morphline = pipeline.getMorphline();

  try {
    LOG.trace("Input Record: {}", inputRecord);

    // Process the Record
    Notifications.notifyStartSession(morphline);
    boolean success = morphline.process(inputRecord);
    Notifications.notifyCommitTransaction(morphline);

    if (!success) {
      throw new MorphlineRuntimeException("Morphline failed to process incoming Record: " + inputRecord);
    }

    // Collect the output
    List<Record> outputRecords = pipeline.getCollector().getRecords();
    if (!outputRecords.iterator().hasNext()) {
      throw new MorphlineRuntimeException("Morphline did not produce output Record(s)");
    }
    LOG.trace("Output Record(s): {}", outputRecords);

    return outputRecords;

  } catch (RuntimeException e) {
    Notifications.notifyRollbackTransaction(morphline);
    // TODO : Review exception handling
    LOG.warn("Morphline failed to execute properly on incoming Record: " + inputRecord, e);
    throw e;
  }
}
 

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
@Test (expected = MorphlineRuntimeException.class)
public void executePipelineProcessError(
    final @Mocked MorphlineUtils.Pipeline pipeline,
    final @Mocked Command morphline
) throws Exception {

  final Record inputRecord = new Record();

  new Expectations() {{
    morphline.process(inputRecord); result = false;
  }};

  MorphlineUtils.executePipeline(pipeline, inputRecord);
}
 

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
@Test (expected = MorphlineRuntimeException.class)
public void executePipelineNoRecords(
    final @Mocked MorphlineUtils.Pipeline pipeline,
    final @Mocked Command morphline
) throws Exception {

  final Record inputRecord = new Record();

  new Expectations() {{
    morphline.process(inputRecord); result = true;
    pipeline.getCollector().getRecords(); result = Lists.newArrayList();
  }};

  MorphlineUtils.executePipeline(pipeline, inputRecord);
}
 

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
public SanitizeUnknownSolrFields(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);      
  
  Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator");
  SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
  LOG.debug("solrLocator: {}", locator);
  this.schema = locator.getIndexSchema();
  Preconditions.checkNotNull(schema);
  LOG.trace("Solr schema: \n{}", Joiner.on("\n").join(new TreeMap(schema.getFields()).values()));
  
  String str = getConfigs().getString(config, "renameToPrefix", "").trim();
  this.renameToPrefix = str.length() > 0 ? str : null;  
  validateArguments();
}
 

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
public LoadSolr(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator");
  SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
  LOG.debug("solrLocator: {}", locator);
  this.loader = locator.getLoader();
  Config boostsConfig = getConfigs().getConfig(config, "boosts", ConfigFactory.empty());
  for (Map.Entry<String, Object> entry : new Configs().getEntrySet(boostsConfig)) {
    String fieldName = entry.getKey();        
    float boost = Float.parseFloat(entry.getValue().toString().trim());
    boosts.put(fieldName, boost);
  }
  validateArguments();
  this.elapsedTime = getTimer(Metrics.ELAPSED_TIME);
}
 

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
protected Command parse(String file, String collection) throws IOException {
  SolrLocator locator = new SolrLocator(createMorphlineContext());
  locator.setCollectionName(collection);
  locator.setZkHost(zkServer.getZkAddress());
  //locator.setServerUrl(cloudJettys.get(0).url); // TODO: download IndexSchema from solrUrl not yet implemented
  //locator.setSolrHomeDir(SOLR_HOME_DIR.getPath());
  Config config = new Compiler().parse(new File(RESOURCES_DIR + "/" + file + ".conf"), locator.toConfig("SOLR_LOCATOR"));
  config = config.getConfigList("morphlines").get(0);
  return createMorphline(config);
}
 

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
public ToUpperCase(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
    super(builder, config, parent, child, context);
    this.fieldName = getConfigs().getString(config, "field");
    this.locale = getConfigs().getLocale(config, "locale", Locale.ROOT);
    LOG.debug("fieldName: {}", fieldName);
    validateArguments();
}
 

import org.kitesdk.morphline.api.Command; //导入依赖的package包/类
public ToLowerCase(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
    super(builder, config, parent, child, context);
    this.fieldName = getConfigs().getString(config, "field");
    this.locale = getConfigs().getLocale(config, "locale", Locale.ROOT);
    LOG.debug("fieldName: {}", fieldName);
    validateArguments();
}