Java RowProducer类代码示例

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
/**
 * Create, execute, and return the row listener attached to the output step with complete results from the
 * execution.
 * 
 * @param checkSumType Type of checksum to use (the array index of {@link CheckSumMeta#checksumtypeCodes})
 * @param compatibilityMode Use compatibility mode for CheckSum
 * @param input String to calculate checksum for
 * @return RowListener with results.
 */
private MockRowListener executeHexTest(int checkSumType, boolean compatibilityMode, String input) throws Exception {
  Trans trans = buildHexadecimalChecksumTrans(checkSumType, compatibilityMode);

  trans.prepareExecution(null);

  StepInterface output = trans.getRunThread("Output", 0); //$NON-NLS-1$
  MockRowListener listener = new MockRowListener();
  output.addRowListener(listener);

  RowProducer rp = trans.addRowProducer("CheckSum", 0); //$NON-NLS-1$
  RowMeta inputRowMeta = createStringRowMeta();
  ((BaseStep) trans.getRunThread("CheckSum", 0)).setInputRowMeta(inputRowMeta); //$NON-NLS-1$

  trans.startThreads();

  rp.putRow(inputRowMeta, new Object[] { input });
  rp.finished();

  trans.waitUntilFinished();
  trans.stopAll();
  trans.cleanup();
  return listener;
}
 

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
public void injectValue( Object key, int keyOrdinal, ITypeConverter inConverterK, Object value, int valueOrdinal,
                         ITypeConverter inConverterV, RowMeta injectorRowMeta, RowProducer rowProducer,
                         Reporter reporter )
  throws Exception {
  Object[] row = new Object[ injectorRowMeta.size() ];
  row[ keyOrdinal ] =
    inConverterK != null ? inConverterK.convert( injectorRowMeta.getValueMeta( keyOrdinal ), key ) : key;
  row[ valueOrdinal ] =
    inConverterV != null ? inConverterV.convert( injectorRowMeta.getValueMeta( valueOrdinal ), value )
      : value;

  if ( debug ) {
    setDebugStatus( reporter, "Injecting input record [" + row[ keyOrdinal ] + "] - [" + row[ valueOrdinal ] + "]" );
  }

  rowProducer.putRow( injectorRowMeta, row );
}
 

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
public void injectValue( Object key, int keyOrdinal, ITypeConverter inConverterK,
                         Object value, int valueOrdinal, ITypeConverter inConverterV,
                         RowMetaInterface injectorRowMeta, RowProducer rowProducer, Reporter reporter )
  throws Exception {
  Object[] row = new Object[ injectorRowMeta.size() ];
  row[ keyOrdinal ] =
    inConverterK != null ? inConverterK.convert( injectorRowMeta.getValueMeta( keyOrdinal ), key ) : key;
  row[ valueOrdinal ] =
    inConverterV != null ? inConverterV.convert( injectorRowMeta.getValueMeta( valueOrdinal ), value ) : value;

  if ( debug ) {
    setDebugStatus( reporter, "Injecting input record [" + row[ keyOrdinal ] + "] - [" + row[ valueOrdinal ] + "]" );
  }

  rowProducer.putRow( injectorRowMeta, row );
}
 

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
@Test
public void subTransOutputIsAccessibleOutside() throws Exception {
  TransExecutorMeta executorMeta = getExecutorMeta( transExecutor );
  executorMeta.setOutputRowsSourceStepMeta( dummy );

  Trans trans = createTrans( transMeta );
  RowStepCollector endRc = listenExecutor( trans );
  RowProducer rp = trans.addRowProducer( injector.getName(), 0 );

  trans.startThreads();

  RowMetaAndData testInput = new RowMetaAndData( createRowMetaForOneField(), SAMPLE_INPUT );
  rp.putRow( testInput.getRowMeta(), testInput.getData() );
  rp.finished();

  trans.waitUntilFinished();

  assertEquals( EXPECTED_SUBTRANS_OUTPUT_AMOUNT, endRc.getRowsWritten().size() );
  assertThat( asList( endRc.getRowsWritten().get( 0 ).getData() ),
    hasItem( (Object) EXPECTED_SUBTRANS_OUTPUT_PATTERN )
  );
}
 

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
@Test
public void executorsInputIsStraightlyCopiedToOutput() throws Exception {
  TransExecutorMeta executorMeta = getExecutorMeta( transExecutor );
  executorMeta.setExecutorsOutputStepMeta( dummy );

  Trans trans = createTrans( transMeta );
  RowStepCollector endRc = listenExecutor( trans );
  RowProducer rp = trans.addRowProducer( injector.getName(), 0 );

  trans.startThreads();

  RowMetaAndData testInput = new RowMetaAndData( createRowMetaForOneField(), SAMPLE_INPUT );
  rp.putRow( testInput.getRowMeta(), testInput.getData() );
  rp.finished();

  trans.waitUntilFinished();

  assertEquals( testInput.size(), endRc.getRowsWritten().size() );
  assertThat( asList( endRc.getRowsWritten().get( 0 ).getData() ),
    hasItem( (Object) SAMPLE_INPUT )
  );
}
 

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
@Test
public void subTransExecutionStatisticsIsCollected() throws Exception {
  TransExecutorMeta executorMeta = getExecutorMeta( transExecutor );
  executorMeta.setExecutionTimeField( "time" );
  executorMeta.setExecutionResultTargetStepMeta( dummy );

  Trans trans = createTrans( transMeta );
  RowStepCollector endRc = listenExecutor( trans );
  RowProducer rp = trans.addRowProducer( injector.getName(), 0 );

  trans.startThreads();

  RowMetaAndData testInput = new RowMetaAndData( createRowMetaForOneField(), SAMPLE_INPUT );
  rp.putRow( testInput.getRowMeta(), testInput.getData() );
  rp.finished();

  trans.waitUntilFinished();

  assertFalse( endRc.getRowsWritten().isEmpty() );
  // execution time field
  assertNotNull( endRc.getRowsWritten().get( 0 ).getData()[ 0 ] );
}
 

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
/**
 * Create, execute, and return the row listener attached to the output step with complete results from the execution.
 *
 * @param checkSumType
 *          Type of checksum to use (the array index of {@link CheckSumMeta#checksumtypeCodes})
 * @param compatibilityMode
 *          Use compatibility mode for CheckSum
 * @param input
 *          String to calculate checksum for
 * @param meta
 *          meta to be used
 * @return RowListener with results.
 */
private MockRowListener executeHexTest( int checkSumType, boolean compatibilityMode, Object input, ValueMetaInterface meta, boolean oldChecksumBehaviour ) throws Exception {
  Trans trans = buildHexadecimalChecksumTrans( checkSumType, compatibilityMode, oldChecksumBehaviour );

  trans.prepareExecution( null );

  StepInterface output = trans.getRunThread( "Output", 0 );
  MockRowListener listener = new MockRowListener();
  output.addRowListener( listener );

  RowProducer rp = trans.addRowProducer( "CheckSum", 0 );
  RowMeta inputRowMeta = createStringRowMeta( meta );
  ( (BaseStep) trans.getRunThread( "CheckSum", 0 ) ).setInputRowMeta( inputRowMeta );

  trans.startThreads();

  rp.putRow( inputRowMeta, new Object[] { input } );
  rp.finished();

  trans.waitUntilFinished();
  trans.stopAll();
  trans.cleanup();
  return listener;
}
 

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
protected Trans createAndTestTrans( PluginRegistry registry, TransMeta transMeta, StepMeta inputStep,
    RowStepCollector rowStepCollector, String name, int limit ) throws KettleException {
  //
  // Create a dummy step
  //
  String dummyStepname = "dummy step";
  DummyTransMeta dm1 = new DummyTransMeta();

  String dummyPid1 = registry.getPluginId( StepPluginType.class, dm1 );
  StepMeta dummyStep1 = new StepMeta( dummyPid1, dummyStepname, dm1 );
  transMeta.addStep( dummyStep1 );

  TransHopMeta hi3 = new TransHopMeta( inputStep, dummyStep1 );
  transMeta.addTransHop( hi3 );

  // Now execute the transformation...
  Trans trans = new Trans( transMeta );

  trans.prepareExecution( null );

  StepInterface si = trans.getStepInterface( dummyStepname, 0 );
  si.addRowListener( rowStepCollector );

  RowProducer rp = trans.addRowProducer( inputStep.getName(), 0 );

  RowMeta rowMeta = new RowMeta();
  rowMeta.addValueMeta( new ValueMetaString( "pageSize" ) );
  rowMeta.addValueMeta( new ValueMetaString( "name" ) );
  rp.putRow( rowMeta, new Object[] { Integer.valueOf( limit ), name } );

  rp.finished();
  return trans;
}
 

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
private void processRow( TransProcessControl control ) throws Exception {
  Trans trans = new Trans( transMeta );
  trans.prepareExecution( null );

  RowProducer rp = trans.addRowProducer( injectorStepname, 0 );
  trans.startThreads();
  generateData( rp );
  rp.finished();
  StepInterface si = trans.getStepInterface( synchronizeAfterMergeStepname, 0 );
  switch ( control ) {
    case ITTERUPT:
      trans.stopAll();
      while (  !si.getStatus().equals( StepExecutionStatus.STATUS_STOPPED ) ) {
        //wait until transformation does not stopped
      };
      break;
    case WAIT:
    default:
      trans.waitUntilFinished();
      assertEquals( "Step still started", StepExecutionStatus.STATUS_FINISHED, si.getStatus() );
      break;
  }
  assertEquals( "Unexpected error occurred",  0, si.getErrors() );

  Field field = SynchronizeAfterMerge.class.getDeclaredField( "data" );
  field.setAccessible( true );
  SynchronizeAfterMergeData  data = (SynchronizeAfterMergeData) field.get( si );
  //should be closed and set null after finish transformation
  assertNull( data.db.getConnection() );
}
 

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
private void generateData( RowProducer rp ) {
  RowMetaInterface rm = new RowMeta();
  rm.addValueMeta( new ValueMetaInteger( "ID" ) );
  rm.addValueMeta( new ValueMetaString( "personName" ) );
  rm.addValueMeta( new ValueMetaString( "flag" ) );
  for ( int i = 0; i < ROW_FOR_UPDATE; i++ ) {
    rp.putRow( rm.clone(), new Object[] { "personNameUpdated" + i, i, UPDATE_FLAG } );
  }
  for ( int i = ROW_FOR_UPDATE; i < ROW_FOR_UPDATE + ROW_FOR_DELETE; i++ ) {
    rp.putRow( rm.clone(), new Object[] { "personName" + i, i, DELETE_FLAG } );
  }
  for ( int i = ROW_FOR_UPDATE + ROW_FOR_DELETE; i < ROW_FOR_UPDATE + ROW_FOR_DELETE + ROW_FOR_INSERT; i++ ) {
    rp.putRow( rm.clone(), new Object[] { "personNameInserted" + i, i, INSERT_FLAG } );
  }
}
 

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
@Before
public void setup() throws Exception {
  stepMockHelper =
      new StepMockHelper<SimpleMappingMeta, SimpleMappingData>( "SIMPLE_MAPPING_TEST", SimpleMappingMeta.class,
          SimpleMappingData.class );
  when( stepMockHelper.logChannelInterfaceFactory.create( any(), any( LoggingObjectInterface.class ) ) ).thenReturn(
      stepMockHelper.logChannelInterface );
  when( stepMockHelper.trans.isRunning() ).thenReturn( true );

  // Mock for MappingInput
  MappingInput mpInputMock = mock( MappingInput.class );
  when( mpInputMock.getStepname() ).thenReturn( MAPPING_INPUT_STEP_NAME );

  // Mock for MappingOutput
  MappingOutput mpOutputMock = mock( MappingOutput.class );
  when( mpOutputMock.getStepname() ).thenReturn( MAPPING_OUTPUT_STEP_NAME );

  // Mock for RowDataInputMapper
  RowDataInputMapper rdInputMpMock = mock( RowDataInputMapper.class );
  RowMetaInterface rwMetaInMock = mock( RowMeta.class );
  doReturn( Boolean.TRUE ).when( rdInputMpMock ).putRow( rwMetaInMock, new Object[] { } );

  // Mock for RowProducer
  RowProducer rProducerMock = mock( RowProducer.class );
  when( rProducerMock.putRow( any( RowMetaInterface.class ), any( Object[].class ), anyBoolean() ) )
    .thenReturn( true );

  // Mock for MappingIODefinition
  MappingIODefinition mpIODefMock = mock( MappingIODefinition.class );

  // Set up real SimpleMappingData with some mocked elements
  simpleMpData.mappingInput = mpInputMock;
  simpleMpData.mappingOutput = mpOutputMock;
  simpleMpData.rowDataInputMapper = rdInputMpMock;
  simpleMpData.mappingTrans = stepMockHelper.trans;

  when( stepMockHelper.trans.findStepInterface( MAPPING_OUTPUT_STEP_NAME, 0 ) ).thenReturn( mpOutputMock );
  when( stepMockHelper.trans.addRowProducer( MAPPING_INPUT_STEP_NAME, 0 ) ).thenReturn( rProducerMock );
  when( stepMockHelper.processRowsStepMetaInterface.getInputMapping() ).thenReturn( mpIODefMock );
}
 

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
@Test
public void testStepShouldStopProcessingInput_IfUnderlyingTransitionIsStopped() throws Exception {

  MappingInput mappingInput = mock( MappingInput.class );
  when( mappingInput.getStepname() ).thenReturn( MAPPING_INPUT_STEP_NAME );
  stepMockHelper.processRowsStepDataInterface.mappingInput = mappingInput;

  RowProducer rowProducer = mock( RowProducer.class );
  when( rowProducer.putRow( any( RowMetaInterface.class ), any( Object[].class ), anyBoolean() ) )
    .thenReturn( true );

  StepInterface stepInterface = mock( StepInterface.class );

  Trans mappingTrans = mock( Trans.class );
  when( mappingTrans.addRowProducer( anyString(), anyInt() ) ).thenReturn( rowProducer );
  when( mappingTrans.findStepInterface( anyString(), anyInt() ) ).thenReturn( stepInterface );
  when( mappingTrans.isFinishedOrStopped() ).thenReturn( Boolean.FALSE ).thenReturn( Boolean.TRUE );
  stepMockHelper.processRowsStepDataInterface.mappingTrans = mappingTrans;

  MappingOutput mappingOutput = mock( MappingOutput.class );
  when( mappingOutput.getStepname() ).thenReturn( MAPPING_OUTPUT_STEP_NAME );
  stepMockHelper.processRowsStepDataInterface.mappingOutput = mappingOutput;


  smp = new SimpleMapping( stepMockHelper.stepMeta, stepMockHelper.stepDataInterface, 0, stepMockHelper.transMeta,
    stepMockHelper.trans );
  smp.init( stepMockHelper.initStepMetaInterface, simpleMpData );
  smp.getInputRowSets().add( stepMockHelper.getMockInputRowSet( new Object[] { } ) );
  smp.getInputRowSets().add( stepMockHelper.getMockInputRowSet( new Object[] { } ) );

  assertTrue(
    smp.processRow( stepMockHelper.processRowsStepMetaInterface, stepMockHelper.processRowsStepDataInterface ) );
  assertFalse(
    smp.processRow( stepMockHelper.processRowsStepMetaInterface, stepMockHelper.processRowsStepDataInterface ) );

}
 

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
private void injectDataSetIntoStep( final Trans trans, final TransMeta transMeta,
  final String dataSetName, final MetaStoreFactory<DataSet> dataSetFactory,
  final Repository repository, final IMetaStore metaStore, final StepMeta stepMeta, 
  TransUnitTestSetLocation inputLocation ) throws MetaStoreException, KettleException {

  final DataSet dataSet = dataSetFactory.loadElement( dataSetName );
  final DataSetGroup group = dataSet.getGroup();
  final Database database = new Database( trans, group.getDatabaseMeta() );
  final LogChannelInterface log = trans.getLogChannel();
  
  final RowMetaInterface injectRowMeta = DataSetConst.getStepOutputFields(log, transMeta, stepMeta, dataSet, inputLocation);    
  final RowProducer rowProducer = trans.addRowProducer( stepMeta.getName(), 0 );

  // Look for the step into which we'll inject rows...
  //
  StepMetaDataCombi combi = null;
  for ( StepMetaDataCombi step : trans.getSteps() ) {
    if ( step.stepname.equals( stepMeta.getName() ) ) {
      combi = step;
      break;
    }
  }

  if ( combi != null ) {

    log.logBasic( "Injecting data set '" + dataSetName + "' into step '" + stepMeta.getName() + "', fields: "+injectRowMeta.toStringMeta() );
    final List<Object[]> rows = dataSet.getAllRows(log, inputLocation, injectRowMeta);
    
    // Pass rows
    try {
      Runnable runnable = new Runnable() {
        @Override
        public void run() {
          try {
            
            for( Object[] row : rows ) {
              // pass the row with the external names
              //
              rowProducer.putRow( injectRowMeta, row );                
            }
            rowProducer.finished();
            
          } catch ( Exception e ) {
            throw new RuntimeException( "Problem injecting data set '" + dataSetName + "' row into step '" + stepMeta.getName() + "'", e );
          }
        }
      };
      Thread thread = new Thread( runnable );
      thread.start();

    } finally {
      database.disconnect();
    }
  }
}
 

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
/**
 * Runs the transformation with the below input parameters
 * @param inputData JSON string
 * @param ignoreMissingPath boolean
 * @param defaultPathLeafToNull boolean
 * @return Transformation Results
 */
private List<RowMetaAndData> test(String inputData, boolean ignoreMissingPath, boolean defaultPathLeafToNull)
        throws Exception {
    KettleEnvironment.init();

    // Create a new transformation
    TransMeta transMeta = new TransMeta();
    transMeta.setName("testFastJsonInput");
    PluginRegistry registry = PluginRegistry.getInstance();

    // Create Injector
    String injectorStepName = "injector step";
    StepMeta injectorStep = TestUtilities.createInjectorStep(injectorStepName, registry);
    transMeta.addStep(injectorStep);

    // Create a FastJsonInput step
    String fastJsonInputName = "FastJsonInput step";
    StepMeta fastJsonInputStep = createFastJsonInputStep(fastJsonInputName, registry, ignoreMissingPath,
            defaultPathLeafToNull);
    transMeta.addStep(fastJsonInputStep);

    // TransHopMeta between injector step and FastJsonInput
    TransHopMeta injector_hop_fjis = new TransHopMeta(injectorStep, fastJsonInputStep);
    transMeta.addTransHop(injector_hop_fjis);

    // Create a dummy step
    String dummyStepName = "dummy step";
    StepMeta dummyStep = TestUtilities.createDummyStep(dummyStepName, registry);
    transMeta.addStep(dummyStep);

    // TransHopMeta between FastJsonInput and Dummy
    TransHopMeta fjis_hop_dummy = new TransHopMeta(fastJsonInputStep, dummyStep);
    transMeta.addTransHop(fjis_hop_dummy);

    // Execute the transformation
    Trans trans = new Trans(transMeta);
    trans.prepareExecution(null);

    // Create a row collector and add it to the dummy step interface
    StepInterface si = trans.getStepInterface(dummyStepName, 0);
    RowStepCollector dummyRowCollector = new RowStepCollector();
    si.addRowListener(dummyRowCollector);

    // Create a row producer
    RowProducer rowProducer = trans.addRowProducer(injectorStepName, 0);
    trans.startThreads();

    // create the rows
    List<RowMetaAndData> inputList = createInputData(inputData);
    for (RowMetaAndData rowMetaAndData : inputList) {
        rowProducer.putRow(rowMetaAndData.getRowMeta(), rowMetaAndData.getData());
    }
    rowProducer.finished();

    trans.waitUntilFinished();

    return dummyRowCollector.getRowsWritten();
}
 

import org.pentaho.di.trans.RowProducer; //导入依赖的package包/类
/**
* Test case for injector step... also a show case on how
* to use injector.
*/
  public void testInjector() throws Exception
  {
      EnvUtil.environmentInit();

      //
      // Create a new transformation...
      //
      TransMeta transMeta = new TransMeta();
      transMeta.setName("injectortest");
  	
      StepLoader steploader = StepLoader.getInstance();            

      // 
      // create an injector step...
      //
      String injectorStepname = "injector step";
      InjectorMeta im = new InjectorMeta();
      
      // Set the information of the injector.
              
      String injectorPid = steploader.getStepPluginID(im);
      StepMeta injectorStep = new StepMeta(injectorPid, injectorStepname, (StepMetaInterface)im);
      transMeta.addStep(injectorStep);

      // 
      // Create a dummy step
      //
      String dummyStepname = "dummy step";            
      DummyTransMeta dm = new DummyTransMeta();

      String dummyPid = steploader.getStepPluginID(dm);
      StepMeta dummyStep = new StepMeta(dummyPid, dummyStepname, (StepMetaInterface)dm);
      transMeta.addStep(dummyStep);                              

      TransHopMeta hi = new TransHopMeta(injectorStep, dummyStep);
      transMeta.addTransHop(hi);
              
      // Now execute the transformation...
      Trans trans = new Trans(transMeta);

      trans.prepareExecution(null);
              
      StepInterface si = trans.getStepInterface(dummyStepname, 0);
      RowStepCollector rc = new RowStepCollector();
      si.addRowListener(rc);
      
      RowProducer rp = trans.addRowProducer(injectorStepname, 0);
      trans.startThreads();
      
      // add rows
      List<RowMetaAndData> inputList = createData();
      for (RowMetaAndData rm : inputList )
      {
      	rp.putRow(rm.getRowMeta(), rm.getData());
      }   
      rp.finished();

      trans.waitUntilFinished();   
      
      List<RowMetaAndData> resultRows = rc.getRowsWritten();
      checkRows(resultRows, inputList);
  }