Java TaskAttemptContext.getConfiguration方法代码示例

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext context)
  throws IOException, InterruptedException {

  // what follows is very questionable but a quick test
  // the file is read from HDFS and copied to a temporary location
  FileSplit split = (FileSplit)inputSplit;
  Configuration job = context.getConfiguration();
  Path file = split.getPath();
  FileSystem fs = file.getFileSystem(job);
  java.nio.file.Path tmpFile = Files.createTempFile("tmp", ".zip"); // consider using job and task IDs?
  FSDataInputStream fileIn = fs.open(file);
  FileOutputStream fileOut = new FileOutputStream(tmpFile.toFile());
  LOG.info("Copying from {} to {}", file, tmpFile);
  IOUtils.copyBytes(fileIn, fileOut, 100000, true);

  // having copied the file out of HDFS onto the local FS in a temp folder, we prepare it (sorts files)
  java.nio.file.Path tmpSpace = Files.createTempDirectory("tmp-" + context.getTaskAttemptID().getJobID().getId() +
                                                          ":" + context.getTaskAttemptID().getId());

  reader = new DwCAReader(tmpFile.toAbsolutePath().toString(), tmpSpace.toAbsolutePath().toString());
  nextKeyValue();
}
 

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
/**
 * 定义每条数据的输出格式，输入数据是由reduce任务每次执行write方法输出的数据
 */
@Override
public RecordWriter<BaseDimension, BaseStatsValueWritable> getRecordWriter(TaskAttemptContext context)
		throws IOException, InterruptedException {
	Configuration conf = context.getConfiguration();
	Connection conn = null;
	IDimensionConverter converter = new DimensionConverterImpl();
	try {
		conn = JdbcManager.getConnection(conf, GlobalConstants.WAREHOUSE_OF_REPORT);
		conn.setAutoCommit(false);
	} catch (SQLException e) {
		logger.error("获取数据库连接失败", e);
		throw new IOException("获取数据库连接失败", e);
	}
	return new TransformerRecordWriter(conn, conf, converter);
}
 

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
public OraOopDBRecordWriterUpdate(TaskAttemptContext context, int mapperId,
    UpdateMode updateMode, boolean useAppendValuesOracleHint)
    throws ClassNotFoundException, SQLException {

  super(context, mapperId);

  Configuration conf = context.getConfiguration();

  this.updateColumnNames =
      OraOopUtilities.getExportUpdateKeyColumnNames(conf);
  this.useAppendValuesOracleHint = useAppendValuesOracleHint;
  this.updateMode = updateMode;
  this.tableHasOraOopPartitions =
      conf.getBoolean(OraOopConstants.EXPORT_TABLE_HAS_ORAOOP_PARTITIONS,
          false);
}
 

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
protected void applyMapperJdbcUrl(TaskAttemptContext context, int mapperId) {

    Configuration conf = context.getConfiguration();

    // Retrieve the JDBC URL that should be used by this mapper.
    // We achieve this by modifying the JDBC URL property in the
    // configuration, prior to the OraOopDBRecordWriter's (ancestral)
    // constructor using the configuration to establish a connection
    // to the database - via DBConfiguration.getConnection()...
    String mapperJdbcUrlPropertyName =
        OraOopUtilities.getMapperJdbcUrlPropertyName(mapperId, conf);

    // Get this mapper's JDBC URL
    String mapperJdbcUrl = conf.get(mapperJdbcUrlPropertyName, null);

    LOG.debug(String.format("Mapper %d has a JDBC URL of: %s", mapperId,
        mapperJdbcUrl == null ? "<null>" : mapperJdbcUrl));

    if (mapperJdbcUrl != null) {
      conf.set(DBConfiguration.URL_PROPERTY, mapperJdbcUrl);
    }
  }
 

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
public SQLServerExportRecordWriter(TaskAttemptContext context)
    throws IOException {
  conf = context.getConfiguration();

  recordsPerStmt = conf.getInt(
    AsyncSqlOutputFormat.RECORDS_PER_STATEMENT_KEY,
    DEFAULT_RECORDS_PER_STATEMENT);

  // Create the lists to host incoming records
  List<SqoopRecord> newList;
  for (int i = 0; i < LIST_COUNT; ++i) {
    newList = new ArrayList<SqoopRecord>(recordsPerStmt);
    recordsLists.add(newList);
  }
  currentList = recordsLists.get(0);
  // Initialize the DB exec Thread
  initializeExecThread();

  // Start the DB exec thread
  execThread.start();
}
 

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
@Override
public void initialize(InputSplit split, 
                       TaskAttemptContext context
                       ) throws IOException, InterruptedException {
  FileSplit fileSplit = (FileSplit) split;
  conf = context.getConfiguration();    
  Path path = fileSplit.getPath();
  FileSystem fs = path.getFileSystem(conf);
  this.in = new SequenceFile.Reader(fs, path, conf);
  this.end = fileSplit.getStart() + fileSplit.getLength();

  if (fileSplit.getStart() > in.getPosition()) {
    in.sync(fileSplit.getStart());                  // sync to start
  }

  this.start = in.getPosition();
  more = start < end;
}
 

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
public OraOopDBRecordWriterBase(TaskAttemptContext context, int mapperId)
    throws ClassNotFoundException, SQLException {

  super(context);
  this.mapperId = mapperId;
  this.mapperRowNumber = 1;

  Configuration conf = context.getConfiguration();

  // Log any info that might be useful to us...
  logBatchSettings();

  // Connect to Oracle...
  Connection connection = this.getConnection();

  String thisOracleInstanceName =
      OraOopOracleQueries.getCurrentOracleInstanceName(connection);
  LOG.info(String.format(
      "This record writer is connected to Oracle via the JDBC URL: \n"
          + "\t\"%s\"\n" + "\tto the Oracle instance: \"%s\"", connection
          .toString(), thisOracleInstanceName));

  // Initialize the Oracle session...
  OracleConnectionFactory.initializeOracleConnection(connection, conf);
  connection.setAutoCommit(false);
}
 

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
protected void getExportTableAndColumns(TaskAttemptContext context)
    throws SQLException {

  Configuration conf = context.getConfiguration();

  String schema =
      context.getConfiguration().get(OraOopConstants.ORAOOP_TABLE_OWNER);
  String localTableName =
      context.getConfiguration().get(OraOopConstants.ORAOOP_TABLE_NAME);

  if (schema == null || schema.isEmpty() || localTableName == null
      || localTableName.isEmpty()) {
    throw new RuntimeException(
        "Unable to recall the schema and name of the Oracle table "
        + "being exported.");
  }

  this.oracleTable = new OracleTable(schema, localTableName);

  setOracleTableColumns(OraOopOracleQueries.getTableColumns(this
      .getConnection(), this.oracleTable, OraOopUtilities
      .omitLobAndLongColumnsDuringImport(conf), OraOopUtilities
      .recallSqoopJobType(conf), true // <- onlyOraOopSupportedTypes
      , false // <- omitOraOopPseudoColumns
      ));
}
 

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
public AsyncSqlRecordWriter(TaskAttemptContext context)
    throws ClassNotFoundException, SQLException {
  this.conf = context.getConfiguration();

  this.rowsPerStmt = conf.getInt(
      AsyncSqlOutputFormat.RECORDS_PER_STATEMENT_KEY,
      AsyncSqlOutputFormat.DEFAULT_RECORDS_PER_STATEMENT);
  int stmtsPerTx = conf.getInt(
      AsyncSqlOutputFormat.STATEMENTS_PER_TRANSACTION_KEY,
      AsyncSqlOutputFormat.DEFAULT_STATEMENTS_PER_TRANSACTION);

  DBConfiguration dbConf = new DBConfiguration(conf);
  this.connection = dbConf.getConnection();
  this.connection.setAutoCommit(false);

  this.records = new ArrayList<SqoopRecord>(this.rowsPerStmt);

  this.execThread = new AsyncSqlOutputFormat.AsyncSqlExecThread(
      connection, stmtsPerTx);
  this.execThread.setDaemon(true);
  this.startedExecThread = false;

  this.closed = false;
}
 

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
@Override
/** {@inheritDoc} */
public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context)
    throws IOException {
  DBConfiguration dbConf = new DBConfiguration(context.getConfiguration());
  String tableName = dbConf.getOutputTableName();
  String[] fieldNames = dbConf.getOutputFieldNames();

  if (fieldNames == null) {
    fieldNames = new String[dbConf.getOutputFieldCount()];
  }

  try {
    Connection connection = dbConf.getConnection();
    PreparedStatement statement = null;

    statement = connection.prepareStatement(
                  constructQuery(tableName, fieldNames));
    return new com.cloudera.sqoop.mapreduce.db.DBOutputFormat.DBRecordWriter(
                   connection, statement);
  } catch (Exception ex) {
    throw new IOException(ex);
  }
}
 

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
public void initialize(InputSplit split, TaskAttemptContext context) 
    throws IOException, InterruptedException {
  Path path = ((FileSplit)split).getPath();
  Configuration conf = context.getConfiguration();
  FileSystem fs = path.getFileSystem(conf);
  this.in = new SequenceFile.Reader(fs, path, conf);
  this.end = ((FileSplit)split).getStart() + split.getLength();
  if (((FileSplit)split).getStart() > in.getPosition()) {
    in.sync(((FileSplit)split).getStart());    // sync to start
  }
  this.start = in.getPosition();
  vbytes = in.createValueBytes();
  done = start >= end;
}
 

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
@Override
public void initialize(InputSplit inputSplit,
    TaskAttemptContext taskAttemptContext)
    throws IOException, InterruptedException {

  split = (MainframeDatasetInputSplit)inputSplit;
  conf = taskAttemptContext.getConfiguration();
  inputClass = (Class<T>) (conf.getClass(
              DBConfiguration.INPUT_CLASS_PROPERTY, null));
  key = null;
  datasetRecord = null;
  numberRecordRead = 0;
  datasetProcessed = 0;
}
 

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
@Override
public void initialize(InputSplit genericSplit, TaskAttemptContext ctxt)
throws IOException, InterruptedException {
  final LoadSplit split = (LoadSplit)genericSplit;
  final Configuration conf = ctxt.getConfiguration();
  factory = 
    new ReadRecordFactory(split.getLength(), split.getInputRecords(), 
                          new FileQueue(split, conf), conf);
}
 

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
/**
 * Implementation for RecordReader::initialize(). Initializes the internal RecordReader to read from chunks.
 *
 * @param inputSplit The InputSplit for the map. Ignored entirely.
 * @param taskAttemptContext The AttemptContext.
 * @throws IOException, on failure.
 * @throws InterruptedException
 */
@Override
public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
  throws IOException, InterruptedException {
  numRecordsPerChunk = DynamicInputFormat.getNumEntriesPerChunk(taskAttemptContext.getConfiguration());
  this.taskAttemptContext = taskAttemptContext;
  configuration = taskAttemptContext.getConfiguration();
  taskId = taskAttemptContext.getTaskAttemptID().getTaskID();
  chunk = DynamicInputChunk.acquire(this.taskAttemptContext);
  timeOfLastChunkDirScan = System.currentTimeMillis();
  isChunkDirAlreadyScanned = false;

  totalNumRecords = getTotalNumRecords();

}
 

import org.apache.hadoop.mapreduce.TaskAttemptContext; //导入方法依赖的package包/类
@Override
protected void getExportTableAndColumns(TaskAttemptContext context)
    throws SQLException {

  Configuration conf = context.getConfiguration();

  this.oracleTable = createUniqueMapperTable(context);
  setOracleTableColumns(OraOopOracleQueries.getTableColumns(this
      .getConnection(), this.oracleTable, OraOopUtilities
      .omitLobAndLongColumnsDuringImport(conf), OraOopUtilities
      .recallSqoopJobType(conf), true // <- onlyOraOopSupportedTypes
      , false) // <- omitOraOopPseudoColumns
  );
}