本文整理汇总了Java中org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil.mergeConf方法的典型用法代码示例。如果您正苦于以下问题:Java ConfigurationUtil.mergeConf方法的具体用法?Java ConfigurationUtil.mergeConf怎么用?Java ConfigurationUtil.mergeConf使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil
的用法示例。
在下文中一共展示了ConfigurationUtil.mergeConf方法的10个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: initialize
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; //导入方法依赖的package包/类
@Override
public void initialize(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
// initialize the underlying actual RecordReader with the right Context
// object - this is achieved by merging the Context corresponding to
// the input split this Reader is supposed to process with the context
// passed in.
this.pigSplit = (PigSplit)split;
this.context = context;
ConfigurationUtil.mergeConf(context.getConfiguration(),
inputSpecificConf);
// Pass loader signature to LoadFunc and to InputFormat through
// the conf
PigInputFormat.passLoadSignature(loadfunc, pigSplit.getInputIndex(),
context.getConfiguration());
// now invoke initialize() on underlying RecordReader with
// the "adjusted" conf
if (null != curReader) {
curReader.initialize(pigSplit.getWrappedSplit(), context);
loadfunc.prepareToRead(curReader, pigSplit);
}
if (pigSplit.isMultiInputs() && !pigSplit.disableCounter()) {
counterName = getMultiInputsCounerName(pigSplit, inputSpecificConf);
}
}
示例2: setLocation
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; //导入方法依赖的package包/类
/**
* Before delegating calls to underlying OutputFormat or OutputCommitter
* Pig needs to ensure the Configuration in the JobContext contains
* the output location and StoreFunc
* for the specific store - so set these up in the context for this specific
* store
* @param jobContext the {@link JobContext}
* @param store the POStore
* @throws IOException on failure
*/
public static void setLocation(JobContext jobContext, POStore store) throws
IOException {
Job storeJob = new Job(jobContext.getConfiguration());
StoreFuncInterface storeFunc = store.getStoreFunc();
String outputLocation = store.getSFile().getFileName();
storeFunc.setStoreLocation(outputLocation, storeJob);
// the setStoreLocation() method would indicate to the StoreFunc
// to set the output location for its underlying OutputFormat.
// Typically OutputFormat's store the output location in the
// Configuration - so we need to get the modified Configuration
// containing the output location (and any other settings the
// OutputFormat might have set) and merge it with the Configuration
// we started with so that when this method returns the Configuration
// supplied as input has the updates.
ConfigurationUtil.mergeConf(jobContext.getConfiguration(),
storeJob.getConfiguration());
}
示例3: setLocation
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; //导入方法依赖的package包/类
/**
* Before delegating calls to underlying OutputFormat or OutputCommitter
* Pig needs to ensure the Configuration in the JobContext contains
* the output location and StoreFunc
* for the specific store - so set these up in the context for this specific
* store
* @param jobContext the {@link JobContext}
* @param store the POStore
* @throws IOException on failure
*/
public static void setLocation(JobContext jobContext, POStore store) throws
IOException {
Job storeJob = new Job(jobContext.getConfiguration());
StoreFuncInterface storeFunc = store.getStoreFunc();
String outputLocation = store.getSFile().getFileName();
storeFunc.setStoreLocation(outputLocation, storeJob);
// the setStoreLocation() method would indicate to the StoreFunc
// to set the output location for its underlying OutputFormat.
// Typically OutputFormat's store the output location in the
// Configuration - so we need to get the modified Configuration
// containing the output location (and any other settings the
// OutputFormat might have set) and merge it with the Configuration
// we started with so that when this method returns the Configuration
// supplied as input has the updates.
ConfigurationUtil.mergeConf(jobContext.getConfiguration(),
storeJob.getConfiguration());
}
示例4: mergeSplitSpecificConf
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; //导入方法依赖的package包/类
/**
* get the corresponding configuration for the input on which the split
* is based and merge it with the Conf supplied
*
* package level access so that this is not publicly used elsewhere
* @throws IOException
*/
static void mergeSplitSpecificConf(LoadFunc loadFunc, PigSplit pigSplit, Configuration originalConf)
throws IOException {
// set up conf with entries from input specific conf
Job job = new Job(originalConf);
loadFunc.setLocation(getLoadLocation(pigSplit.getInputIndex(),
originalConf), job);
// The above setLocation call could write to the conf within
// the job - merge that updated conf with original conf
ConfigurationUtil.mergeConf(originalConf, job.getConfiguration());
}
示例5: setHdfsServers
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; //导入方法依赖的package包/类
static void setHdfsServers(String absolutePath, PigContext pigContext) throws URISyntaxException {
// Get native host
String defaultFS = (String)pigContext.getProperties().get("fs.default.name");
if (defaultFS==null)
defaultFS = (String)pigContext.getProperties().get("fs.defaultFS");
URI defaultFSURI = new URI(defaultFS);
Configuration conf = new Configuration(true);
ConfigurationUtil.mergeConf(conf, ConfigurationUtil.toConfiguration(pigContext.getProperties()));
Set<String> remoteHosts = getRemoteHosts(absolutePath, defaultFSURI, conf);
String hdfsServersString = (String)pigContext.getProperties().get(MRConfiguration.JOB_HDFS_SERVERS);
if (hdfsServersString == null) hdfsServersString = "";
String hdfsServers[] = hdfsServersString.split(",");
for (String remoteHost : remoteHosts) {
boolean existing = false;
for (String hdfsServer : hdfsServers) {
if (hdfsServer.equals(remoteHost)) {
existing = true;
}
}
if (!existing) {
if (!hdfsServersString.isEmpty()) {
hdfsServersString = hdfsServersString + ",";
}
hdfsServersString = hdfsServersString + remoteHost;
}
}
if (!hdfsServersString.isEmpty()) {
pigContext.getProperties().setProperty(MRConfiguration.JOB_HDFS_SERVERS, hdfsServersString);
}
}
示例6: mergeSplitSpecificConf
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; //导入方法依赖的package包/类
/**
* get the corresponding configuration for the input on which the split
* is based and merge it with the Conf supplied
*
* package level access so that this is not publicly used elsewhere
* @throws IOException
*/
static void mergeSplitSpecificConf(LoadFunc loadFunc, PigSplit pigSplit, Configuration originalConf)
throws IOException {
// set up conf with entries from input specific conf
Job job = new Job(originalConf);
loadFunc.setLocation(getLoadLocation(pigSplit.getInputIndex(),
originalConf), job);
// The above setLocation call could write to the conf within
// the job - merge that updated conf with original conf
ConfigurationUtil.mergeConf(originalConf, job.getConfiguration());
}
示例7: initialize
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; //导入方法依赖的package包/类
@Override
public void initialize(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
// initialize the underlying actual RecordReader with the right Context
// object - this is achieved by merging the Context corresponding to
// the input split this Reader is supposed to process with the context
// passed in.
this.pigSplit = (PigSplit)split;
this.context = context;
ConfigurationUtil.mergeConf(context.getConfiguration(),
inputSpecificConf);
// Pass loader signature to LoadFunc and to InputFormat through
// the conf
PigInputFormat.passLoadSignature(loadfunc, pigSplit.getInputIndex(),
context.getConfiguration());
// now invoke initialize() on underlying RecordReader with
// the "adjusted" conf
if (null != curReader) {
curReader.initialize(pigSplit.getWrappedSplit(), context);
loadfunc.prepareToRead(curReader, pigSplit);
}
if (pigSplit.isMultiInputs() && !pigSplit.disableCounter()) {
counterName = getMultiInputsCounerName(pigSplit, inputSpecificConf);
if (counterName != null) {
// Create the counter. This is needed because incrCounter() may
// never be called in case of empty file.
reporter.incrCounter(PigStatsUtil.MULTI_INPUTS_COUNTER_GROUP, counterName, 0);
}
}
}
示例8: fetchFilesInternal
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; //导入方法依赖的package包/类
/**
* Copies the files from remote to local filesystem.
* When 'multipleFiles' is set the path could point to multiple files
* through globs or a directory. In this case, return array contains multiple
* files, otherwise a single file is returned.
*
* If pig.jars.relative.to.dfs is true then a relative path is assumed to be
* relative to the default filesystem's active directory.
* Else they are assumed to be relative to the local working directory.
*
* @param properties
* @param filePath
* @param multipleFiles
* @return
*/
private static FetchFileRet[] fetchFilesInternal(Properties properties,
String filePath,
boolean multipleFiles) throws IOException {
Path path = new Path(filePath);
URI uri = path.toUri();
Configuration conf = new Configuration();
ConfigurationUtil.mergeConf(conf, ConfigurationUtil.toConfiguration(properties));
// if there is no schema or if the schema is "local", then it is
// expected to be a local path.
FileSystem localFs = FileSystem.getLocal(conf);
FileSystem srcFs;
if ( (!"true".equals(properties.getProperty("pig.jars.relative.to.dfs"))
&& uri.getScheme() == null )||
uri.getScheme().equals("local") ) {
srcFs = localFs;
} else {
srcFs = path.getFileSystem(conf);
}
FileStatus[] files;
if (multipleFiles) {
files = srcFs.globStatus(path);
} else {
files = new FileStatus[]{ srcFs.getFileStatus(path) };
}
if (files == null || files.length == 0) {
throw new ExecException("file '" + filePath + "' does not exist.", 101, PigException.INPUT);
}
FetchFileRet[] fetchFiles = new FetchFileRet[files.length];
int idx = 0;
for(FileStatus file : files) {
// should throw an exception if this is not a file?
String pathname = file.getPath().toUri().getPath();
String filename = file.getPath().getName();
if (srcFs == localFs) {
fetchFiles[idx++] = new FetchFileRet(new File(pathname), false);
} else {
// fetch from remote:
File dest = new File(localTempDir, filename);
dest.deleteOnExit();
try {
srcFs.copyToLocalFile(file.getPath(), new Path(dest.getAbsolutePath()));
} catch (IOException e) {
throw new ExecException("Could not copy " + filePath + " to local destination " + dest, 101, PigException.INPUT, e);
}
fetchFiles[idx++] = new FetchFileRet(dest, true);
}
}
return fetchFiles;
}
示例9: fetchFilesInternal
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; //导入方法依赖的package包/类
/**
* Copies the files from remote to local filesystem.
* When 'multipleFiles' is set the path could point to multiple files
* through globs or a directory. In this case, return array contains multiple
* files, otherwise a single file is returned.
*
* If pig.jars.relative.to.dfs is true then a relative path is assumed to be
* relative to the default filesystem's active directory.
* Else they are assumed to be relative to the local working directory.
*
* @param properties
* @param filePath
* @param multipleFiles
* @return
*/
private static FetchFileRet[] fetchFilesInternal(Properties properties,
String filePath,
boolean multipleFiles) throws IOException {
Path path = new Path(filePath);
if (path.getName().isEmpty()) {
return new FetchFileRet[0];
}
URI uri = path.toUri();
Configuration conf = new Configuration();
ConfigurationUtil.mergeConf(conf, ConfigurationUtil.toConfiguration(properties));
// if there is no schema or if the schema is "local", then it is
// expected to be a local path.
FileSystem localFs = FileSystem.getLocal(conf);
FileSystem srcFs;
if ( (!"true".equals(properties.getProperty("pig.jars.relative.to.dfs"))
&& uri.getScheme() == null )||
// For Windows local files
(uri.getScheme() == null && uri.getPath().matches("^/[A-Za-z]:.*")) ||
(uri.getScheme() != null && uri.getScheme().equals("local"))
) {
srcFs = localFs;
} else {
srcFs = path.getFileSystem(conf);
}
FileStatus[] files;
if (multipleFiles) {
files = srcFs.globStatus(path);
} else {
files = new FileStatus[]{ srcFs.getFileStatus(path) };
}
if (files == null || files.length == 0) {
throw new ExecException("file '" + filePath + "' does not exist.", 101, PigException.INPUT);
}
FetchFileRet[] fetchFiles = new FetchFileRet[files.length];
int idx = 0;
for(FileStatus file : files) {
// should throw an exception if this is not a file?
String pathname = file.getPath().toUri().getPath();
String filename = file.getPath().getName();
if (srcFs == localFs) {
fetchFiles[idx++] = new FetchFileRet(new File(pathname), false);
} else {
// fetch from remote:
File dest = new File(localTempDir, filename);
dest.deleteOnExit();
try {
srcFs.copyToLocalFile(file.getPath(), new Path(dest.getAbsolutePath()));
} catch (IOException e) {
throw new ExecException("Could not copy " + filePath + " to local destination " + dest, 101, PigException.INPUT, e);
}
fetchFiles[idx++] = new FetchFileRet(dest, true);
}
}
return fetchFiles;
}
示例10: init
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; //导入方法依赖的package包/类
@SuppressWarnings("resource")
private void init(Properties properties) throws ExecException {
String cluster = null;
String nameNode = null;
// We need to build a configuration object first in the manner described
// below and then get back a properties object to inspect the
// JOB_TRACKER_LOCATION and FILE_SYSTEM_LOCATION. The reason to do this
// is if we looked only at the existing properties object, we may not
// get the right settings. So we want to read the configurations in the
// order specified below and only then look for JOB_TRACKER_LOCATION and
// FILE_SYSTEM_LOCATION.
// Hadoop by default specifies two resources, loaded in-order from the
// classpath:
// 1. hadoop-default.xml : Read-only defaults for hadoop.
// 2. hadoop-site.xml: Site-specific configuration for a given hadoop
// installation.
// Now add the settings from "properties" object to override any
// existing properties All of the above is accomplished in the method
// call below
JobConf jc = getS3Conf();
if (!this.pigContext.getExecType().isLocal()) {
JobConf execConf = getExecConf(properties);
ConfigurationUtil.mergeConf(jc, execConf);
// Trick to invoke static initializer of DistributedFileSystem to
// add hdfs-default.xml into configuration
new DistributedFileSystem();
} else {
// If we are running in local mode we dont read the hadoop conf file
if (properties.getProperty(MRConfiguration.FRAMEWORK_NAME) == null) {
properties.setProperty(MRConfiguration.FRAMEWORK_NAME, LOCAL);
}
properties.setProperty(MRConfiguration.JOB_TRACKER, LOCAL);
properties.setProperty(FILE_SYSTEM_LOCATION, "file:///");
properties.setProperty(ALTERNATIVE_FILE_SYSTEM_LOCATION, "file:///");
JobConf localConf = getLocalConf();
ConfigurationUtil.mergeConf(jc, localConf);
}
// the method below alters the properties object by overriding the
// hadoop properties with the values from properties and recomputing
// the properties
Utils.recomputeProperties(jc, properties);
cluster = jc.get(MRConfiguration.JOB_TRACKER);
nameNode = jc.get(FILE_SYSTEM_LOCATION);
if (nameNode == null) {
nameNode = (String) pigContext.getProperties().get(ALTERNATIVE_FILE_SYSTEM_LOCATION);
}
if (cluster != null && cluster.length() > 0) {
if (!cluster.contains(":") && !cluster.equalsIgnoreCase(LOCAL)) {
cluster = cluster + ":50020";
}
properties.setProperty(MRConfiguration.JOB_TRACKER, cluster);
}
if (nameNode != null && nameNode.length() > 0) {
if (!nameNode.contains(":") && !nameNode.equalsIgnoreCase(LOCAL)) {
nameNode = nameNode + ":8020";
}
properties.setProperty(FILE_SYSTEM_LOCATION, nameNode);
}
LOG.info("Connecting to hadoop file system at: "
+ (nameNode == null ? LOCAL : nameNode));
// constructor sets DEFAULT_REPLICATION_FACTOR_KEY
ds = new HDataStorage(properties);
if (cluster != null && !cluster.equalsIgnoreCase(LOCAL)) {
LOG.info("Connecting to map-reduce job tracker at: "
+ jc.get(MRConfiguration.JOB_TRACKER));
}
}