Java LoadMetadata类代码示例

import org.apache.pig.LoadMetadata; //导入依赖的package包/类
/**
 * Get the total input size in bytes by looking at statistics provided by
 * loaders that implement @{link LoadMetadata}.
 * @param ld
 * @param job
 * @return total input size in bytes, or -1 if unknown or incomplete
 * @throws IOException on error
 */
static long getInputSizeFromLoader(POLoad ld, Job job) throws IOException {
    if (ld.getLoadFunc() == null
            || !(ld.getLoadFunc() instanceof LoadMetadata)
            || ld.getLFile() == null
            || ld.getLFile().getFileName() == null) {
        return -1;
    }

    ResourceStatistics statistics;
    try {
        statistics = ((LoadMetadata) ld.getLoadFunc())
                    .getStatistics(ld.getLFile().getFileName(), job);
    } catch (Exception e) {
        log.warn("Couldn't get statistics from LoadFunc: " + ld.getLoadFunc(), e);
        return -1;
    }

    if (statistics == null || statistics.getSizeInBytes() == null) {
        return -1;
    }

    return statistics.getSizeInBytes();
}
 

import org.apache.pig.LoadMetadata; //导入依赖的package包/类
private LogicalSchema getSchemaFromMetaData() throws FrontendException {
    if (getLoadFunc()!=null && getLoadFunc() instanceof LoadMetadata) {
        try {
            ResourceSchema resourceSchema = ((LoadMetadata)loadFunc).getSchema(getFileSpec().getFileName(), new Job(conf));
            Schema oldSchema = Schema.getPigSchema(resourceSchema);
            return Util.translateSchema(oldSchema);
        } catch (IOException e) {
            throw new FrontendException( this, "Cannot get schema from loadFunc " + loadFunc.getClass().getName(), 2245, e);
        }
    }
    return null;
}
 

import org.apache.pig.LoadMetadata; //导入依赖的package包/类
@Override
        public boolean check(OperatorPlan matched) throws FrontendException {
            loLoad = (LOLoad)matched.getSources().get(0);
            // Match filter.
            List<Operator> succeds = currentPlan.getSuccessors( loLoad );
            if( succeds == null || succeds.size() == 0 || !( succeds.get(0) instanceof LOFilter ) )
                return false;
            loFilter =  (LOFilter)succeds.get(0);
            
            // Filter has dependency other than load, skip optimization
            if (currentPlan.getSoftLinkPredecessors(loFilter)!=null)
                return false;
            
            // we have to check more only if LoadFunc implements LoadMetada
            loadFunc = loLoad.getLoadFunc();
            if(!( loadFunc instanceof LoadMetadata ) ) {
                return false;
            }
            
            loadMetadata = (LoadMetadata)loadFunc;
            try {
				partitionKeys = loadMetadata.getPartitionKeys(
						loLoad.getFileSpec().getFileName(), new Job( loLoad.getConfiguration() ) );
			} catch (IOException e) {
				throw new FrontendException( e );
			}
            if( partitionKeys == null || partitionKeys.length == 0 ) {
            	return false;
            }
            
//            LogicalExpressionPlan filterExpr = filter.getFilterPlan();
            
            // we found a load-filter pattern where the load returns partition keys
            return true;
        }
 

import org.apache.pig.LoadMetadata; //导入依赖的package包/类
@Override
public ResourceSchema getSchema(String location, Job job) throws IOException {
    if (wrappedLoadFunc instanceof LoadMetadata) {
        return ((LoadMetadata) wrappedLoadFunc).getSchema(location, job);
    } else {
        return null;
    }
}
 

import org.apache.pig.LoadMetadata; //导入依赖的package包/类
@Override
public ResourceStatistics getStatistics(String location, Job job) throws IOException {
    if (wrappedLoadFunc instanceof LoadMetadata) {
        return ((LoadMetadata) wrappedLoadFunc).getStatistics(location, job);
    } else {
        return null;
    }
}
 

import org.apache.pig.LoadMetadata; //导入依赖的package包/类
@Override
public String[] getPartitionKeys(String location, Job job) throws IOException {
    if (wrappedLoadFunc instanceof LoadMetadata) {
        return ((LoadMetadata) wrappedLoadFunc).getPartitionKeys(location, job);
    } else {
        return null;
    }
}
 

import org.apache.pig.LoadMetadata; //导入依赖的package包/类
@Override
     public boolean check(OperatorPlan matched) throws FrontendException {
         loLoad = (LOLoad)matched.getSources().get(0);
         // Match filter.
         List<Operator> succeds = currentPlan.getSuccessors( loLoad );
         if( succeds == null || succeds.size() == 0 || !( succeds.get(0) instanceof LOFilter ) )
             return false;
         loFilter =  (LOFilter)succeds.get(0);

         // Filter has dependency other than load, skip optimization
         if (currentPlan.getSoftLinkPredecessors(loFilter)!=null)
             return false;

         // we have to check more only if LoadFunc implements LoadMetada
         loadFunc = loLoad.getLoadFunc();
         if(!( loadFunc instanceof LoadMetadata ) ) {
             return false;
         }

         loadMetadata = (LoadMetadata)loadFunc;
         try {
	partitionKeys = loadMetadata.getPartitionKeys(
			loLoad.getFileSpec().getFileName(), new Job( loLoad.getConfiguration() ) );
} catch (IOException e) {
	throw new FrontendException( e );
}
         if( partitionKeys == null || partitionKeys.length == 0 ) {
             return false;
         }

         return true;
     }
 

import org.apache.pig.LoadMetadata; //导入依赖的package包/类
@Override
public void setPartitionFilter(Expression partitionFilter) throws IOException {
    if (wrappedLoadFunc instanceof LoadMetadata) {
         ((LoadMetadata) wrappedLoadFunc).setPartitionFilter(partitionFilter);
    }
}
 

import org.apache.pig.LoadMetadata; //导入依赖的package包/类
private void init(LoadMetadata loadMetadata) {
    setLoadFunc(loadMetadata);
}