Java FeedHelper.getCluster方法代码示例

import org.apache.falcon.entity.FeedHelper; //导入方法依赖的package包/类
private static List<Referenceable> getInputEntities(org.apache.falcon.entity.v0.cluster.Cluster cluster,
                                                    Feed feed) throws Exception {
    org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName());

    if(feedCluster != null) {
        final CatalogTable table = getTable(feedCluster, feed);
        if (table != null) {
            CatalogStorage storage = new CatalogStorage(cluster, table);
            return createHiveTableInstance(cluster.getName(), storage.getDatabase().toLowerCase(),
                    storage.getTable().toLowerCase());
        } else {
            List<Location> locations = FeedHelper.getLocations(feedCluster, feed);
            if (CollectionUtils.isNotEmpty(locations)) {
                Location dataLocation = FileSystemStorage.getLocation(locations, LocationType.DATA);
                if (dataLocation != null) {
                    final String pathUri = normalize(dataLocation.getPath());
                    LOG.info("Registering DFS Path {} ", pathUri);
                    return fillHDFSDataSet(pathUri, cluster.getName());
                }
            }
        }
    }

    return null;
}
 

import org.apache.falcon.entity.FeedHelper; //导入方法依赖的package包/类
@Override
public Properties newWorkflowSchedule(Feed feed, Date startDate, String clusterName, String user)
    throws FalconException {

    org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, clusterName);
    if (!startDate.before(feedCluster.getValidity().getEnd())) {
        return null;
    }

    Cluster cluster = CONFIG_STORE.get(EntityType.CLUSTER, feedCluster.getName());
    Path bundlePath = new Path(ClusterHelper.getLocation(cluster, "staging"), EntityUtil.getStagingPath(feed));
    Feed feedClone = (Feed) feed.copy();
    EntityUtil.setStartDate(feedClone, clusterName, startDate);

    AbstractOozieEntityMapper<Feed> mapper = new OozieFeedMapper(feedClone);
    if (!mapper.map(cluster, bundlePath)) {
        return null;
    }
    return createAppProperties(clusterName, bundlePath, user);
}
 

import org.apache.falcon.entity.FeedHelper; //导入方法依赖的package包/类
private SYNCDATASET createDataSet(String feedName, Cluster cluster, String datasetName, LocationType locationType)
        throws FalconException {
    Feed feed = (Feed) EntityUtil.getEntity(EntityType.FEED, feedName);

    SYNCDATASET syncdataset = new SYNCDATASET();
    syncdataset.setName(datasetName);
    String locPath = FeedHelper.getLocation(feed, locationType,
            cluster.getName()).getPath();
    syncdataset.setUriTemplate(new Path(locPath).toUri().getScheme() != null ? locPath : "${nameNode}"
            + locPath);
    syncdataset.setFrequency("${coord:" + feed.getFrequency().toString() + "}");

    org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName());
    syncdataset.setInitialInstance(SchemaHelper.formatDateUTC(feedCluster.getValidity().getStart()));
    syncdataset.setTimezone(feed.getTimezone().getID());
    if (feed.getAvailabilityFlag() == null) {
        syncdataset.setDoneFlag("");
    } else {
        syncdataset.setDoneFlag(feed.getAvailabilityFlag());
    }
    return syncdataset;
}
 

import org.apache.falcon.entity.FeedHelper; //导入方法依赖的package包/类
private void addOptionalInputProperties(Properties properties, Input in, String clusterName)
        throws FalconException {
    Feed feed = EntityUtil.getEntity(EntityType.FEED, in.getFeed());
    org.apache.falcon.entity.v0.feed.Cluster cluster = FeedHelper.getCluster(feed, clusterName);
    String inName = in.getName();
    properties.put(inName + ".frequency", String.valueOf(feed.getFrequency().getFrequency()));
    properties.put(inName + ".freq_timeunit", mapToCoordTimeUnit(feed.getFrequency().getTimeUnit()).name());
    properties.put(inName + ".timezone", feed.getTimezone().getID());
    properties.put(inName + ".end_of_duration", Timeunit.NONE.name());
    properties.put(inName + ".initial-instance", SchemaHelper.formatDateUTC(cluster.getValidity().getStart()));
    properties.put(inName + ".done-flag", "notused");

    String locPath = FeedHelper.getLocation(feed, LocationType.DATA, clusterName).getPath().replace('$', '%');
    properties.put(inName + ".uri-template",
            new Path(locPath).toUri().getScheme() != null ? locPath : "${nameNode}" + locPath);

    properties.put(inName + ".start-instance", in.getStart());
    properties.put(inName + ".end-instance", in.getEnd());
}
 

import org.apache.falcon.entity.FeedHelper; //导入方法依赖的package包/类
private COORDINATORAPP getRetentionCoordinator(Cluster cluster, Path bundlePath) throws FalconException {

        Feed feed = getEntity();
        org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName());

        if (feedCluster.getValidity().getEnd().before(new Date())) {
            LOG.warn("Feed Retention is not applicable as Feed's end time for cluster " + cluster.getName()
                    + " is not in the future");
            return null;
        }
        COORDINATORAPP retentionApp = new COORDINATORAPP();
        String coordName = EntityUtil.getWorkflowName(Tag.RETENTION, feed).toString();
        retentionApp.setName(coordName);
        retentionApp.setEnd(SchemaHelper.formatDateUTC(feedCluster.getValidity().getEnd()));
        retentionApp.setStart(SchemaHelper.formatDateUTC(new Date()));
        retentionApp.setTimezone(feed.getTimezone().getID());
        TimeUnit timeUnit = feed.getFrequency().getTimeUnit();
        if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) {
            retentionApp.setFrequency("${coord:hours(6)}");
        } else {
            retentionApp.setFrequency("${coord:days(1)}");
        }

        Path wfPath = getCoordPath(bundlePath, coordName);
        retentionApp.setAction(getRetentionWorkflowAction(cluster, wfPath, coordName));
        return retentionApp;
    }
 

import org.apache.falcon.entity.FeedHelper; //导入方法依赖的package包/类
private ACTION getRetentionWorkflowAction(Cluster cluster, Path wfPath, String wfName) throws FalconException {
    Feed feed = getEntity();
    ACTION retentionAction = new ACTION();
    WORKFLOW retentionWorkflow = new WORKFLOW();
    try {
        //
        WORKFLOWAPP retWfApp = createRetentionWorkflow(cluster);
        retWfApp.setName(wfName);
        marshal(cluster, retWfApp, wfPath);
        retentionWorkflow.setAppPath(getStoragePath(wfPath.toString()));

        Map<String, String> props = createCoordDefaultConfiguration(cluster, wfPath, wfName);

        org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName());
        String feedPathMask = getLocationURI(cluster, feed, LocationType.DATA);
        String metaPathMask = getLocationURI(cluster, feed, LocationType.META);
        String statsPathMask = getLocationURI(cluster, feed, LocationType.STATS);
        String tmpPathMask = getLocationURI(cluster, feed, LocationType.TMP);

        StringBuilder feedBasePaths = new StringBuilder(feedPathMask);
        if (metaPathMask != null) {
            feedBasePaths.append(FEED_PATH_SEP).append(metaPathMask);
        }
        if (statsPathMask != null) {
            feedBasePaths.append(FEED_PATH_SEP).append(statsPathMask);
        }
        if (tmpPathMask != null) {
            feedBasePaths.append(FEED_PATH_SEP).append(tmpPathMask);
        }

        props.put("feedDataPath", feedBasePaths.toString().replaceAll("\\$\\{", "\\?\\{"));
        props.put("timeZone", feed.getTimezone().getID());
        props.put("frequency", feed.getFrequency().getTimeUnit().name());
        props.put("limit", feedCluster.getRetention().getLimit().toString());
        props.put(ARG.operation.getPropName(), EntityOps.DELETE.name());
        props.put(ARG.feedNames.getPropName(), feed.getName());
        props.put(ARG.feedInstancePaths.getPropName(), "IGNORE");

        retentionWorkflow.setConfiguration(getCoordConfig(props));
        retentionAction.setWorkflow(retentionWorkflow);
        return retentionAction;
    } catch (Exception e) {
        throw new FalconException("Unable to create parent/retention workflow", e);
    }
}
 

import org.apache.falcon.entity.FeedHelper; //导入方法依赖的package包/类
@Override
public Date getNextStartTime(Feed feed, String cluster, Date now) throws FalconException {
    org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster);
    return EntityUtil.getNextStartTime(feedCluster.getValidity().getStart(),
            feed.getFrequency(), feed.getTimezone(), now);
}
 

import org.apache.falcon.entity.FeedHelper; //导入方法依赖的package包/类
public static void validateFeedDefinedForCluster(Feed feed, String clusterName) throws FalconException {
    if (FeedHelper.getCluster(feed, clusterName) == null) {
        throw new ValidationException("Feed " + feed.getName() + " is not defined for cluster " + clusterName);
    }
}