Java GroupBy类代码示例

import cascading.pipe.GroupBy; //导入依赖的package包/类
@Override
public void onFields(@SuppressWarnings("rawtypes") Comparable... fields) {
    if(sortComparators != null && sortFields != null) {
        sortFields.setComparators(sortComparators);
    }
    // apply
    pipeWrapper.set(new GroupBy(pipeWrapper.get(), PipeBuilder.getSelector(fields), sortFields, reverseOrder));

    // Optionally set the number of reducers
    if (numberOfReducers > 0) {
        pipeWrapper.get().getStepConfigDef().setProperty("mapred.reduce.tasks", String.valueOf(numberOfReducers));
    }

    // Optionally execute the callbacks AFTER the groupBy
    callbacksSupport.applyToAllCallbacks(pipeWrapper);
}
 

import cascading.pipe.GroupBy; //导入依赖的package包/类
public BoundaryGroupByMatcher() {
	super(
			(new ExpressionGraph()).
					arc(
							new TypeExpression(ElementCapture.Primary, Boundary.class, TypeExpression.Topo.LinearOut),
							ScopeExpression.ALL,
							new TypeExpression(ElementCapture.Secondary, GroupBy.class, TypeExpression.Topo.LinearIn)
			)
	);
}
 

import cascading.pipe.GroupBy; //导入依赖的package包/类
@Override
public void open(Configuration config) {

	this.calledPrepare = false;

	try {

		currentProcess = new FlinkFlowProcess(FlinkConfigConverter.toHadoopConfig(config), getRuntimeContext(), flowNode.getID());

		Set<FlowElement> sources = flowNode.getSourceElements();
		if(sources.size() != 1) {
			throw new RuntimeException("FlowNode for GroupByReducer may only have a single source");
		}
		FlowElement sourceElement = sources.iterator().next();
		if(!(sourceElement instanceof GroupBy)) {
			throw new RuntimeException("Source of GroupByReducer must be a GroupBy");
		}
		GroupBy source = (GroupBy)sourceElement;

		streamGraph = new GroupByStreamGraph( currentProcess, flowNode, source );
		groupSource = this.streamGraph.getGroupSource();

		for( Duct head : streamGraph.getHeads() ) {
			LOG.info("sourcing from: " + ((ElementDuct) head).getFlowElement());
		}

		for( Duct tail : streamGraph.getTails() ) {
			LOG.info("sinking to: " + ((ElementDuct) tail).getFlowElement());
		}
	}
	catch( Throwable throwable ) {

		if( throwable instanceof CascadingException) {
			throw (CascadingException) throwable;
		}

		throw new FlowException( "internal error during GroupByReducer configuration", throwable );
	}

}
 

import cascading.pipe.GroupBy; //导入依赖的package包/类
private DataSet<Tuple> translateGroupBy(DataSet<Tuple> input, FlowNode node, int dop) {

		GroupBy groupBy = (GroupBy) node.getSourceElements().iterator().next();

		Scope outScope = getOutScope(node);
		List<Scope> inScopes = getInputScopes(node, groupBy);

		Fields outFields;
		if(outScope.isEvery()) {
			outFields = outScope.getOutGroupingFields();
		}
		else {
			outFields = outScope.getOutValuesFields();
		}
		registerKryoTypes(outFields);

		// get input scope
		Scope inScope = inScopes.get(0);

		// get grouping keys
		Fields groupKeyFields = groupBy.getKeySelectors().get(inScope.getName());
		// get group sorting keys
		Fields sortKeyFields = groupBy.getSortingSelectors().get(inScope.getName());

		String[] groupKeys = registerKeyFields(input, groupKeyFields);
		String[] sortKeys = null;
		if (sortKeyFields != null) {
			sortKeys = registerKeyFields(input, sortKeyFields);
		}
		Order sortOrder = groupBy.isSortReversed() ? Order.DESCENDING : Order.ASCENDING;

		if(sortOrder == Order.DESCENDING) {
			// translate groupBy with inverse sort order
			return translateInverseSortedGroupBy(input, node, dop, groupKeys, sortKeys, outFields);
		}
		else if(groupKeys == null || groupKeys.length == 0) {
			// translate key-less (global) groupBy
			return translateGlobalGroupBy(input, node, dop, sortKeys, sortOrder, outFields);
		}
		else {

			UnsortedGrouping<Tuple> grouping = input
					.groupBy(groupKeys);

			if(sortKeys != null && sortKeys.length > 0) {
				// translate groupBy with group sorting

				SortedGrouping<Tuple> sortedGrouping = grouping
						.sortGroup(sortKeys[0], Order.ASCENDING);
				for(int i=1; i<sortKeys.length; i++) {
					sortedGrouping = sortedGrouping
							.sortGroup(sortKeys[i], Order.DESCENDING);
				}

				return sortedGrouping
						.reduceGroup(new GroupByReducer(node))
						.returns(new TupleTypeInfo(outFields))
						.withParameters(this.getFlinkNodeConfig(node))
						.setParallelism(dop)
						.name("reduce-" + node.getID());
			}
			else {
				// translate groupBy without group sorting

				return grouping
						.reduceGroup(new GroupByReducer(node))
						.returns(new TupleTypeInfo(outFields))
						.withParameters(this.getFlinkNodeConfig(node))
						.setParallelism(dop)
						.name("reduce-" + node.getID());
			}
		}

	}
 

import cascading.pipe.GroupBy; //导入依赖的package包/类
@Override
protected Gate createGroupByGate(GroupBy groupBy, IORole ioRole) {
	throw new UnsupportedOperationException("Cannot create a GroupBy gate in a HashJoinMapperStreamGraph");
}
 

import cascading.pipe.GroupBy; //导入依赖的package包/类
@Override
protected Gate createGroupByGate(GroupBy groupBy, IORole ioRole) {
	throw new UnsupportedOperationException("Cannot create a GroupBy gate in a HashJoinStreamGraph");
}
 

import cascading.pipe.GroupBy; //导入依赖的package包/类
private void buildGraph( GroupBy groupBy, FlowProcess flowProcess ) {

		this.sourceStage = new GroupByInGate(flowProcess, groupBy, IORole.source);
		addHead( sourceStage );
		handleDuct( groupBy, sourceStage );
	}
 

import cascading.pipe.GroupBy; //导入依赖的package包/类
@Override
protected Gate createGroupByGate(GroupBy groupBy, IORole ioRole) {
	throw new UnsupportedOperationException("Cannot create a GroupBy gate in a GroupByStreamGraph");
}
 

import cascading.pipe.GroupBy; //导入依赖的package包/类
public GroupByInGate(FlowProcess flowProcess, GroupBy splice, IORole ioRole) {
	super(flowProcess, splice, ioRole);

	this.isBufferJoin = splice.getJoiner() instanceof BufferJoin;
}
 

import cascading.pipe.GroupBy; //导入依赖的package包/类
@Override
protected Gate createGroupByGate(GroupBy groupBy, IORole ioRole) {
	throw new UnsupportedOperationException("Cannot create a GroupBy gate in a SinkStreamGraph");
}
 

import cascading.pipe.GroupBy; //导入依赖的package包/类
@Override
protected Gate createGroupByGate(GroupBy groupBy, IORole ioRole) {
	throw new UnsupportedOperationException("Cannot create a GroupBy gate in a MapStreamGraph.");
}
 

import cascading.pipe.GroupBy; //导入依赖的package包/类
@Override
protected Gate createGroupByGate(GroupBy element, IORole role) {
	throw new UnsupportedOperationException("SourceStreamGraph may not have a GroupByGate");
}
 

import cascading.pipe.GroupBy; //导入依赖的package包/类
@Override
protected Gate createGroupByGate(GroupBy groupBy, IORole ioRole) {
	throw new UnsupportedOperationException("Cannot create a GroupBy gate in a CoGroupBufferReduceStreamGraph");
}
 

import cascading.pipe.GroupBy; //导入依赖的package包/类
@Override
protected Gate createGroupByGate(GroupBy groupBy, IORole ioRole) {
	throw new UnsupportedOperationException("Cannot create a GroupBy gate in a CoGroupReduceStreamGraph");
}
 

import cascading.pipe.GroupBy; //导入依赖的package包/类
@Test
public void complexPipe() throws Exception {
    Pipe pipe = PipeBuilder.start()
            .each().select("data").filterOut().nullValues()
            .each().insertField("processingDate").withValue(System.currentTimeMillis())
            .renameField("domains").to("oldDomains")
            .each().select("url").applyFunction(new ExpressionFunction(new Fields("isRelativeUrl"), "url.startsWith(\"http\")", String.class))
            .compressOutput().forThisStepOnly().withGzipCodec().withType("BLOCK").ofTheMappers()
            .discard("data")
            .groupBy().withSortOnFields("url").onFields("domain")
            .count("domains")
            .pipe();
    
    // count
    assertThat(pipe).isInstanceOf(Every.class);
    assertThat(((Every)pipe).getOperation()).isInstanceOf(Count.class);
    
    // groupBy
    pipe = pipe.getPrevious()[0];
    assertThat(pipe).isInstanceOf(GroupBy.class);
    
    // discard
    pipe = pipe.getPrevious()[0];
    assertThat(pipe).isInstanceOf(Discard.class);
    pipe = pipe.getPrevious()[0];
    assertThat(pipe).isInstanceOf(Each.class);
    assertThat(((Each)pipe).getOperation()).isInstanceOf(NoOp.class);
    
    // expression
    pipe = pipe.getPrevious()[0];
    assertThat(pipe).isInstanceOf(Each.class);
    assertThat(((Each)pipe).getOperation()).isInstanceOf(ExpressionFunction.class);
    
    // rename
    pipe = pipe.getPrevious()[0];
    assertThat(pipe).isInstanceOf(Rename.class);
    pipe = pipe.getPrevious()[0];
    assertThat(pipe).isInstanceOf(Each.class);
    assertThat(((Each)pipe).getOperation()).isInstanceOf(Identity.class);
    
    // insert
    pipe = pipe.getPrevious()[0];
    assertThat(pipe).isInstanceOf(Each.class);
    assertThat(((Each)pipe).getOperation()).isInstanceOf(Insert.class);
    
    // filter null
    pipe = pipe.getPrevious()[0];
    assertThat(pipe).isInstanceOf(Each.class);
    assertThat(((Each)pipe).getOperation()).isInstanceOf(FilterNull.class);
    
    // initial pipe
    pipe = pipe.getPrevious()[0];
    assertThat(pipe.getClass()).isEqualTo(Pipe.class);
}