Java Fields类代码示例

import cascading.tuple.Fields; //导入依赖的package包/类
static Tuple coerceToString(SinkCall<?, ?> sinkCall) {
    TupleEntry entry = sinkCall.getOutgoingEntry();
    Fields fields = entry.getFields();
    Tuple tuple = entry.getTuple();

    if (fields.hasTypes()) {
        Type types[] = new Type[fields.size()];
        for (int index = 0; index < fields.size(); index++) {
            Type type = fields.getType(index);
            if (type instanceof CoercibleType<?>) {
                types[index] = String.class;
            }
            else {
                types[index] = type;
            }
        }

        tuple = entry.getCoercedTuple(types);
    }
    return tuple;
}
 

import cascading.tuple.Fields; //导入依赖的package包/类
@Test
public void testUpdateOnlyScript() throws Exception {
    Properties properties = new TestSettings().getProperties();
    properties.put(ConfigurationOptions.ES_WRITE_OPERATION, "update");
    properties.put(ConfigurationOptions.ES_MAPPING_ID, "number");
    properties.put(ConfigurationOptions.ES_INDEX_AUTO_CREATE, "yes");
    properties.put(ConfigurationOptions.ES_UPDATE_RETRY_ON_CONFLICT, "3");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT, "counter = 3");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_LANG, "groovy");
    properties.put(ConfigurationOptions.ES_INPUT_JSON, "yes");

    Tap in = sourceTap();
    // use an existing id to allow the update to succeed
    Tap out = new EsTap("json-cascading-local/createwithid", new Fields("line"));

    Pipe pipe = new Pipe("copy");
    build(properties, in, out, pipe);
}
 

import cascading.tuple.Fields; //导入依赖的package包/类
@Test
public void testUpdateOnlyParamScript() throws Exception {
    Properties properties = new TestSettings().getProperties();
    properties.put(ConfigurationOptions.ES_WRITE_OPERATION, "update");
    properties.put(ConfigurationOptions.ES_MAPPING_ID, "number");
    properties.put(ConfigurationOptions.ES_INDEX_AUTO_CREATE, "yes");
    properties.put(ConfigurationOptions.ES_UPDATE_RETRY_ON_CONFLICT, "3");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT, "counter = param1; anothercounter = param2");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_LANG, "groovy");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_PARAMS, " param1:<1>,   param2:number ");
    properties.put(ConfigurationOptions.ES_INPUT_JSON, "yes");

    Tap in = sourceTap();
    // use an existing id to allow the update to succeed
    Tap out = new EsTap("json-cascading-local/createwithid", new Fields("line"));

    Pipe pipe = new Pipe("copy");
    build(properties, in, out, pipe);
}
 

import cascading.tuple.Fields; //导入依赖的package包/类
@Test
public void testUpdateOnlyParamJsonScript() throws Exception {
    Properties properties = new TestSettings().getProperties();
    properties.put(ConfigurationOptions.ES_WRITE_OPERATION, "update");
    properties.put(ConfigurationOptions.ES_MAPPING_ID, "number");
    properties.put(ConfigurationOptions.ES_UPDATE_RETRY_ON_CONFLICT, "3");

    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT, "counter = param1; anothercounter = param2");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_LANG, "groovy");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_PARAMS_JSON, "{ \"param1\":1, \"param2\":2}");
    properties.put(ConfigurationOptions.ES_INPUT_JSON, "yes");

    Tap in = sourceTap();
    // use an existing id to allow the update to succeed
    Tap out = new EsTap("json-cascading-local/createwithid", new Fields("line"));

    Pipe pipe = new Pipe("copy");
    build(properties, in, out, pipe);
}
 

import cascading.tuple.Fields; //导入依赖的package包/类
@Test
public void testUpsertParamScript() throws Exception {
    Properties properties = new TestSettings().getProperties();
    properties.put(ConfigurationOptions.ES_WRITE_OPERATION, "upsert");
    properties.put(ConfigurationOptions.ES_MAPPING_ID, "number");
    properties.put(ConfigurationOptions.ES_INPUT_JSON, "yes");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT, "counter += param1; anothercounter += param2");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_LANG, "groovy");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_PARAMS, " param1:<1>,   param2:number ");

    Tap in = sourceTap();
    // use an existing id to allow the update to succeed
    Tap out = new EsTap("json-cascading-local/upsert-param-script", new Fields("line"));

    Pipe pipe = new Pipe("copy");
    build(properties, in, out, pipe);
}
 

import cascading.tuple.Fields; //导入依赖的package包/类
@Test
public void testUpsertParamJsonScript() throws Exception {
    Properties properties = new TestSettings().getProperties();
    properties.put(ConfigurationOptions.ES_WRITE_OPERATION, "upsert");
    properties.put(ConfigurationOptions.ES_MAPPING_ID, "number");
    properties.put(ConfigurationOptions.ES_INPUT_JSON, "yes");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT, "ctx._source.counter += param1; ctx._source.anothercounter += param2");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_LANG, "groovy");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_PARAMS_JSON, "{ \"param1\":1, \"param2\":2}");

    Tap in = sourceTap();
    // use an existing id to allow the update to succeed
    Tap out = new EsTap("json-cascading-local/upsert-script-json-script", new Fields("line"));

    Pipe pipe = new Pipe("copy");
    build(properties, in, out, pipe);
}
 

import cascading.tuple.Fields; //导入依赖的package包/类
@Test
public void testUpdateOnlyParamScript() throws Exception {
    Properties properties = new TestSettings().getProperties();
    properties.put(ConfigurationOptions.ES_WRITE_OPERATION, "update");
    properties.put(ConfigurationOptions.ES_MAPPING_ID, "id");
    properties.put(ConfigurationOptions.ES_INDEX_AUTO_CREATE, "yes");
    properties.put(ConfigurationOptions.ES_UPDATE_RETRY_ON_CONFLICT, "3");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT, "counter = param1; anothercounter = param2");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_LANG, "groovy");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_PARAMS, " param1:<1>,   param2:id ");

    Tap in = sourceTap();
    // use an existing id to allow the update to succeed
    Tap out = new EsTap("cascading-local/createwithid", new Fields("id", "name", "url", "picture"));

    Pipe pipe = new Pipe("copy");
    build(properties, in, out, pipe);
}
 

import cascading.tuple.Fields; //导入依赖的package包/类
@Test
public void testUpdateOnlyParamJsonScript() throws Exception {
    Properties properties = new TestSettings().getProperties();
    properties.put(ConfigurationOptions.ES_WRITE_OPERATION, "update");
    properties.put(ConfigurationOptions.ES_MAPPING_ID, "id");
    properties.put(ConfigurationOptions.ES_UPDATE_RETRY_ON_CONFLICT, "3");

    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT, "counter = param1; anothercounter = param2");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_LANG, "groovy");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_PARAMS_JSON, "{ \"param1\":1, \"param2\":2}");

    Tap in = sourceTap();
    // use an existing id to allow the update to succeed
    Tap out = new EsTap("cascading-local/createwithid", new Fields("id", "name", "url", "picture"));

    Pipe pipe = new Pipe("copy");
    build(properties, in, out, pipe);
}
 

import cascading.tuple.Fields; //导入依赖的package包/类
@Test
public void testUpsertParamScript() throws Exception {
    Properties properties = new TestSettings().getProperties();
    properties.put(ConfigurationOptions.ES_WRITE_OPERATION, "upsert");
    properties.put(ConfigurationOptions.ES_MAPPING_ID, "id");

    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT, "counter += param1; anothercounter += param2");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_LANG, "groovy");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_PARAMS, " param1:<1>,   param2:id ");

    Tap in = sourceTap();
    // use an existing id to allow the update to succeed
    Tap out = new EsTap("cascading-local/upsert-param-script", new Fields("id", "name", "url", "picture"));

    Pipe pipe = new Pipe("copy");
    build(properties, in, out, pipe);
}
 

import cascading.tuple.Fields; //导入依赖的package包/类
@Test
public void testUpsertParamJsonScript() throws Exception {
    Properties properties = new TestSettings().getProperties();
    properties.put(ConfigurationOptions.ES_WRITE_OPERATION, "upsert");
    properties.put(ConfigurationOptions.ES_MAPPING_ID, "id");

    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT, "counter += param1; anothercounter += param2");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_LANG, "groovy");
    properties.put(ConfigurationOptions.ES_UPDATE_SCRIPT_PARAMS_JSON, "{ \"param1\":1, \"param2\":2}");

    Tap in = sourceTap();
    // use an existing id to allow the update to succeed
    Tap out = new EsTap("cascading-local/upsert-script-json-script", new Fields("id", "name", "url", "picture"));

    Pipe pipe = new Pipe("copy");
    build(properties, in, out, pipe);
}
 

import cascading.tuple.Fields; //导入依赖的package包/类
/**
 * Tests if correct number of input fields are provided.
 */
@Test(expected = RuntimeException.class)
public void fieldsCountGreaterThanColumnsTest() {

  String sourcePath = "src/test/resources/input/with-headers.txt";
  String sinkPath = "src/test/resources/output/sink-with-headers";

  FlowConnector connector = new Hadoop2MR1FlowConnector();
  CSVFormat sourceFormat = CSVFormat.newFormat(',')
    .withQuote('"')
    .withEscape('\\')
    .withRecordSeparator('\n');

  CSVFormat sinkFormat = CSVFormat.newFormat('\t')
    .withSkipHeaderRecord()
    .withEscape('\\')
    .withRecordSeparator('\n');

  Fields sourceFields = new Fields("id", "last name", "first name", "phone");
  Tap source = new Hfs(new CsvScheme(sourceFields, sourceFormat), sourcePath);
  Tap sink = new Hfs(new CsvScheme(sinkFormat), sinkPath);
  Pipe pipe = new Pipe("pipe");

  connector.connect(source, sink, pipe).complete();

}
 

import cascading.tuple.Fields; //导入依赖的package包/类
/**
 * Helper method used for assertion of fields generated by CsvScheme.
 */
@SuppressWarnings("unchecked")
private void testSchemeFields(String sourcePath, CsvScheme sourceSchema, String sinkPath, CsvScheme sinkScheme, Set<String> expected) {

  Tap source = new Hfs(sourceSchema, sourcePath);
  Tap sink = new Hfs(sinkScheme, sinkPath);
  Pipe pipe = new Pipe("pipe");

  FlowConnector connector = new Hadoop2MR1FlowConnector();
  connector.connect(source, sink, pipe).complete();

  Fields sinkFields = sink.getSinkFields();
  for (int i = 0; i < sinkFields.size(); i++) {
    assertTrue("Unexpected column " + sinkFields.get(i), expected.contains(sinkFields.get(i)));
    expected.remove(sinkFields.get(i));
  }

  assertTrue("Not all expected values are found", expected.isEmpty());

}
 

import cascading.tuple.Fields; //导入依赖的package包/类
private DataSet<Tuple> translateSource(FlowProcess flowProcess, ExecutionEnvironment env, FlowNode node, int dop) {

		Tap tap = this.getSingle(node.getSourceTaps());
		JobConf tapConfig = new JobConf(this.getNodeConfig(node));
		tap.sourceConfInit(flowProcess, tapConfig);
		tapConfig.set( "cascading.step.source", Tap.id( tap ) );

		Fields outFields = tap.getSourceFields();
		registerKryoTypes(outFields);

		JobConf sourceConfig = new JobConf(this.getNodeConfig(node));
		MultiInputFormat.addInputFormat(sourceConfig, tapConfig);

		DataSet<Tuple> src = env
				.createInput(new TapInputFormat(node), new TupleTypeInfo(outFields))
						.name(tap.getIdentifier())
						.setParallelism(dop)
						.withParameters(FlinkConfigConverter.toFlinkConfig(new Configuration(sourceConfig)));

		return src;

	}
 

import cascading.tuple.Fields; //导入依赖的package包/类
private void registerKryoTypes(Fields fields) {

		if(fields.hasTypes()) {
			Class[] fieldTypeClasses = fields.getTypesClasses();
			for(Class fieldTypeClass : fieldTypeClasses) {
				if(!fieldTypeClass.isPrimitive() &&
						!fieldTypeClass.equals(String.class) &&
						!Writable.class.isAssignableFrom(fieldTypeClass)) {
					// register type if it is neither a primitive, String, or Writable

					env.getConfig().registerKryoType(fieldTypeClass);
				}
			}
		}

	}
 

import cascading.tuple.Fields; //导入依赖的package包/类
private TupleBuilder makeJoinedBuilder( final Fields[] joinFields )
{
	final Fields[] fields = isSelfJoin() ? new Fields[ size() ] : joinFields;

	if( isSelfJoin() ) {
		Arrays.fill(fields, 0, fields.length, joinFields[0]);
	}

	return new TupleBuilder()
	{
		Tuple result = TupleViews.createComposite(fields);

		@Override
		public Tuple makeResult( Tuple[] tuples )
		{
			return TupleViews.reset( result, tuples );
		}
	};
}