Java DataType.BAG属性代码示例

@Override
public Schema outputSchema(final Schema input) {
  try {
    if (input == null || input.size() == 0
            || input.getField(0).type != DataType.BYTEARRAY) {
      throw new IllegalArgumentException("Input to GetVarOptSamples must be a DataByteArray: "
              + (input == null ? "null" : input.toString()));
    }

    final Schema weightedSampleSchema = new Schema();
    weightedSampleSchema.add(new Schema.FieldSchema(WEIGHT_ALIAS, DataType.DOUBLE));
    weightedSampleSchema.add(new Schema.FieldSchema(RECORD_ALIAS, DataType.TUPLE));

    return new Schema(new Schema.FieldSchema(getSchemaName(this
            .getClass().getName().toLowerCase(), input), weightedSampleSchema, DataType.BAG));
  } catch (final FrontendException e) {
    throw new RuntimeException(e);
  }
}
 

/**
 *
 * @param caster LoadCaster to be used to convert the bytes into a field.
 * @param bytes
 * @param fieldSchema schema of Bag or Tuple; pass in null if a simple type.
 * @param dataType type from DataType
 * @return converted object.
 * @throws IOException
 */
public static Object convertToType(LoadCaster caster, byte[] bytes,
        ResourceFieldSchema fieldSchema, byte dataType) throws IOException {
    switch (dataType) {
    case (DataType.BAG): return caster.bytesToBag(bytes, fieldSchema);
    case (DataType.BYTEARRAY): return new DataByteArray(bytes);
    case (DataType.CHARARRAY): return caster.bytesToCharArray(bytes);
    case (DataType.DOUBLE): return caster.bytesToDouble(bytes);
    case (DataType.FLOAT): return caster.bytesToFloat(bytes);
    case (DataType.INTEGER): return caster.bytesToInteger(bytes);
    case (DataType.BIGINTEGER): return caster.bytesToBigInteger(bytes);
    case (DataType.BIGDECIMAL): return caster.bytesToBigDecimal(bytes);
    case (DataType.LONG): return caster.bytesToLong(bytes);
    case (DataType.BOOLEAN): return caster.bytesToBoolean(bytes);
    case (DataType.DATETIME): return caster.bytesToDateTime(bytes);
    case (DataType.MAP): return caster.bytesToMap(bytes, fieldSchema);
    case (DataType.TUPLE): return caster.bytesToTuple(bytes, fieldSchema);
    default: throw new IOException("Unknown type " + dataType);
    }
}
 

@Override
public Schema outputSchema(Schema input)
{
  try
  {
    if (input.size() != 1)
    {
      throw new RuntimeException("Expected only a single field as input");
    }
    
    if (input.getField(0).type != DataType.BAG)
    {
      throw new RuntimeException("Expected a BAG as input, but found " + DataType.findTypeName(input.getField(0).type));
    }
    
    // get the size of the tuple within the bag
    int innerTupleSize = input.getField(0).schema.getField(0).schema.getFields().size();
    getInstanceProperties().put("tuplesize", innerTupleSize);
  }
  catch (FrontendException e)
  {
    throw new RuntimeException(e);
  }
  return input;
}
 

@Override
public Schema outputSchema(Schema input) {

	if (input.size() != 1) {
		throw new RuntimeException("Expected a single field of type bag, but found " + input.size() + " fields");
	}

	FieldSchema field;
	try {
		field = input.getField(0);

		if (field.type != DataType.BAG) {
			throw new RuntimeException("Expected a bag but got: " + DataType.findTypeName(field.type));
		}
	} catch (FrontendException e) {
		throw new RuntimeException(e);
	}

	return new Schema(new FieldSchema("CountDistinctUpTo", DataType.INTEGER));
}
 

@Override
public Schema outputSchema(Schema input) {

    try {

  // subschema describing the fields in the tuples of the bag
        List<Schema.FieldSchema> tokenFs = new ArrayList<Schema.FieldSchema>();
    //tokenFs.add(new Schema.FieldSchema(null, this.fieldType )); 
    tokenFs.add(new Schema.FieldSchema( this.tupleColName.toUpperCase(), this.fieldType )); 

        Schema tupleSchema = new Schema( tokenFs );
        Schema.FieldSchema tupleFs = new Schema.FieldSchema( this.bagColName.toUpperCase(), tupleSchema, DataType.TUPLE);

        Schema bagSchema = new Schema(tupleFs);
        bagSchema.setTwoLevelAccessRequired(true);
        Schema.FieldSchema bagFs = new Schema.FieldSchema( null, bagSchema, DataType.BAG);

        return new Schema(bagFs);

    } catch (FrontendException e) {
        throw new RuntimeException("Unable to create schema for BAG.");
    }
 
}
 

private Object consumeComplexType(PushbackInputStream in, ResourceFieldSchema complexFieldSchema) throws IOException {
    Object field;
    switch (complexFieldSchema.getType()) {
    case DataType.BAG:
        field = consumeBag(in, complexFieldSchema);
        break;
    case DataType.TUPLE:
        field = consumeTuple(in, complexFieldSchema);
        break;
    case DataType.MAP:
        field = consumeMap(in, complexFieldSchema);
        break;
    default:
        throw new IOException("Unknown complex data type");
    }
    return field;
}
 

public String typeName(byte type) {
    switch(type) {
        case (DataType.INTEGER): return "int";
        case (DataType.LONG): return "long";
        case (DataType.FLOAT): return "float";
        case (DataType.DOUBLE): return "double";
        case (DataType.BYTEARRAY): return "byte[]";
        case (DataType.CHARARRAY): return "String";
        case (DataType.BOOLEAN): return "boolean";
        case (DataType.DATETIME): return "DateTime";
        case (DataType.BIGDECIMAL): return "BigDecimal";
        case (DataType.BIGINTEGER): return "BigInteger";
        case (DataType.TUPLE): return "Tuple";
        case (DataType.BAG): return "DataBag";
        case (DataType.MAP): return "Map";
        default: throw new RuntimeException("Can't return String for given type " + DataType.findTypeName(type));
    }
}
 

private Result getResult(ExpressionOperator op) throws ExecException {
    Result res;
    switch (op.getResultType()) {
    case DataType.BAG:
    case DataType.BOOLEAN:
    case DataType.BYTEARRAY:
    case DataType.CHARARRAY:
    case DataType.DOUBLE:
    case DataType.FLOAT:
    case DataType.INTEGER:
    case DataType.LONG:
    case DataType.BIGINTEGER:
    case DataType.BIGDECIMAL:
    case DataType.DATETIME:
    case DataType.MAP:
    case DataType.TUPLE:
        res = op.getNext(op.getResultType());
        break;
    default:
        String msg = "Invalid result type: "
                + DataType.findType(op.getResultType());
        throw new ExecException(msg, 2270, PigException.BUG);
    }

    return res;
}
 

private Type toCanonicalType(final LogicalSchema.LogicalFieldSchema field) {
    if (PigTypeMapping.getPIG_TO_CANONICAL().containsKey(field.type)) {
        return PigTypeMapping.getPIG_TO_CANONICAL().get(field.type);
    }
    switch (field.type) {
        case DataType.MAP:
            return toCanonicalMapType(field);
        case DataType.BAG:
            return toCanonicalArrayType(field);
        case DataType.TUPLE:
            return toCanonicalRowType(field);
        default:
    }
    throw new IllegalArgumentException(String.format("Invalid for Pig converter: '%s'", field.toString()));
}
 

@Override
public Schema outputSchema(Schema input) {
	try {
		Schema bagSchema = new Schema();
		bagSchema.add(new Schema.FieldSchema("named_entity",
				DataType.CHARARRAY));
		return new Schema(new Schema.FieldSchema(getSchemaName(
				"named_entities", input), bagSchema, DataType.BAG));
	} catch (Exception e) {
		return null;
	}
}
 

@Override
public Schema outputSchema(Schema input) {
	try {
		Schema bagSchema = new Schema();
		bagSchema
				.add(new Schema.FieldSchema("sentence", DataType.CHARARRAY));

		return new Schema(new Schema.FieldSchema(
				getSchemaName(null, input), bagSchema, DataType.BAG));
	} catch (Exception e) {
		return null;
	}

}
 

@Override
public Schema outputSchema(final Schema inputSchema) {
  final Schema tupleSchema = new Schema();
  tupleSchema.add(new Schema.FieldSchema("item", DataType.CHARARRAY));
  tupleSchema.add(new Schema.FieldSchema("estimate", DataType.LONG));
  tupleSchema.add(new Schema.FieldSchema("lower_bound", DataType.LONG));
  tupleSchema.add(new Schema.FieldSchema("upper_bound", DataType.LONG));
  try {
    final Schema bagSchema = new Schema(new Schema.FieldSchema("item_tuple", tupleSchema, DataType.TUPLE));
    return new Schema(new Schema.FieldSchema("bag_of_item_tuples", bagSchema, DataType.BAG));
  } catch (final FrontendException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public Schema outputSchema(final Schema input) {
  try {
    if (input == null || input.size() == 0) {
      throw new IllegalArgumentException("Degenerate input schema to VarOptSampling");
    }

    // first element must be a bag, weightIdx_ element of tuples must be a float or double
    if (input.getField(0).type != DataType.BAG) {
      throw new IllegalArgumentException("VarOpt input must be a data bag: "
              + input.toString());
    }

    final Schema record = input.getField(0).schema; // record has a tuple in field 0
    final Schema fields = record.getField(0).schema;
    if (fields.getField(weightIdx_).type != DataType.DOUBLE
            && fields.getField(weightIdx_).type != DataType.FLOAT) {
      throw new IllegalArgumentException("weightIndex item of VarOpt tuple must be a "
              + "weight (double/float), found " + fields.getField(0).type
              + ": " + fields.toString());
    }

    final Schema weightedSampleSchema = new Schema();
    weightedSampleSchema.add(new Schema.FieldSchema(WEIGHT_ALIAS, DataType.DOUBLE));
    weightedSampleSchema.add(new Schema.FieldSchema(RECORD_ALIAS, record, DataType.TUPLE));

    return new Schema(new Schema.FieldSchema(getSchemaName(this
            .getClass().getName().toLowerCase(), record), weightedSampleSchema, DataType.BAG));
  }
  catch (final FrontendException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public Schema outputSchema(final Schema input) {
  if (input != null && input.size() > 0) {
    try {
      Schema source = input;

      // if we have a bag, grab one level down to get a tuple
      if (source.size() == 1 && source.getField(0).type == DataType.BAG) {
        source = source.getField(0).schema;
      }

      final Schema recordSchema = new Schema();
      recordSchema.add(new Schema.FieldSchema(N_ALIAS, DataType.LONG));
      recordSchema.add(new Schema.FieldSchema(K_ALIAS, DataType.INTEGER));

      // this should add a bag to the output
      recordSchema.add(new Schema.FieldSchema(SAMPLES_ALIAS, source, DataType.BAG));

      return new Schema(new Schema.FieldSchema(getSchemaName(this
              .getClass().getName().toLowerCase(), source), recordSchema, DataType.TUPLE));
    }
    catch (final FrontendException e) {
      throw new RuntimeException(e);
    }
  }
  return null;
}
 

@Override
public Schema outputSchema(final Schema input) {
  try {
    if (input == null || input.size() == 0) {
      throw new IllegalArgumentException("Degenerate input schema to VarOptSampling");
    }

    // first element must be a bag
    if (input.getField(0).type != DataType.BAG) {
      throw new IllegalArgumentException("VarOpt input must be a data bag: "
              + input.toString());
    }

    final Schema record = input.getField(0).schema; // record has a tuple in field 0
    final Schema fields = record.getField(0).schema;
    if (fields.getField(weightIdx_).type != DataType.DOUBLE
            && fields.getField(weightIdx_).type != DataType.FLOAT) {
      throw new IllegalArgumentException("weightIndex item of VarOpt tuple must be a "
              + "weight (double/float), found " + fields.getField(0).type
              + ": " + fields.toString());
    }

    return new Schema(new Schema.FieldSchema(getSchemaName(this
            .getClass().getName().toLowerCase(), input), DataType.BYTEARRAY));
  }
  catch (final FrontendException e) {
    throw new RuntimeException(e);
  }
}