Java DataType.BYTEARRAY属性代码示例

@Override
public ResourceSchema getSchema(final String s, final Job job) throws IOException {
    ResourceSchema.ResourceFieldSchema remoteAddrFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("remoteAddr", DataType.BYTEARRAY));
    ResourceSchema.ResourceFieldSchema accessTimeFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("accessTime", DataType.BYTEARRAY));
    ResourceSchema.ResourceFieldSchema methodFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("method", DataType.BYTEARRAY));

    ResourceSchema.ResourceFieldSchema urlFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("url", DataType.BYTEARRAY));
    ResourceSchema.ResourceFieldSchema protocolFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("protocol", DataType.BYTEARRAY));
    ResourceSchema.ResourceFieldSchema agentFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("agent", DataType.BYTEARRAY));

    ResourceSchema.ResourceFieldSchema referFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("refer", DataType.BYTEARRAY));
    ResourceSchema.ResourceFieldSchema statusFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("status", DataType.INTEGER));
    ResourceSchema.ResourceFieldSchema lengthFieldSchema = new ResourceSchema.ResourceFieldSchema(new Schema.FieldSchema("length", DataType.INTEGER));

    ResourceSchema resourceSchema = new ResourceSchema();
    resourceSchema.setFields(new ResourceSchema.ResourceFieldSchema[]{remoteAddrFieldSchema, accessTimeFieldSchema, methodFieldSchema, urlFieldSchema, protocolFieldSchema, agentFieldSchema, referFieldSchema, statusFieldSchema, lengthFieldSchema});
    return resourceSchema;
}
 

static void updateUnion(final DataBag bag, final Union union) throws ExecException {
  // Bag is not empty. process each innerTuple in the bag
  for (final Tuple innerTuple : bag) {
    final Object f0 = innerTuple.get(0); // consider only field 0
    if (f0 == null) {
      continue;
    }
    final byte type = innerTuple.getType(0);
    if (type == DataType.BYTEARRAY) {
      final DataByteArray dba = (DataByteArray) f0;
      union.update(HllSketch.wrap(Memory.wrap(dba.get())));
    } else {
      throw new IllegalArgumentException("Field type was not DataType.BYTEARRAY: " + type);
    }
  }
}
 

/*************************************************************************************************
 * Updates a union from a bag of sketches
 *
 * @param bag A bag of sketchTuples.
 * @param union The union to update
 */
private static void updateUnion(final DataBag bag, final com.yahoo.sketches.theta.Union union) {
  // Bag is not empty. process each innerTuple in the bag
  for (Tuple innerTuple : bag) {
    // validate the inner Tuples
    final Object f0 = extractFieldAtIndex(innerTuple, 0);
    if (f0 == null) {
      continue;
    }
    final Byte type = extractTypeAtIndex(innerTuple, 0);
    if (type == null) {
      continue;
    }
    // add only the first field of the innerTuple to the union
    if (type == DataType.BYTEARRAY) {
      final DataByteArray dba = (DataByteArray) f0;
      if (dba.size() > 0) {
        union.update(Memory.wrap(dba.get()));
      }
    } else {
      throw new IllegalArgumentException("Field type was not DataType.BYTEARRAY: " + type);
    }
  }
}
 

/*************************************************************************************************
 * Updates an intersection from a bag of sketches
 *
 * @param bag A bag of sketchTuples.
 * @param intersection The intersection to update
 * @param seed to check against incoming sketches
 */
private static void updateIntersection(final DataBag bag, final Intersection intersection,
    final long seed) {
  //Bag is not empty. process each innerTuple in the bag
  for (Tuple innerTuple : bag) {
    //validate the inner Tuples
    final Object f0 = extractFieldAtIndex(innerTuple, 0);
    if (f0 == null) {
    continue;
  }
    final Byte type = extractTypeAtIndex(innerTuple, 0);
  // add only the first field of the innerTuple to the intersection
  if (type == DataType.BYTEARRAY) {
    final DataByteArray dba = (DataByteArray) f0;
    final Memory srcMem = Memory.wrap(dba.get());
    final Sketch sketch = Sketch.wrap(srcMem, seed);
    intersection.update(sketch);
  }
  else {
    throw new IllegalArgumentException(
        "Field type was not DataType.BYTEARRAY: " + type);
    }
  }
}
 

@Override
public Schema outputSchema(final Schema input) {
  try {
    if (input == null || input.size() == 0
            || input.getField(0).type != DataType.BYTEARRAY) {
      throw new IllegalArgumentException("Input to GetVarOptSamples must be a DataByteArray: "
              + (input == null ? "null" : input.toString()));
    }

    final Schema weightedSampleSchema = new Schema();
    weightedSampleSchema.add(new Schema.FieldSchema(WEIGHT_ALIAS, DataType.DOUBLE));
    weightedSampleSchema.add(new Schema.FieldSchema(RECORD_ALIAS, DataType.TUPLE));

    return new Schema(new Schema.FieldSchema(getSchemaName(this
            .getClass().getName().toLowerCase(), input), weightedSampleSchema, DataType.BAG));
  } catch (final FrontendException e) {
    throw new RuntimeException(e);
  }
}
 

/**
 * Convert Pig primitive type to Avro type
 * 
 */
protected static Schema convertPrimitiveType(byte pigType) throws IOException {

    if (pigType == DataType.BOOLEAN) {
        return AvroStorageUtils.BooleanSchema;
    } else if (pigType == DataType.BYTEARRAY) {
        return AvroStorageUtils.BytesSchema;
    } else if (pigType == DataType.CHARARRAY
                                    || pigType == DataType.BIGCHARARRAY) {
        return AvroStorageUtils.StringSchema;
    } else if (pigType == DataType.DOUBLE) {
        return AvroStorageUtils.DoubleSchema;
    } else if (pigType == DataType.FLOAT) {
        return AvroStorageUtils.FloatSchema;
    } else if (pigType == DataType.INTEGER) {
        return AvroStorageUtils.IntSchema;
    } else if (pigType == DataType.LONG) {
        return AvroStorageUtils.LongSchema;
    } else
        throw new IOException("unsupported pig type:"
                                        + DataType.findTypeName(pigType));

}
 

@Override
public LogicalSchema getSchema() throws FrontendException {
    if (schema!=null)
        return schema;
    
    if (isCastInserted()) {
        schema = new LogicalSchema();
        for (int i=0;i<scriptSchema.size();i++) {
            LogicalSchema.LogicalFieldSchema fs = scriptSchema.getField(i).deepCopy();
            fs.type = DataType.BYTEARRAY;
            schema.addField(fs);
        }
    } else {
        if (scriptSchema!=null)
            schema = scriptSchema.deepCopy();
    }
    
    if (schema!=null)
        uidOnlySchema = schema.mergeUid(uidOnlySchema);

    return schema;
}
 

/**
 * Checks to see if any field of the input schema is a byte array
 * @param func
 * @param s - input schema
 * @return true if found else false
 * @throws VisitorException
 */
private boolean byteArrayFound(UserFuncExpression func, Schema s) throws VisitorException {
    for(int i=0;i<s.size();i++){
        try {
            FieldSchema fs=s.getField(i);
            if(fs == null)
                return false;
            if(fs.type==DataType.BYTEARRAY){
                return true;
            }
        } catch (FrontendException fee) {
            int errCode = 1043;
            String msg = "Unable to retrieve field schema.";
            throw new TypeCheckerException(func, msg, errCode, PigException.INPUT, fee);
        }
    }
    return false;
}
 

public String typeName(byte type) {
    switch(type) {
        case (DataType.INTEGER): return "int";
        case (DataType.LONG): return "long";
        case (DataType.FLOAT): return "float";
        case (DataType.DOUBLE): return "double";
        case (DataType.BYTEARRAY): return "byte[]";
        case (DataType.CHARARRAY): return "String";
        case (DataType.BOOLEAN): return "boolean";
        case (DataType.DATETIME): return "DateTime";
        case (DataType.BIGDECIMAL): return "BigDecimal";
        case (DataType.BIGINTEGER): return "BigInteger";
        case (DataType.TUPLE): return "Tuple";
        case (DataType.BAG): return "DataBag";
        case (DataType.MAP): return "Map";
        default: throw new RuntimeException("Can't return String for given type " + DataType.findTypeName(type));
    }
}
 

private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecException {
    if (data == null) {
        return null;
    }
    byte type = detail.type;
    switch (type) {
        case DataType.BOOLEAN:
            return TypeSystem.asTypedObject(DataType.toBoolean(data, type));
        case DataType.INTEGER:
        case DataType.LONG:
            return TypeSystem.asTypedObject(DataType.toLong(data, type));
        case DataType.FLOAT:
        case DataType.DOUBLE:
            return TypeSystem.asTypedObject(DataType.toDouble(data, type));
        case DataType.DATETIME:
            return TypeSystem.asTypedObject(new Timestamp(DataType.toDateTime(data, type).getMillis()));
        case DataType.BYTE:
        case DataType.BYTEARRAY:
        case DataType.CHARARRAY:
            return TypeSystem.asTypedObject(DataType.toString(data, type));
        case DataType.BIGINTEGER:
        case DataType.BIGDECIMAL:
            return TypeSystem.asTypedObject(DataType.toBigDecimal(data, type));
        default:
            //TUPLE, BAG, MAP, INTERNALMAP, GENERIC_WRITABLECOMPARABLE, ERROR, UNKNOWN, NULL and anything else
            return null;
    }
}
 

@Override
public Schema outputSchema(final Schema input) {
  try {
    if (input == null || input.size() == 0) {
      throw new IllegalArgumentException("Degenerate input schema to VarOptSampling");
    }

    // first element must be a bag
    if (input.getField(0).type != DataType.BAG) {
      throw new IllegalArgumentException("VarOpt input must be a data bag: "
              + input.toString());
    }

    final Schema record = input.getField(0).schema; // record has a tuple in field 0
    final Schema fields = record.getField(0).schema;
    if (fields.getField(weightIdx_).type != DataType.DOUBLE
            && fields.getField(weightIdx_).type != DataType.FLOAT) {
      throw new IllegalArgumentException("weightIndex item of VarOpt tuple must be a "
              + "weight (double/float), found " + fields.getField(0).type
              + ": " + fields.toString());
    }

    return new Schema(new Schema.FieldSchema(getSchemaName(this
            .getClass().getName().toLowerCase(), input), DataType.BYTEARRAY));
  }
  catch (final FrontendException e) {
    throw new RuntimeException(e);
  }
}
 

@Test
public void schema() throws Exception {
  EvalFunc<Tuple> func = new SketchToEstimateAndErrorBounds();
  Schema inputSchema = new Schema(new Schema.FieldSchema("Sketch", DataType.BYTEARRAY));
  Schema outputSchema = func.outputSchema(inputSchema);
  Assert.assertNotNull(outputSchema);
  Assert.assertEquals(outputSchema.size(), 1);
  Assert.assertEquals(DataType.findTypeName(outputSchema.getField(0).type), "tuple");
  Schema innerSchema = outputSchema.getField(0).schema;
  Assert.assertEquals(innerSchema.size(), 3);
  Assert.assertEquals(DataType.findTypeName(innerSchema.getField(0).type), "double");
  Assert.assertEquals(DataType.findTypeName(innerSchema.getField(1).type), "double");
  Assert.assertEquals(DataType.findTypeName(innerSchema.getField(2).type), "double");
}
 

@Override
public Result getNextBoolean() throws ExecException {
    Result left, right;

    switch (operandType) {
    case DataType.BYTEARRAY:
    case DataType.DOUBLE:
    case DataType.FLOAT:
    case DataType.INTEGER:
    case DataType.BIGINTEGER:
    case DataType.BIGDECIMAL:
    case DataType.LONG:
    case DataType.DATETIME:
    case DataType.CHARARRAY: {
        Result r = accumChild(null, operandType);
        if (r != null) {
            return r;
        }
        left = lhs.getNext(operandType);
        right = rhs.getNext(operandType);
        return doComparison(left, right);
    }

    default: {
        int errCode = 2067;
        String msg = this.getClass().getSimpleName() + " does not know how to " +
        "handle type: " + DataType.findTypeName(operandType);
        throw new ExecException(msg, errCode, PigException.BUG);
    }

    }
}
 

private Type convertWithName(FieldSchema fieldSchema, String name) {
  try {
    switch (fieldSchema.type) {
    case DataType.BAG:
      return convertBag(name, fieldSchema);
    case DataType.TUPLE:
      return convertTuple(name, fieldSchema, Repetition.OPTIONAL);
    case DataType.MAP:
      return convertMap(name, fieldSchema);
    case DataType.BOOLEAN:
      return primitive(name, PrimitiveTypeName.BOOLEAN);
    case DataType.CHARARRAY:
      return primitive(name, PrimitiveTypeName.BINARY, OriginalType.UTF8);
    case DataType.INTEGER:
      return primitive(name, PrimitiveTypeName.INT32);
    case DataType.LONG:
      return primitive(name, PrimitiveTypeName.INT64);
    case DataType.FLOAT:
      return primitive(name, PrimitiveTypeName.FLOAT);
    case DataType.DOUBLE:
      return primitive(name, PrimitiveTypeName.DOUBLE);
    case DataType.DATETIME:
      throw new UnsupportedOperationException();
    case DataType.BYTEARRAY:
      return primitive(name, PrimitiveTypeName.BINARY);
    default:
      throw new SchemaConversionException("Unknown type " + fieldSchema.type + " " + DataType.findTypeName(fieldSchema.type));
    }
  } catch (FrontendException e) {
    throw new SchemaConversionException("can't convert "+fieldSchema, e);
  }
}
 

@Override
public Result getNextBoolean() throws ExecException {
    Result left, right;

    switch (operandType) {
    case DataType.BYTEARRAY:
    case DataType.DOUBLE:
    case DataType.FLOAT:
    case DataType.INTEGER:
    case DataType.BIGINTEGER:
    case DataType.BIGDECIMAL:
    case DataType.LONG:
    case DataType.DATETIME:
    case DataType.CHARARRAY: {
        Result r = accumChild(null, operandType);
        if (r != null) {
            return r;
        }
        left = lhs.getNext(operandType);
        right = rhs.getNext(operandType);
        return doComparison(left, right);
    }
    default: {
        int errCode = 2067;
        String msg = this.getClass().getSimpleName() + " does not know how to " +
        "handle type: " + DataType.findTypeName(operandType);
        throw new ExecException(msg, errCode, PigException.BUG);
    }

    }
}