Java BytesColumnVector类代码示例

import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
static void compareStringcolumn(ColumnVector oneStringColumn, boolean[] isNull,
                                 long currentBatchRow, Vec h2oFrame, Long startRowIndex, String columnType) {
  byte[][] oneColumn = ((BytesColumnVector) oneStringColumn).vector;
  int[] stringLength = ((BytesColumnVector) oneStringColumn).length;
  int[] stringStart = ((BytesColumnVector) oneStringColumn).start;
  long frameRowIndex = startRowIndex;
  BufferedString tempH2o = new BufferedString();
  BufferedString tempOrc = new BufferedString();

  for (int rowIndex = 0; rowIndex < currentBatchRow; rowIndex++) {
    if (isNull[rowIndex])
      assertEquals("Na is found: ", true, h2oFrame.isNA(frameRowIndex));
    else {
      if (!oneStringColumn.isRepeating || rowIndex == 0)
        tempOrc.set(oneColumn[rowIndex], stringStart[rowIndex], stringLength[rowIndex]);
      h2oFrame.atStr(tempH2o, frameRowIndex);
      assertEquals("isRepeating = " + oneStringColumn.isRepeating + " String/char elements should equal: ", true, tempOrc.equals(tempH2o));
    }

    frameRowIndex++;
  }
}
 

import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
@Override
public void setColumnVector( final ColumnVector vector , final IExpressionIndex indexList , final int start , final int length ) throws IOException{
  BytesColumnVector columnVector = (BytesColumnVector)vector;

  PrimitiveObject[] primitiveObjectArray = column.getPrimitiveObjectArray( indexList , start , length );
  for( int i = 0 ; i < length ; i++ ){
    if( primitiveObjectArray[i] == null ){
      VectorizedBatchUtil.setNullColIsNullValue( columnVector , i );
    }
    else{
      if( primitiveObjectArray[i] instanceof IBytesLink ){
        IBytesLink linkObj = (IBytesLink)primitiveObjectArray[i];
        columnVector.vector[i] = linkObj.getLinkBytes();
        columnVector.start[i] = linkObj.getStart();
        columnVector.length[i] = linkObj.getLength();
      }
      else{
        byte[] strBytes = primitiveObjectArray[i].getBytes();
        if( strBytes == null ){
          VectorizedBatchUtil.setNullColIsNullValue( columnVector , i );
        }
        else{
          columnVector.vector[i] = strBytes;
          columnVector.start[i] = 0;
          columnVector.length[i] = strBytes.length;
        }
      }
    }
  }
}
 

import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
@Override
void writeBatch(ColumnVector vector, int offset, int length) throws IOException {
  super.writeBatch(vector, offset, length);
  BytesColumnVector vec = (BytesColumnVector) vector;
  if (vector.isRepeating) {
    if (vector.noNulls || !vector.isNull[0]) {
      for (int i = 0; i < length; ++i) {
        stream.write(vec.vector[0], vec.start[0], vec.length[0]);
        this.length.write(vec.length[0]);
      }
      indexStatistics.updateBinary(vec.vector[0], vec.start[0], vec.length[0], length);
      if (createBloomFilter) {
        bloomFilter.addBytes(vec.vector[0], vec.start[0], vec.length[0]);
      }
    }
  } else {
    for (int i = 0; i < length; ++i) {
      if (vec.noNulls || !vec.isNull[i + offset]) {
        stream.write(vec.vector[offset + i], vec.start[offset + i], vec.length[offset + i]);
        this.length.write(vec.length[offset + i]);
        indexStatistics.updateBinary(vec.vector[offset + i], vec.start[offset + i],
            vec.length[offset + i], 1);
        if (createBloomFilter) {
          bloomFilter.addBytes(vec.vector[offset + i], vec.start[offset + i],
              vec.length[offset + i]);
        }
      }
    }
  }
}
 

import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
@Override
public void process(RelationContainer container) {
    DecimalColumnVector lat = (DecimalColumnVector) batch.cols[3];
    DecimalColumnVector lon = (DecimalColumnVector) batch.cols[4];
    ListColumnVector members = (ListColumnVector) batch.cols[6];

    checkLimit();
    addCommonProperties(container);

    lat.isNull[row] = true;
    lon.isNull[row] = true;
    lat.set(row, (HiveDecimal) null);
    lon.set(row, (HiveDecimal) null);

    Relation relation = container.getEntity();

    members.lengths[row] = relation.getMembers().size();
    members.childCount += members.lengths[row];
    members.child.ensureSize(members.childCount, members.offsets[row] != 0);

    for (int j = 0; j < relation.getMembers().size(); j++) {
        StructColumnVector membersStruct = (StructColumnVector) members.child;

        ((BytesColumnVector) membersStruct.fields[0]).setVal((int) members.offsets[row] + j, relation.getMembers().get(j).getMemberType().toString().toLowerCase().getBytes());
        ((LongColumnVector) membersStruct.fields[1]).vector[(int) members.offsets[row] + j] = relation.getMembers().get(j).getMemberId();
        ((BytesColumnVector) membersStruct.fields[2]).setVal((int) members.offsets[row] + j, relation.getMembers().get(j).getMemberRole().getBytes());
    }
}
 

import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
public void convert(JsonElement value, ColumnVector vect, int row) {
    if (value == null || value.isJsonNull()) {
        vect.noNulls = false;
        vect.isNull[row] = true;
    } else {
        BytesColumnVector vector = (BytesColumnVector) vect;
        byte[] bytes = value.getAsString().getBytes(
                StandardCharsets.UTF_8);
        vector.setRef(row, bytes, 0, bytes.length);
    }
}
 

import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
private void setBytesColumnVector( BytesColumnVector bytesColumnVector, String value ) {
  if ( value == null ) {
    setBytesColumnVector( bytesColumnVector, new byte[ 0 ] );
  } else {
    setBytesColumnVector( bytesColumnVector, value.getBytes() );
  }
}
 

import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
private boolean flush(BufferSegment segment, String path, TypeDescription schema)
    {
        Configuration conf = new Configuration();
        try {
            Writer writer = OrcFile.createWriter(new Path(path),
                    OrcFile.writerOptions(conf)
                            .setSchema(schema)
                            .stripeSize(orcFileStripeSize)
                            .bufferSize(orcFileBufferSize)
                            .blockSize(orcFileBlockSize)
                            .compress(CompressionKind.ZLIB)
                            .version(OrcFile.Version.V_0_12));
            VectorizedRowBatch batch = schema.createRowBatch();
            while (segment.hasNext()) {
                String[] contents = segment.getNext();
                int rowCount = batch.size++;
//                    System.out.println("contents : message.getValues() : " + Arrays.toString(contents));
                System.out.println("contents.length : " + contents.length);
                for (int i = 0; i < contents.length; i++) {
                    ((BytesColumnVector) batch.cols[i]).setVal(rowCount, contents[i].getBytes());
                    //batch full
                    if (batch.size == batch.getMaxSize()) {
                        writer.addRowBatch(batch);
                        batch.reset();
                    }
                }
                if (batch.size != 0) {
                    writer.addRowBatch(batch);
                    batch.reset();
                }
                writer.close();
                segment.setFilePath(path);
                System.out.println("path : " + path);
            }
            return true;
        }
        catch (IOException e) {
            e.printStackTrace();
            return false;
        }
    }
 

import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
private void addCommonProperties(EntityContainer container) {
    LongColumnVector id = (LongColumnVector) batch.cols[0];
    BytesColumnVector type = (BytesColumnVector) batch.cols[1];
    MapColumnVector tags = (MapColumnVector) batch.cols[2];
    ListColumnVector nds = (ListColumnVector) batch.cols[5];
    ListColumnVector members = (ListColumnVector) batch.cols[6];
    LongColumnVector changeset = (LongColumnVector) batch.cols[7];
    TimestampColumnVector timestamp = (TimestampColumnVector) batch.cols[8];
    LongColumnVector uid = (LongColumnVector) batch.cols[9];
    BytesColumnVector user = (BytesColumnVector) batch.cols[10];
    LongColumnVector version = (LongColumnVector) batch.cols[11];
    LongColumnVector visible = (LongColumnVector) batch.cols[12];

    Entity entity = container.getEntity();

    id.vector[row] = entity.getId();
    changeset.vector[row] = entity.getChangesetId();
    type.setVal(row, entity.getType().toString().toLowerCase().getBytes());

    tags.offsets[row] = tags.childCount;
    tags.lengths[row] = entity.getTags().size(); // number of key/value pairings
    tags.childCount += tags.lengths[row];
    tags.keys.ensureSize(tags.childCount, tags.offsets[row] != 0);
    tags.values.ensureSize(tags.childCount, tags.offsets[row] != 0);

    int i = 0;
    for (Tag tag : entity.getTags()) {
        ((BytesColumnVector) tags.keys).setVal((int) tags.offsets[row] + i, tag.getKey().getBytes());
        ((BytesColumnVector) tags.values).setVal((int) tags.offsets[row] + i, tag.getValue().getBytes());

        i++;
    }

    timestamp.time[row] = entity.getTimestamp().getTime();
    timestamp.nanos[row] = 0;

    uid.vector[row] = entity.getUser().getId();

    user.setVal(row, entity.getUser().getName().getBytes());

    version.vector[row] = entity.getVersion();

    visible.vector[row] = 1;
    if (entity.getMetaTags().get("visible") == Boolean.FALSE) {
        visible.vector[row] = 0;
    }

    nds.offsets[row] = nds.childCount;
    nds.lengths[row] = 0;

    members.offsets[row] = members.childCount;
    members.lengths[row] = 0;
}
 

import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
static void setValue(JSONWriter writer, ColumnVector vector,
        TypeDescription schema, int row) throws JSONException {
    if (vector.isRepeating) {
        row = 0;
    }
    if (vector.noNulls || !vector.isNull[row]) {
        switch (schema.getCategory()) {
        case BOOLEAN:
            writer.value(((LongColumnVector) vector).vector[row] != 0);
            break;
        case BYTE:
        case SHORT:
        case INT:
        case LONG:
            writer.value(((LongColumnVector) vector).vector[row]);
            break;
        case FLOAT:
        case DOUBLE:
            writer.value(((DoubleColumnVector) vector).vector[row]);
            break;
        case STRING:
        case CHAR:
        case VARCHAR:
            writer.value(((BytesColumnVector) vector).toString(row));
            break;
        case DECIMAL:
            writer.value(((DecimalColumnVector) vector).vector[row]
                    .toString());
            break;
        case DATE:
            writer.value(new DateWritable(
                    (int) ((LongColumnVector) vector).vector[row])
                    .toString());
            break;
        case TIMESTAMP:
            writer.value(((TimestampColumnVector) vector)
                    .asScratchTimestamp(row).toString());
            break;
        case LIST:
            setList(writer, (ListColumnVector) vector, schema, row);
            break;
        case STRUCT:
            setStruct(writer, (StructColumnVector) vector, schema, row);
            break;
        case UNION:
            // printUnion(writer, (UnionColumnVector) vector, schema, row);
            break;
        case BINARY:
            // printBinary(writer, (BytesColumnVector) vector, row);
            break;
        case MAP:
            // printMap(writer, (MapColumnVector) vector, schema, row);
            break;
        default:
            throw new IllegalArgumentException("Unknown type "
                    + schema.toString());
        }
    } else {
        writer.value(null);
    }
}
 

import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; //导入依赖的package包/类
protected static Object convertFromSourceToTargetDataType( ColumnVector columnVector, int currentBatchRow,
                                                           int orcValueMetaInterface ) {

  if ( columnVector.isNull[currentBatchRow] ) {
    return null;
  }
  switch ( orcValueMetaInterface ) {
    case ValueMetaInterface.TYPE_INET:
      try {
        return InetAddress.getByName( new String( ( (BytesColumnVector) columnVector ).vector[ currentBatchRow ],
          ( (BytesColumnVector) columnVector ).start[ currentBatchRow ],
          ( (BytesColumnVector) columnVector ).length[ currentBatchRow ] ) );
      } catch ( UnknownHostException e ) {
        e.printStackTrace();
      }

    case ValueMetaInterface.TYPE_STRING:
      return new String( ( (BytesColumnVector) columnVector ).vector[ currentBatchRow ],
        ( (BytesColumnVector) columnVector ).start[ currentBatchRow ],
        ( (BytesColumnVector) columnVector ).length[ currentBatchRow ] );

    case ValueMetaInterface.TYPE_INTEGER:
      return (long) ( (LongColumnVector) columnVector ).vector[ currentBatchRow ];

    case ValueMetaInterface.TYPE_NUMBER:
      return ( (DoubleColumnVector) columnVector ).vector[ currentBatchRow ];

    case ValueMetaInterface.TYPE_BIGNUMBER:
      HiveDecimalWritable obj = ( (DecimalColumnVector) columnVector ).vector[ currentBatchRow ];
      return obj.getHiveDecimal().bigDecimalValue();

    case ValueMetaInterface.TYPE_TIMESTAMP:
      Timestamp timestamp = new Timestamp( ( (TimestampColumnVector) columnVector ).time[ currentBatchRow ] );
      timestamp.setNanos( ( (TimestampColumnVector) columnVector ).nanos[ currentBatchRow ] );
      return timestamp;

    case ValueMetaInterface.TYPE_DATE:
      LocalDate localDate = LocalDate.ofEpochDay( 0 ).plusDays( ( (LongColumnVector) columnVector ).vector[ currentBatchRow ] );
      Date dateValue = Date.from( localDate.atStartOfDay( ZoneId.systemDefault() ).toInstant() );
      return dateValue;

    case ValueMetaInterface.TYPE_BOOLEAN:
      return ( (LongColumnVector) columnVector ).vector[ currentBatchRow ] == 0 ? false : true;

    case ValueMetaInterface.TYPE_BINARY:
      byte[] origBytes = ( (BytesColumnVector) columnVector ).vector[ currentBatchRow ];
      int startPos = ( (BytesColumnVector) columnVector ).start[ currentBatchRow ];
      byte[] newBytes = Arrays.copyOfRange( origBytes, startPos,
        startPos + ( (BytesColumnVector) columnVector ).length[ currentBatchRow ] );
      return newBytes;
  }

  //if none of the cases match return a null
  return null;
}