Java BytesWritable.set方法代码示例

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
/**
 * Read raw bytes from a SequenceFile.
 */
public synchronized boolean next(BytesWritable key, BytesWritable val)
    throws IOException {
  if (done) return false;
  long pos = in.getPosition();
  boolean eof = -1 == in.nextRawKey(buffer);
  if (!eof) {
    key.set(buffer.getData(), 0, buffer.getLength());
    buffer.reset();
    in.nextRawValue(vbytes);
    vbytes.writeUncompressedBytes(buffer);
    val.set(buffer.getData(), 0, buffer.getLength());
    buffer.reset();
  }
  return !(done = (eof || (pos >= end && in.syncSeen())));
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
@Override
protected void map(ImmutableBytesWritable key, Result value, Context context)
    throws IOException, InterruptedException {
  BytesWritable bwKey = new BytesWritable(key.get());
  BytesWritable bwVal = new BytesWritable();
  for (Cell kv : value.listCells()) {
    if (Bytes.compareTo(TEST_QUALIFIER, 0, TEST_QUALIFIER.length,
                        kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()) == 0) {
      context.write(bwKey, EMPTY);
    } else {
      bwVal.set(kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength());
      context.write(bwVal, bwKey);
    }
  }
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
public void next(BytesWritable key, BytesWritable value, boolean dupKey) {
  if (dupKey) {
    key.set(lastKey);
  }
  else {
    fillKey(key);
  }
  fillValue(value);
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
private void fillBuffer(Random rng, BytesWritable bw, byte[] tmp, int len) {
  int n = 0;
  while (n < len) {
    byte[] word = dictionary[rng.nextInt(dictionary.length)];
    int l = Math.min(word.length, len - n);
    System.arraycopy(word, 0, tmp, n, l);
    n += l;
  }
  bw.set(tmp, 0, len);
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
private void timeWrite(Path path, KVAppendable appendable, int baseKlen,
    int baseVlen, long fileSize) throws IOException {
  int maxKlen = baseKlen * 2;
  int maxVlen = baseVlen * 2;
  BytesWritable key = new BytesWritable();
  BytesWritable value = new BytesWritable();
  byte[] keyBuffer = new byte[maxKlen];
  byte[] valueBuffer = new byte[maxVlen];
  Random rng = new Random(options.seed);
  long totalBytes = 0;
  printlnWithTimestamp("Start writing: " + path.getName() + "...");
  startTime();

  for (long i = 0; true; ++i) {
    if (i % 1000 == 0) { // test the size for every 1000 rows.
      if (fs.getFileStatus(path).getLen() >= fileSize) {
        break;
      }
    }
    int klen = rng.nextInt(baseKlen) + baseKlen;
    int vlen = rng.nextInt(baseVlen) + baseVlen;
    fillBuffer(rng, key, keyBuffer, klen);
    fillBuffer(rng, value, valueBuffer, vlen);
    key.set(keyBuffer, 0, klen);
    value.set(valueBuffer, 0, vlen);
    appendable.append(key, value);
    totalBytes += klen;
    totalBytes += vlen;
  }
  stopTime();
  appendable.close();
  reportStats(path, totalBytes);
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
private void writeOutput(RemoteIterator<? extends FileStatus> input) throws IOException {
    Path outPath = new Path(output);
    if (distribFs.exists(outPath)) {
        throw new IllegalArgumentException("Output file already exists, Not overwriting it:" + output);
    }

    Writer writer = SequenceFile.createWriter(distribFs.getConf(),
            Writer.file(outPath),
            Writer.keyClass(Text.class),
            Writer.valueClass(BytesWritable.class),
            Writer.compression(SequenceFile.CompressionType.RECORD));
    Text key = new Text();
    BytesWritable value = new BytesWritable();
    long skipped = 0;
    long copied = 0;
    while (input.hasNext()) {
        FileStatus next = input.next();
        if (filter(next)) {
            key.set(next.getPath().toString());
            FSDataInputStream stream = localFs.open(next.getPath());
            //CAUTION : this could cause memory overflow
            byte[] bytes = IOUtils.toByteArray(stream);
            value.set(bytes, 0, bytes.length);
            writer.append(key, value);
            copied++;
        } else {
            skipped++;
        }
    }
    writer.close();
    System.out.println("Files copied ::" + copied);
    System.out.println("Files skipped ::" + skipped);
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
private Object convertBinaryTypes(Object val, String javaColType) {
  byte[] bb = (byte[]) val;
  if (javaColType.equals(BYTESWRITABLE)) {
    BytesWritable bw = new BytesWritable();
    bw.set(bb, 0, bb.length);
    return bw;
  }
  return null;
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
/**
 * Helper method to write using monarch-record-writer.
 *
 * @param lines the data/records to be written
 * @param conf the writer configuration
 * @return the key-prefix used by the monarch writer
 * @throws IOException
 */
public static String writeUsingRecordWriter(List<String> lines, Configuration conf) throws IOException {
  final MonarchRecordWriter mrw = new MonarchRecordWriter(conf);
  final BytesWritable bytesWritable = new BytesWritable();
  byte[] bytes;
  for (final String line : lines) {
    bytes = line.getBytes();
    bytesWritable.set(bytes, 0, bytes.length);
    mrw.write(bytesWritable);
  }
  mrw.close(true);
  return mrw.getKeyPrefix();
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
@Override
public synchronized boolean next(LongWritable key, BytesWritable value)
    throws IOException {
  boolean dataRead = reader.nextKeyValue();
  if (dataRead) {
    LongWritable newKey = reader.getCurrentKey();
    BytesWritable newValue = reader.getCurrentValue();
    key.set(newKey.get());
    value.set(newValue);
  }
  return dataRead;
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
private BytesWritable makeByteWritable(Event e) {
  BytesWritable bytesObject = new BytesWritable();
  bytesObject.set(e.getBody(), 0, e.getBody().length);
  return bytesObject;
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
/**
 * Convert from Avro type to Sqoop's java representation of the SQL type
 * see SqlManager#toJavaType
 */
public static Object fromAvro(Object avroObject, Schema schema, String type) {
  if (avroObject == null) {
    return null;
  }

  switch (schema.getType()) {
    case NULL:
      return null;
    case BOOLEAN:
    case INT:
    case FLOAT:
    case DOUBLE:
      return avroObject;
    case LONG:
      if (type.equals(DATE_TYPE)) {
        return new Date((Long) avroObject);
      } else if (type.equals(TIME_TYPE)) {
        return new Time((Long) avroObject);
      } else if (type.equals(TIMESTAMP_TYPE)) {
        return new Timestamp((Long) avroObject);
      }
      return avroObject;
    case BYTES:
      ByteBuffer bb = (ByteBuffer) avroObject;
      BytesWritable bw = new BytesWritable();
      bw.set(bb.array(), bb.arrayOffset() + bb.position(), bb.remaining());
      if (type.equals(BLOB_REF_TYPE)) {
        // TODO: Should convert BytesWritable to BlobRef properly. (SQOOP-991)
        throw new UnsupportedOperationException("BlobRef not supported");
      }
      return bw;
    case STRING:
      if (type.equals(BIG_DECIMAL_TYPE)) {
        return new BigDecimal(avroObject.toString());
      } else if (type.equals(DATE_TYPE)) {
        return Date.valueOf(avroObject.toString());
      } else if (type.equals(TIME_TYPE)) {
        return Time.valueOf(avroObject.toString());
      } else if (type.equals(TIMESTAMP_TYPE)) {
        return Timestamp.valueOf(avroObject.toString());
      }
      return avroObject.toString();
    case ENUM:
      return avroObject.toString();
    case UNION:
      List<Schema> types = schema.getTypes();
      if (types.size() != 2) {
        throw new IllegalArgumentException("Only support union with null");
      }
      Schema s1 = types.get(0);
      Schema s2 = types.get(1);
      if (s1.getType() == Schema.Type.NULL) {
        return fromAvro(avroObject, s2, type);
      } else if (s2.getType() == Schema.Type.NULL) {
        return fromAvro(avroObject, s1, type);
      } else {
        throw new IllegalArgumentException("Only support union with null");
      }
    case FIXED:
      return new BytesWritable(((GenericFixed) avroObject).bytes());
    case RECORD:
    case ARRAY:
    case MAP:
    default:
      throw new IllegalArgumentException("Cannot convert Avro type "
          + schema.getType());
  }
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
private static String textifyBytes(Text t) {
  BytesWritable b = new BytesWritable();
  b.set(t.getBytes(), 0, t.getLength());
  return b.toString();
}