Java BytesWritable.getBytes方法代码示例

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
/**
 * @param compressionAlgo
 *          The compression algorithm to be used to for compression.
 * @throws IOException
 */
public WBlockState(Algorithm compressionAlgo, FSDataOutputStream fsOut,
    BytesWritable fsOutputBuffer, Configuration conf) throws IOException {
  this.compressAlgo = compressionAlgo;
  this.fsOut = fsOut;
  this.posStart = fsOut.getPos();

  fsOutputBuffer.setCapacity(TFile.getFSOutputBufferSize(conf));

  this.fsBufferedOutput =
      new SimpleBufferedOutputStream(this.fsOut, fsOutputBuffer.getBytes());
  this.compressor = compressAlgo.getCompressor();

  try {
    this.out =
        compressionAlgo.createCompressionStream(fsBufferedOutput,
            compressor, 0);
  } catch (IOException e) {
    compressAlgo.returnCompressor(compressor);
    throw e;
  }
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
@Override
public void map(LongWritable key, BytesWritable value, Context context)
    throws IOException, InterruptedException {

  String fileName = new String(value.getBytes(), 0,
      value.getLength(), charsetUTF8);
  Path path = new Path(fileName);

  FSDataOutputStream dos =
      FileSystem.create(fs, path, new FsPermission(GRIDMIX_DISTCACHE_FILE_PERM));

  int size = 0;
  for (long bytes = key.get(); bytes > 0; bytes -= size) {
    r.nextBytes(val.getBytes());
    size = (int)Math.min(val.getLength(), bytes);
    dos.write(val.getBytes(), 0, size);// Write to distCache file
  }
  dos.close();
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
public void next(BytesWritable key) {
  key.setSize(Math.max(MIN_KEY_LEN, keyLenRNG.nextInt()));
  random.nextBytes(key.getBytes());
  int n = random.nextInt(max - min) + min;
  byte[] b = key.getBytes();
  b[0] = (byte) (n >> 24);
  b[1] = (byte) (n >> 16);
  b[2] = (byte) (n >> 8);
  b[3] = (byte) n;
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
public static void writeBytesWritable(BytesWritable val, int paramIdx,
    int sqlType, PreparedStatement s) throws SQLException {
  if (null == val) {
    s.setNull(paramIdx, sqlType);
  } else {
    // val.getBytes() is only valid in [0, len)
    byte [] rawBytes = val.getBytes();
    int len = val.getLength();
    byte [] outBytes = new byte[len];
    System.arraycopy(rawBytes, 0, outBytes, 0, len);
    s.setBytes(paramIdx, outBytes);
  }
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
public String getString(Object k) {
  BytesWritable bytes = get(k);
  if (null == bytes) {
    return null;
  } else {
    try {
      return new String(bytes.getBytes(), 0, bytes.getLength(), "UTF-8");
    } catch (UnsupportedEncodingException uee) {
      // Shouldn't happen; UTF-8 is always supported.
      throw new RuntimeException(uee);
    }
  }
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
private static Rename[] readRenameOptions(DataInputStream in) throws IOException {
  BytesWritable writable = new BytesWritable();
  writable.readFields(in);

  byte[] bytes = writable.getBytes();
  Rename[] options = new Rename[bytes.length];

  for (int i = 0; i < bytes.length; i++) {
    options[i] = Rename.valueOf(bytes[i]);
  }
  return options;
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
/**
 * Validate setupGenerateDistCacheData by validating <li>permissions of the
 * distributed cache directory and <li>content of the generated sequence file.
 * This includes validation of dist cache file paths and their file sizes.
 */
private void doValidateSetupGenDC(
    RecordReader<LongWritable, BytesWritable> reader, FileSystem fs,
    long[] sortedFileSizes) throws IOException, InterruptedException {

  // Validate permissions of dist cache directory
  Path distCacheDir = dce.getDistributedCacheDir();
  assertEquals(
      "Wrong permissions for distributed cache dir " + distCacheDir,
      fs.getFileStatus(distCacheDir).getPermission().getOtherAction()
          .and(FsAction.EXECUTE), FsAction.EXECUTE);

  // Validate the content of the sequence file generated by
  // dce.setupGenerateDistCacheData().
  LongWritable key = new LongWritable();
  BytesWritable val = new BytesWritable();
  for (int i = 0; i < sortedFileSizes.length; i++) {
    assertTrue("Number of files written to the sequence file by "
        + "setupGenerateDistCacheData is less than the expected.",
        reader.nextKeyValue());
    key = reader.getCurrentKey();
    val = reader.getCurrentValue();
    long fileSize = key.get();
    String file = new String(val.getBytes(), 0, val.getLength());

    // Dist Cache files should be sorted based on file size.
    assertEquals("Dist cache file size is wrong.", sortedFileSizes[i],
        fileSize);

    // Validate dist cache file path.

    // parent dir of dist cache file
    Path parent = new Path(file).getParent().makeQualified(fs.getUri(),fs.getWorkingDirectory());
    // should exist in dist cache dir
    assertTrue("Public dist cache file path is wrong.",
        distCacheDir.equals(parent));
  }
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
  checkArgsSize(arguments, 2, 2);

  checkArgPrimitive(arguments, 0);
  checkArgPrimitive(arguments, 1);

  // the function should support both string and binary input types
  if (canParam0BeStr()) {
    checkArgGroups(arguments, 0, inputTypes, STRING_GROUP, BINARY_GROUP);
  } else {
    checkArgGroups(arguments, 0, inputTypes, BINARY_GROUP);
  }
  checkArgGroups(arguments, 1, inputTypes, STRING_GROUP, BINARY_GROUP);

  if (isStr0 = PrimitiveObjectInspectorUtils.getPrimitiveGrouping(inputTypes[0]) == STRING_GROUP) {
    obtainStringConverter(arguments, 0, inputTypes, converters);
  } else {
    obtainBinaryConverter(arguments, 0, inputTypes, converters);
  }

  isKeyConstant = arguments[1] instanceof ConstantObjectInspector;
  byte[] key = null;
  int keyLength = 0;

  if (isStr1 = PrimitiveObjectInspectorUtils.getPrimitiveGrouping(inputTypes[1]) == STRING_GROUP) {
    if (isKeyConstant) {
      String keyStr = getConstantStringValue(arguments, 1);
      if (keyStr != null) {
        key = keyStr.getBytes();
        keyLength = key.length;
      }
    } else {
      obtainStringConverter(arguments, 1, inputTypes, converters);
    }
  } else {
    if (isKeyConstant) {
      BytesWritable keyWr = getConstantBytesValue(arguments, 1);
      if (keyWr != null) {
        key = keyWr.getBytes();
        keyLength = keyWr.getLength();
      }
    } else {
      obtainBinaryConverter(arguments, 1, inputTypes, converters);
    }
  }

  if (key != null) {
    secretKey = getSecretKey(key, keyLength);
  }

  try {
    cipher = Cipher.getInstance("AES");
  } catch (NoSuchPaddingException | NoSuchAlgorithmException e) {
    throw new RuntimeException(e);
  }

  ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
  return outputOI;
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
private Object toHCat(Object val, HCatFieldSchema hfs) {
  HCatFieldSchema.Type hfsType = hfs.getType();
  if (val == null) {
    return null;
  }

  Object retVal = null;

  if (val instanceof Number) {
    retVal = convertNumberTypes(val, hfs);
  } else if (val instanceof Boolean) {
    retVal = convertBooleanTypes(val, hfs);
  } else if (val instanceof String) {
    retVal = convertStringTypes(val, hfs);
  } else if (val instanceof java.util.Date) {
    retVal = converDateTypes(val, hfs);
  } else if (val instanceof BytesWritable) {
    if (hfsType == HCatFieldSchema.Type.BINARY) {
      BytesWritable bw = (BytesWritable) val;
      retVal = bw.getBytes();
    }
  } else if (val instanceof BlobRef) {
    if (hfsType == HCatFieldSchema.Type.BINARY) {
      BlobRef br = (BlobRef) val;
      byte[] bytes = br.isExternal() ? br.toString().getBytes() : br
        .getData();
      retVal = bytes;
    }
  } else if (val instanceof ClobRef) {
    retVal = convertClobType(val, hfs);
  } else {
    throw new UnsupportedOperationException("Objects of type "
      + val.getClass().getName() + " are not suported");
  }
  if (retVal == null) {
    LOG.error("Unable to convert [" + val
      + "]  of type " + val.getClass().getName()
      + " to HCatalog type " + hfs.getTypeString());
  }
  return retVal;
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
@Override
protected InputStream getInternalSource(BytesWritable data) {
  return new ByteArrayInputStream(data.getBytes(), 0, data.getLength());
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
@Override
public void reduce(BytesWritable key, Iterable<BytesWritable> values, Context context)
    throws IOException, InterruptedException {
  int defCount = 0;
  boolean lostFamilies = false;
  refs.clear();
  for (BytesWritable type : values) {
    if (type.getLength() == DEF.getLength()) {
      defCount++;
      if (type.getBytes()[0] == 1) {
        lostFamilies = true;
      }
    } else {
      byte[] bytes = new byte[type.getLength()];
      System.arraycopy(type.getBytes(), 0, bytes, 0, type.getLength());
      refs.add(bytes);
    }
  }

  // TODO check for more than one def, should not happen
  StringBuilder refsSb = null;
  String keyString = Bytes.toStringBinary(key.getBytes(), 0, key.getLength());
  if (defCount == 0 || refs.size() != 1) {
    refsSb = dumpExtraInfoOnRefs(key, context, refs);
    LOG.error("LinkedListError: key=" + keyString + ", reference(s)=" +
      (refsSb != null? refsSb.toString(): ""));
  }
  if (lostFamilies) {
    LOG.error("LinkedListError: key=" + keyString + ", lost big or tiny families");
    context.getCounter(Counts.LOST_FAMILIES).increment(1);
    context.write(key, LOSTFAM);
  }

  if (defCount == 0 && refs.size() > 0) {
    // This is bad, found a node that is referenced but not defined. It must have been
    // lost, emit some info about this node for debugging purposes.
    // Write out a line per reference. If more than one, flag it.;
    for (int i = 0; i < refs.size(); i++) {
      byte[] bs = refs.get(i);
      int ordinal;
      if (i <= 0) {
        ordinal = Counts.UNDEFINED.ordinal();
        context.write(key, new BytesWritable(addPrefixFlag(ordinal, bs)));
        context.getCounter(Counts.UNDEFINED).increment(1);
      } else {
        ordinal = Counts.EXTRA_UNDEF_REFERENCES.ordinal();
        context.write(key, new BytesWritable(addPrefixFlag(ordinal, bs)));
      }
    }
    if (rows.addAndGet(1) < MISSING_ROWS_TO_LOG) {
      // Print out missing row; doing get on reference gives info on when the referencer
      // was added which can help a little debugging. This info is only available in mapper
      // output -- the 'Linked List error Key...' log message above. What we emit here is
      // useless for debugging.
      context.getCounter("undef", keyString).increment(1);
    }
  } else if (defCount > 0 && refs.size() == 0) {
    // node is defined but not referenced
    context.write(key, UNREF);
    context.getCounter(Counts.UNREFERENCED).increment(1);
    if (rows.addAndGet(1) < MISSING_ROWS_TO_LOG) {
      context.getCounter("unref", keyString).increment(1);
    }
  } else {
    if (refs.size() > 1) {
      // Skip first reference.
      for (int i = 1; i < refs.size(); i++) {
        context.write(key,
          new BytesWritable(addPrefixFlag(Counts.EXTRAREFERENCES.ordinal(), refs.get(i))));
      }
      context.getCounter(Counts.EXTRAREFERENCES).increment(refs.size() - 1);
    }
    // node is defined and referenced
    context.getCounter(Counts.REFERENCED).increment(1);
  }
}
 

import org.apache.hadoop.io.BytesWritable; //导入方法依赖的package包/类
/**
 * Constructing a ByteArray from a {@link BytesWritable}.
 * 
 * @param other
 */
public ByteArray(BytesWritable other) {
  this(other.getBytes(), 0, other.getLength());
}