Java LobFile.Writer方法代码示例

import com.cloudera.sqoop.io.LobFile; //导入方法依赖的package包/类
/**
 * Returns the path being written to by a given LobFile.Writer, relative
 * to the working directory of this LargeObjectLoader.
 * @param w the LobFile.Writer whose path should be examined.
 * @return the path this is writing to, relative to the current working dir.
 */
private String getRelativePath(LobFile.Writer w) {
  Path writerPath = w.getPath();

  String writerPathStr = writerPath.toString();
  String workPathStr = workPath.toString();
  if (!workPathStr.endsWith(File.separator)) {
    workPathStr = workPathStr + File.separator;
  }

  if (writerPathStr.startsWith(workPathStr)) {
    return writerPathStr.substring(workPathStr.length());
  }

  // Outside the working dir; return the whole thing.
  return writerPathStr;
}
 

import com.cloudera.sqoop.io.LobFile; //导入方法依赖的package包/类
/**
 * @return the current LobFile writer for BLOBs, creating one if necessary.
 */
private LobFile.Writer getBlobWriter() throws IOException {
  if (null == this.curBlobWriter) {
    this.curBlobWriter = LobFile.create(getNextLobFilePath(), conf, false);
  }

  return this.curBlobWriter;
}
 

import com.cloudera.sqoop.io.LobFile; //导入方法依赖的package包/类
/**
 * @return the current LobFile writer for CLOBs, creating one if necessary.
 */
private LobFile.Writer getClobWriter() throws IOException {
  if (null == this.curClobWriter) {
    this.curClobWriter = LobFile.create(getNextLobFilePath(), conf, true);
  }

  return this.curClobWriter;
}
 

import com.cloudera.sqoop.io.LobFile; //导入方法依赖的package包/类
/**
 * Actually read a BlobRef instance from the ResultSet and materialize
 * the data either inline or to a file.
 *
 * @param colNum the column of the ResultSet's current row to read.
 * @param r the ResultSet to read from.
 * @return a BlobRef encapsulating the data in this field.
 * @throws IOException if an error occurs writing to the FileSystem.
 * @throws SQLException if an error occurs reading from the database.
 */
public com.cloudera.sqoop.lib.BlobRef readBlobRef(int colNum, ResultSet r)
    throws IOException, InterruptedException, SQLException {

  long maxInlineLobLen = conf.getLong(
      MAX_INLINE_LOB_LEN_KEY,
      DEFAULT_MAX_LOB_LENGTH);

  Blob b = r.getBlob(colNum);
  if (null == b) {
    return null;
  } else if (b.length() > maxInlineLobLen) {
    // Deserialize very large BLOBs into separate files.
    long len = b.length();
    LobFile.Writer lobWriter = getBlobWriter();

    long recordOffset = lobWriter.tell();
    InputStream is = null;
    OutputStream os = lobWriter.writeBlobRecord(len);
    try {
      is = b.getBinaryStream();
      copyAll(is, os);
    } finally {
      if (null != os) {
        os.close();
      }

      if (null != is) {
        is.close();
      }

      // Mark the record as finished.
      lobWriter.finishRecord();
    }

    return new com.cloudera.sqoop.lib.BlobRef(
        getRelativePath(curBlobWriter), recordOffset, len);
  } else {
    // This is a 1-based array.
    return new com.cloudera.sqoop.lib.BlobRef(
        b.getBytes(1, (int) b.length()));
  }
}
 

import com.cloudera.sqoop.io.LobFile; //导入方法依赖的package包/类
/**
 * Actually read a ClobRef instance from the ResultSet and materialize
 * the data either inline or to a file.
 *
 * @param colNum the column of the ResultSet's current row to read.
 * @param r the ResultSet to read from.
 * @return a ClobRef encapsulating the data in this field.
 * @throws IOException if an error occurs writing to the FileSystem.
 * @throws SQLException if an error occurs reading from the database.
 */
public com.cloudera.sqoop.lib.ClobRef readClobRef(int colNum, ResultSet r)
    throws IOException, InterruptedException, SQLException {

  long maxInlineLobLen = conf.getLong(
      MAX_INLINE_LOB_LEN_KEY,
      DEFAULT_MAX_LOB_LENGTH);

  Clob c = r.getClob(colNum);
  if (null == c) {
    return null;
  } else if (c.length() > maxInlineLobLen) {
    // Deserialize large CLOB into separate file.
    long len = c.length();
    LobFile.Writer lobWriter = getClobWriter();

    long recordOffset = lobWriter.tell();
    Reader reader = null;
    Writer w = lobWriter.writeClobRecord(len);
    try {
      reader = c.getCharacterStream();
      copyAll(reader, w);
    } finally {
      if (null != w) {
        w.close();
      }

      if (null != reader) {
        reader.close();
      }

      // Mark the record as finished.
      lobWriter.finishRecord();
    }

    return new com.cloudera.sqoop.lib.ClobRef(
        getRelativePath(lobWriter), recordOffset, len);
  } else {
    // This is a 1-based array.
    return new com.cloudera.sqoop.lib.ClobRef(
        c.getSubString(1, (int) c.length()));
  }
}
 

import com.cloudera.sqoop.io.LobFile; //导入方法依赖的package包/类
private void doExternalTest(final String data, final String filename)
    throws IOException {

  Configuration conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  FileSystem fs = FileSystem.get(conf);
  String tmpDir = System.getProperty("test.build.data", "/tmp/");

  Path tmpPath = new Path(tmpDir);
  Path clobFile = new Path(tmpPath, filename);

  // make any necessary parent dirs.
  Path clobParent = clobFile.getParent();
  if (!fs.exists(clobParent)) {
    fs.mkdirs(clobParent);
  }

  LobFile.Writer lw = LobFile.create(clobFile, conf, true);
  try {
    long off = lw.tell();
    long len = data.length();
    Writer w = lw.writeClobRecord(len);
    w.append(data);
    w.close();
    lw.close();

    String refString = "externalLob(lf," + filename
              + "," + off + "," + len + ")";
    ClobRef clob = ClobRef.parse(refString);
    assertTrue(clob.isExternal());
    assertEquals(refString, clob.toString());
    Reader r = clob.getDataStream(conf, tmpPath);
    assertNotNull(r);

    char [] buf = new char[4096];
    int chars = r.read(buf, 0, 4096);
    r.close();

    String str = new String(buf, 0, chars);
    assertEquals(data, str);
  } finally {
    fs.delete(clobFile, false);
  }
}
 

import com.cloudera.sqoop.io.LobFile; //导入方法依赖的package包/类
private void doExternalTest(final byte [] data, final String filename)
    throws IOException {

  Configuration conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  FileSystem fs = FileSystem.get(conf);
  String tmpDir = System.getProperty("test.build.data", "/tmp/");

  Path tmpPath = new Path(tmpDir);
  Path blobFile = new Path(tmpPath, filename);

  // make any necessary parent dirs.
  Path blobParent = blobFile.getParent();
  if (!fs.exists(blobParent)) {
    fs.mkdirs(blobParent);
  }

  LobFile.Writer lw = LobFile.create(blobFile, conf, false);
  try {
    long off = lw.tell();
    long len = data.length;
    OutputStream os = lw.writeBlobRecord(len);
    os.write(data, 0, data.length);
    os.close();
    lw.close();

    String refString = "externalLob(lf," + filename
        + "," + off + "," + len + ")";
    BlobRef blob = BlobRef.parse(refString);
    assertTrue(blob.isExternal());
    assertEquals(refString, blob.toString());
    InputStream is = blob.getDataStream(conf, tmpPath);
    assertNotNull(is);

    byte [] buf = new byte[4096];
    int bytes = is.read(buf, 0, 4096);
    is.close();

    assertEquals(data.length, bytes);
    for (int i = 0; i < bytes; i++) {
      assertEquals(data[i], buf[i]);
    }
  } finally {
    fs.delete(blobFile, false);
  }
}