Java LobFile类代码示例

import com.cloudera.sqoop.io.LobFile; //导入依赖的package包/类
/**
 * Returns the path being written to by a given LobFile.Writer, relative
 * to the working directory of this LargeObjectLoader.
 * @param w the LobFile.Writer whose path should be examined.
 * @return the path this is writing to, relative to the current working dir.
 */
private String getRelativePath(LobFile.Writer w) {
  Path writerPath = w.getPath();

  String writerPathStr = writerPath.toString();
  String workPathStr = workPath.toString();
  if (!workPathStr.endsWith(File.separator)) {
    workPathStr = workPathStr + File.separator;
  }

  if (writerPathStr.startsWith(workPathStr)) {
    return writerPathStr.substring(workPathStr.length());
  }

  // Outside the working dir; return the whole thing.
  return writerPathStr;
}
 

import com.cloudera.sqoop.io.LobFile; //导入依赖的package包/类
/**
 * Open a LobFile for read access, returning a cached reader if one is
 * available, or a new reader otherwise.
 * @param path the path to the LobFile to open
 * @param conf the configuration to use to access the FS.
 * @throws IOException if there's an error opening the file.
 */
public LobFile.Reader get(Path path, Configuration conf)
    throws IOException {

  LobFile.Reader reader = null;
  Path canonicalPath = qualify(path, conf);
  // Look up an entry in the cache.
  synchronized(this) {
    reader = readerMap.remove(canonicalPath);
  }

  if (null != reader && !reader.isClosed()) {
    // Cache hit. return it.
    LOG.debug("Using cached reader for " + canonicalPath);
    return reader;
  }

  // Cache miss; open the file.
  LOG.debug("No cached reader available for " + canonicalPath);
  return LobFile.open(path, conf);
}
 

import com.cloudera.sqoop.io.LobFile; //导入依赖的package包/类
/**
 * Return a reader back to the cache. If there's already a reader for
 * this path, then the current reader is closed.
 * @param reader the opened reader. Any record-specific subreaders should be
 * closed.
 * @throws IOException if there's an error accessing the path's filesystem.
 */
public void recycle(LobFile.Reader reader) throws IOException {
  Path canonicalPath = reader.getPath();

  // Check if the cache has a reader for this path already. If not, add this.
  boolean cached = false;
  synchronized(this) {
    if (readerMap.get(canonicalPath) == null) {
      LOG.debug("Caching reader for path: " + canonicalPath);
      readerMap.put(canonicalPath, reader);
      cached = true;
    }
  }

  if (!cached) {
    LOG.debug("Reader already present for path: " + canonicalPath
        + "; closing.");
    reader.close();
  }
}
 

import com.cloudera.sqoop.io.LobFile; //导入依赖的package包/类
/**
 * @return the current LobFile writer for BLOBs, creating one if necessary.
 */
private LobFile.Writer getBlobWriter() throws IOException {
  if (null == this.curBlobWriter) {
    this.curBlobWriter = LobFile.create(getNextLobFilePath(), conf, false);
  }

  return this.curBlobWriter;
}
 

import com.cloudera.sqoop.io.LobFile; //导入依赖的package包/类
/**
 * @return the current LobFile writer for CLOBs, creating one if necessary.
 */
private LobFile.Writer getClobWriter() throws IOException {
  if (null == this.curClobWriter) {
    this.curClobWriter = LobFile.create(getNextLobFilePath(), conf, true);
  }

  return this.curClobWriter;
}
 

import com.cloudera.sqoop.io.LobFile; //导入依赖的package包/类
/**
 * Actually read a BlobRef instance from the ResultSet and materialize
 * the data either inline or to a file.
 *
 * @param colNum the column of the ResultSet's current row to read.
 * @param r the ResultSet to read from.
 * @return a BlobRef encapsulating the data in this field.
 * @throws IOException if an error occurs writing to the FileSystem.
 * @throws SQLException if an error occurs reading from the database.
 */
public com.cloudera.sqoop.lib.BlobRef readBlobRef(int colNum, ResultSet r)
    throws IOException, InterruptedException, SQLException {

  long maxInlineLobLen = conf.getLong(
      MAX_INLINE_LOB_LEN_KEY,
      DEFAULT_MAX_LOB_LENGTH);

  Blob b = r.getBlob(colNum);
  if (null == b) {
    return null;
  } else if (b.length() > maxInlineLobLen) {
    // Deserialize very large BLOBs into separate files.
    long len = b.length();
    LobFile.Writer lobWriter = getBlobWriter();

    long recordOffset = lobWriter.tell();
    InputStream is = null;
    OutputStream os = lobWriter.writeBlobRecord(len);
    try {
      is = b.getBinaryStream();
      copyAll(is, os);
    } finally {
      if (null != os) {
        os.close();
      }

      if (null != is) {
        is.close();
      }

      // Mark the record as finished.
      lobWriter.finishRecord();
    }

    return new com.cloudera.sqoop.lib.BlobRef(
        getRelativePath(curBlobWriter), recordOffset, len);
  } else {
    // This is a 1-based array.
    return new com.cloudera.sqoop.lib.BlobRef(
        b.getBytes(1, (int) b.length()));
  }
}
 

import com.cloudera.sqoop.io.LobFile; //导入依赖的package包/类
@Override
protected synchronized void finalize() throws Throwable {
  for (LobFile.Reader r : readerMap.values()) {
    r.close();
  }

  super.finalize();
}
 

import com.cloudera.sqoop.io.LobFile; //导入依赖的package包/类
/**
 * Actually read a ClobRef instance from the ResultSet and materialize
 * the data either inline or to a file.
 *
 * @param colNum the column of the ResultSet's current row to read.
 * @param r the ResultSet to read from.
 * @return a ClobRef encapsulating the data in this field.
 * @throws IOException if an error occurs writing to the FileSystem.
 * @throws SQLException if an error occurs reading from the database.
 */
public com.cloudera.sqoop.lib.ClobRef readClobRef(int colNum, ResultSet r)
    throws IOException, InterruptedException, SQLException {

  long maxInlineLobLen = conf.getLong(
      MAX_INLINE_LOB_LEN_KEY,
      DEFAULT_MAX_LOB_LENGTH);

  Clob c = r.getClob(colNum);
  if (null == c) {
    return null;
  } else if (c.length() > maxInlineLobLen) {
    // Deserialize large CLOB into separate file.
    long len = c.length();
    LobFile.Writer lobWriter = getClobWriter();

    long recordOffset = lobWriter.tell();
    Reader reader = null;
    Writer w = lobWriter.writeClobRecord(len);
    try {
      reader = c.getCharacterStream();
      copyAll(reader, w);
    } finally {
      if (null != w) {
        w.close();
      }

      if (null != reader) {
        reader.close();
      }

      // Mark the record as finished.
      lobWriter.finishRecord();
    }

    return new com.cloudera.sqoop.lib.ClobRef(
        getRelativePath(lobWriter), recordOffset, len);
  } else {
    // This is a 1-based array.
    return new com.cloudera.sqoop.lib.ClobRef(
        c.getSubString(1, (int) c.length()));
  }
}
 

import com.cloudera.sqoop.io.LobFile; //导入依赖的package包/类
@Override
protected Reader getExternalSource(LobFile.Reader reader)
    throws IOException {
  return reader.readClobRecord();
}
 

import com.cloudera.sqoop.io.LobFile; //导入依赖的package包/类
@Override
protected InputStream getExternalSource(LobFile.Reader reader)
    throws IOException {
  return reader.readBlobRecord();
}
 

import com.cloudera.sqoop.io.LobFile; //导入依赖的package包/类
protected LobReaderCache() {
  this.readerMap = new TreeMap<Path, LobFile.Reader>();
}
 

import com.cloudera.sqoop.io.LobFile; //导入依赖的package包/类
private void doExternalTest(final String data, final String filename)
    throws IOException {

  Configuration conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  FileSystem fs = FileSystem.get(conf);
  String tmpDir = System.getProperty("test.build.data", "/tmp/");

  Path tmpPath = new Path(tmpDir);
  Path clobFile = new Path(tmpPath, filename);

  // make any necessary parent dirs.
  Path clobParent = clobFile.getParent();
  if (!fs.exists(clobParent)) {
    fs.mkdirs(clobParent);
  }

  LobFile.Writer lw = LobFile.create(clobFile, conf, true);
  try {
    long off = lw.tell();
    long len = data.length();
    Writer w = lw.writeClobRecord(len);
    w.append(data);
    w.close();
    lw.close();

    String refString = "externalLob(lf," + filename
              + "," + off + "," + len + ")";
    ClobRef clob = ClobRef.parse(refString);
    assertTrue(clob.isExternal());
    assertEquals(refString, clob.toString());
    Reader r = clob.getDataStream(conf, tmpPath);
    assertNotNull(r);

    char [] buf = new char[4096];
    int chars = r.read(buf, 0, 4096);
    r.close();

    String str = new String(buf, 0, chars);
    assertEquals(data, str);
  } finally {
    fs.delete(clobFile, false);
  }
}
 

import com.cloudera.sqoop.io.LobFile; //导入依赖的package包/类
private void doExternalTest(final byte [] data, final String filename)
    throws IOException {

  Configuration conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  FileSystem fs = FileSystem.get(conf);
  String tmpDir = System.getProperty("test.build.data", "/tmp/");

  Path tmpPath = new Path(tmpDir);
  Path blobFile = new Path(tmpPath, filename);

  // make any necessary parent dirs.
  Path blobParent = blobFile.getParent();
  if (!fs.exists(blobParent)) {
    fs.mkdirs(blobParent);
  }

  LobFile.Writer lw = LobFile.create(blobFile, conf, false);
  try {
    long off = lw.tell();
    long len = data.length;
    OutputStream os = lw.writeBlobRecord(len);
    os.write(data, 0, data.length);
    os.close();
    lw.close();

    String refString = "externalLob(lf," + filename
        + "," + off + "," + len + ")";
    BlobRef blob = BlobRef.parse(refString);
    assertTrue(blob.isExternal());
    assertEquals(refString, blob.toString());
    InputStream is = blob.getDataStream(conf, tmpPath);
    assertNotNull(is);

    byte [] buf = new byte[4096];
    int bytes = is.read(buf, 0, 4096);
    is.close();

    assertEquals(data.length, bytes);
    for (int i = 0; i < bytes; i++) {
      assertEquals(data[i], buf[i]);
    }
  } finally {
    fs.delete(blobFile, false);
  }
}
 

import com.cloudera.sqoop.io.LobFile; //导入依赖的package包/类
/**
 * Using the LobFile reader, get an accessor InputStream or Reader to the
 * underlying data.
 */
protected abstract ACCESSORTYPE getExternalSource(LobFile.Reader reader)
    throws IOException;