Java UploadFileVec類代碼示例

import water.fvec.UploadFileVec; //導入依賴的package包/類
/**
 * Discover the parse setup needed to correctly parse all files.
 * This takes a ParseSetup as guidance.  Each file is examined
 * individually and then results merged.  If a conflict exists
 * between any results all files are re-examined using the
 * best guess from the first examination.
 *
 * @param fkeys Keys to input vectors to be parsed
 * @param userSetup Setup guidance from user
 * @return ParseSetup settings from looking at all files
 */
public static ParseSetup guessSetup( Key[] fkeys, ParseSetup userSetup ) {
  //Guess setup of each file and collect results
  GuessSetupTsk t = new GuessSetupTsk(userSetup);
  t.doAll(fkeys).getResult();

  //Calc chunk-size
  Iced ice = DKV.getGet(fkeys[0]);
  if (ice instanceof Frame && ((Frame) ice).vec(0) instanceof UploadFileVec) {
    t._gblSetup._chunk_size = FileVec.DFLT_CHUNK_SIZE;
  } else {
    t._gblSetup._chunk_size = FileVec.calcOptimalChunkSize(t._totalParseSize, t._gblSetup._number_columns);
  }

  return t._gblSetup;
}
 

import water.fvec.UploadFileVec; //導入依賴的package包/類
@Override
protected void doPost(HttpServletRequest request, HttpServletResponse response) {
  String uri = JettyHTTPD.getDecodedUri(request);

  try {
    String destination_frame = request.getParameter("destination_frame");
    if (destination_frame == null) {
      destination_frame = "upload" + Key.rand();
    }

    //
    // Here is an example of how to upload a file from the command line.
    //
    // curl -v -F "[email protected]_headers.zip" "http://localhost:54321/3/PostFile.bin?destination_frame=a.zip"
    //
    // JSON Payload returned is:
    //     { "destination_frame": "key_name", "total_bytes": nnn }
    //
    InputStream is = JettyHTTPD.extractPartInputStream(request, response);
    if (is == null) {
      return;
    }

    UploadFileVec.ReadPutStats stats = new UploadFileVec.ReadPutStats();
    UploadFileVec.readPut(destination_frame, is, stats);
    String responsePayload = "{ " +
        "\"destination_frame\": \"" + destination_frame + "\", " +
        "\"total_bytes\": " + stats.total_bytes + " " +
        "}\n";
    response.setContentType("application/json");
    response.getWriter().write(responsePayload);
  } catch (Exception e) {
    JettyHTTPD.sendErrorResponse(response, e, uri);
  } finally {
    JettyHTTPD.logRequest("POST", request, response);
  }
}
 

import water.fvec.UploadFileVec; //導入依賴的package包/類
private void fileUpload(String boundary, InputStream in, Properties parms) throws InterruptedException {
  try {
    String line = readLine(in);
    int i = line.indexOf(boundary);
    if (i!=2)
      sendError( HTTP_BADREQUEST, "BAD REQUEST: Content type is multipart/form-data but next chunk does not start with boundary. Usage: GET /example/file.html" );
    if (line.substring(i+boundary.length()).startsWith("--"))
      return;
    // read the header
    Properties item = new Properties();
    line = readLine(in);
    while ((line != null) && (line.trim().length()>0)) {
      int p = line.indexOf(':');
      if (p != -1)
        item.put( line.substring(0,p).trim().toLowerCase(), line.substring(p+1).trim());
      line = readLine(in);
    }
    // analyze the header
    if (line!=null) {
      String contentDisposition = item.getProperty("content-disposition");
      if (contentDisposition == null) {
        sendError( HTTP_BADREQUEST, "BAD REQUEST: Content type is multipart/form-data but no content-disposition info found. Usage: GET /example/file.html" );
      }
      String key = parms.getProperty("key");
      UploadFileVec.readPut(key, new InputStreamWrapper(in, boundary.getBytes()));
    }
  }
  catch (Exception e) {
    sendError( HTTP_INTERNALERROR, "SERVER INTERNAL ERROR: Exception: " + e.getMessage());
  }
}
 

import water.fvec.UploadFileVec; //導入依賴的package包/類
@Override
protected void doPost(HttpServletRequest request,
                      HttpServletResponse response) throws IOException, ServletException {
  String uri = getDecodedUri(request);

  try {
    String destination_frame = request.getParameter("destination_frame");
    if (destination_frame == null) {
      destination_frame = "upload" + Key.rand();
    }
    if (!validKeyName(destination_frame)) {
      setResponseStatus(response, HttpServletResponse.SC_BAD_REQUEST);
      response.getWriter().write("Invalid key name, contains illegal characters");
      return;
    }

    //
    // Here is an example of how to upload a file from the command line.
    //
    // curl -v -F "[email protected]_headers.zip" "http://localhost:54321/PostFile.bin?destination_frame=a.zip"
    //
    // JSON Payload returned is:
    //     { "destination_frame": "key_name", "total_bytes": nnn }
    //
    InputStream is = extractPartInputStream(request, response);
    if (is == null) {
      return;
    }

    UploadFileVec.ReadPutStats stats = new UploadFileVec.ReadPutStats();
    UploadFileVec.readPut(destination_frame, is, stats);
    String responsePayload = "{ "       +
            "\"destination_frame\": \"" + destination_frame   + "\", " +
            "\"total_bytes\": "         + stats.total_bytes + " " +
            "}\n";
    response.setContentType("application/json");
    response.getWriter().write(responsePayload);
  }
  catch (Exception e) {
    sendErrorResponse(response, e, uri);
  } finally {
    logRequest("POST", request, response);
  }
}
 

import water.fvec.UploadFileVec; //導入依賴的package包/類
/**
 * From a path produce a list of files and keys for parsing.
 *
 * Use as follows:
 *
 * ArrayList<String> files = new ArrayList();
 * ArrayList<String> keys = new ArrayList();
 * ArrayList<String> fails = new ArrayList();
 * ArrayList<String> dels = new ArrayList();
 * importFiles(importFiles.path, files, keys, fails, dels);
 *
 * @param path  (Input) Path to import data from
 * @param pattern (Input) Regex pattern to match files by
 * @param files (Output) List of files found
 * @param keys  (Output) List of keys corresponding to files
 * @param fails (Output) List of failed files which mismatch among nodes
 * @param dels  (Output) I don't know what this is
 */
public void importFiles(String path, String pattern, ArrayList<String> files, ArrayList<String> keys, ArrayList<String> fails, ArrayList<String> dels) {
  URI uri = FileUtils.getURI(path);
  String scheme = uri.getScheme();
  if (scheme == null || "file".equals(scheme)) {
    I[Value.NFS].importFiles(path, pattern, files, keys, fails, dels);
  } else if ("http".equals(scheme) || "https".equals(scheme)) {
    try {
      java.net.URL url = new URL(path);
      Key destination_key = Key.make(path);
      java.io.InputStream is = url.openStream();
      UploadFileVec.ReadPutStats stats = new UploadFileVec.ReadPutStats();
      UploadFileVec.readPut(destination_key, is, stats);
      files.add(path);
      keys.add(destination_key.toString());
    } catch( Throwable e) {
      fails.add(path); // Fails for e.g. broken sockets silently swallow exceptions and just record the failed path
    }
  } else if ("s3".equals(scheme)) {
    if (I[Value.S3] == null) throw new H2OIllegalArgumentException("S3 support is not configured");
    I[Value.S3].importFiles(path, pattern, files, keys, fails, dels);
  } else if ("hdfs".equals(scheme) ||
      "s3n:".equals(scheme) ||
      "s3a:".equals(scheme) ||
      "maprfs:".equals(scheme) ||
      (useHdfsAsFallback() && I[Value.HDFS] != null && I[Value.HDFS].canHandle(path))) {
    if (I[Value.HDFS] == null) throw new H2OIllegalArgumentException("HDFS, S3N, and S3A support is not configured");
    I[Value.HDFS].importFiles(path, pattern, files, keys, fails, dels);
  }

  if(pattern != null && !pattern.isEmpty()) {
    files.retainAll(matchPattern(path,files,pattern)); //New files ArrayList after matching pattern of choice
    keys.retainAll(matchPattern(path,keys,pattern)); //New keys ArrayList after matching pattern of choice
    //New fails ArrayList after matching pattern of choice. Only show failures that match pattern
    if(!fails.isEmpty()) {
      fails.retainAll(matchPattern(path, fails, pattern));
    }
  }

  return;
}