Java SolrOutputFormat类代码示例

import org.apache.solr.hadoop.SolrOutputFormat; //导入依赖的package包/类
private FileStatus[] listSortedOutputShardDirs(Job job, Path outputReduceDir, FileSystem fs) throws FileNotFoundException,
        IOException {

    final String dirPrefix = SolrOutputFormat.getOutputName(job);
    FileStatus[] dirs = fs.listStatus(outputReduceDir, new PathFilter() {
        @Override
        public boolean accept(Path path) {
            return path.getName().startsWith(dirPrefix);
        }
    });
    for (FileStatus dir : dirs) {
        if (!dir.isDirectory()) {
            throw new IllegalStateException("Not a directory: " + dir.getPath());
        }
    }

    // use alphanumeric sort (rather than lexicographical sort) to properly handle more than 99999 shards
    Arrays.sort(dirs, new Comparator<FileStatus>() {
        @Override
        public int compare(FileStatus f1, FileStatus f2) {
            return new PublicAlphaNumericComparator().compare(f1.getPath().getName(), f2.getPath().getName());
        }
    });

    return dirs;
}
 

import org.apache.solr.hadoop.SolrOutputFormat; //导入依赖的package包/类
public SolrRecordWriter(TaskAttemptContext context, Path outputShardDir, int batchSize) {
  this.batchSize = batchSize;
  this.batch = new ArrayList(batchSize);
  Configuration conf = context.getConfiguration();

  // setLogLevel("org.apache.solr.core", "WARN");
  // setLogLevel("org.apache.solr.update", "WARN");

  heartBeater = new HeartBeater(context);
  try {
    heartBeater.needHeartBeat();

    Path solrHomeDir = SolrRecordWriter.findSolrConfig(conf);
    FileSystem fs = outputShardDir.getFileSystem(conf);
    EmbeddedSolrServer solr = createEmbeddedSolrServer(solrHomeDir, fs, outputShardDir);
    batchWriter = new BatchWriter(solr, batchSize,
        context.getTaskAttemptID().getTaskID(),
        SolrOutputFormat.getSolrWriterThreadCount(conf),
        SolrOutputFormat.getSolrWriterQueueSize(conf));

  } catch (Exception e) {
    throw new IllegalStateException(String.format(Locale.ENGLISH,
        "Failed to initialize record writer for %s, %s", context.getJobName(), conf
            .get("mapred.task.id")), e);
  } finally {
    heartBeater.cancelHeartBeat();
  }
}
 

import org.apache.solr.hadoop.SolrOutputFormat; //导入依赖的package包/类
private SolrInputDocumentWriter wrapInBufferedWriter(Context context, SolrInputDocumentWriter writer)
        throws MalformedURLException {
    int bufferSize = context.getConfiguration().getInt(SolrOutputFormat.SOLR_RECORD_WRITER_BATCH_SIZE, 100);

    return new BufferedSolrInputDocumentWriter(
            writer,
            bufferSize,
            context.getCounter(HBaseIndexerCounters.OUTPUT_INDEX_DOCUMENTS),
            context.getCounter(HBaseIndexerCounters.OUTPUT_INDEX_DOCUMENT_BATCHES));
}
 

import org.apache.solr.hadoop.SolrOutputFormat; //导入依赖的package包/类
public static Path findSolrConfig(Configuration conf) throws IOException {
  Path solrHome = null;
  // FIXME when mrunit supports the new cache apis
  //URI[] localArchives = context.getCacheArchives();
  Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
  if (localArchives.length == 0) {
    throw new IOException(String.format(Locale.ENGLISH,
        "No local cache archives, where is %s:%s", SolrOutputFormat
            .getSetupOk(), SolrOutputFormat.getZipName(conf)));
  }
  for (Path unpackedDir : localArchives) {
    // Only logged if debugging
    if (LOG.isDebugEnabled()) {
      LOG.debug(String.format(Locale.ENGLISH, "Examining unpack directory %s for %s",
          unpackedDir, SolrOutputFormat.getZipName(conf)));

      ProcessBuilder lsCmd = new ProcessBuilder(new String[] { "/bin/ls",
          "-lR", unpackedDir.toString() });
      lsCmd.redirectErrorStream();
      Process ls = lsCmd.start();
      byte[] buf = new byte[16 * 1024];
      InputStream all = ls.getInputStream();
      try {
        int count;
        while ((count = all.read(buf)) >= 0) {
          System.err.write(buf, 0, count);
        }
      } catch (IOException ignore) {
      } finally {
        all.close();
      }
      String exitValue;
      try {
        exitValue = String.valueOf(ls.waitFor());
      } catch (InterruptedException e) {
        exitValue = "interrupted";
      }
      System.err.format(Locale.ENGLISH, "Exit value of 'ls -lR' is %s%n", exitValue);
    }
    if (unpackedDir.getName().equals(SolrOutputFormat.getZipName(conf))) {
      LOG.info("Using this unpacked directory as solr home: {}", unpackedDir);
      solrHome = unpackedDir;
      break;
    }
  }

  return solrHome;
}