Java LineReader.close方法代码示例

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
public void testUTF8() throws Exception {
  LineReader in = null;

  try {
    in = makeStream("abcd\u20acbdcd\u20ac");
    Text line = new Text();
    in.readLine(line);
    assertEquals("readLine changed utf8 characters",
                 "abcd\u20acbdcd\u20ac", line.toString());
    in = makeStream("abc\u200axyz");
    in.readLine(line);
    assertEquals("split on fake newline", "abc\u200axyz", line.toString());
  } finally {
    if (in != null) {
      in.close();
    }
  }
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
public void testNewLines() throws Exception {
  LineReader in = null;
  try {
    in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
    Text out = new Text();
    in.readLine(out);
    assertEquals("line1 length", 1, out.getLength());
    in.readLine(out);
    assertEquals("line2 length", 2, out.getLength());
    in.readLine(out);
    assertEquals("line3 length", 0, out.getLength());
    in.readLine(out);
    assertEquals("line4 length", 3, out.getLength());
    in.readLine(out);
    assertEquals("line5 length", 4, out.getLength());
    in.readLine(out);
    assertEquals("line5 length", 5, out.getLength());
    assertEquals("end of file", 0, in.readLine(out));
  } finally {
    if (in != null) {
      in.close();
    }
  }
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
/**
 * Returns a split for each store files directory using the block location
 * of each file as locality reference.
 */
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
  List<InputSplit> splits = new ArrayList<InputSplit>();
  List<FileStatus> files = listStatus(job);

  Text key = new Text();
  for (FileStatus file: files) {
    Path path = file.getPath();
    FileSystem fs = path.getFileSystem(job.getConfiguration());
    LineReader reader = new LineReader(fs.open(path));
    long pos = 0;
    int n;
    try {
      while ((n = reader.readLine(key)) > 0) {
        String[] hosts = getStoreDirHosts(fs, path);
        splits.add(new FileSplit(path, pos, n, hosts));
        pos += n;
      }
    } finally {
      reader.close();
    }
  }

  return splits;
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
public static boolean distributeCache(String chrList, Job job, String cacheName)
		throws IOException, URISyntaxException {
	job.addCacheFile(new URI(chrList + "#" + cacheName));

	Configuration conf = job.getConfiguration();
	Path refPath = new Path(chrList);
	FileSystem fs = refPath.getFileSystem(conf);
	FSDataInputStream refin = fs.open(refPath);
	LineReader in = new LineReader(refin);
	Text line = new Text();

	String chrFile = "";
	String[] chrs = new String[3];
	while ((in.readLine(line)) != 0) {
		chrFile = line.toString();
		chrs = chrFile.split("\t");
		File fileTest = new File(chrs[1]);
		if (fileTest.isFile()) {
			chrs[1] = "file://" + chrs[1];
		}
		job.addCacheFile(new URI(chrs[1] + "#" + chrs[0]));
	}
	in.close();
	refin.close();
	return true;
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
protected void loadChromosomeList(Path refPath) throws NumberFormatException, IOException{
	Configuration conf = new Configuration();
	FileSystem fs = refPath.getFileSystem(conf);
	FSDataInputStream refin = fs.open(refPath);
	LineReader in = new LineReader(refin);
	Text line = new Text();
	
	String chrFile = "";
	String[] chrs = new String[3];
	while((in.readLine(line)) != 0){
		chrFile = line.toString();
		chrs = chrFile.split("\t");
		
		// insert chr
		if(!addChromosome(chrs[0])) {
			in.close();
			throw new RuntimeException("map Chromosome "+chrs[1]+" Failed.");
		}
		setChromosome(chrs[1],chrs[0],Integer.parseInt(chrs[2]));
	}
	in.close();
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
public void readFromHdfs(Path path, Configuration conf,
		FastqQualityControlReport report) throws IOException {
	FileSystem fs = path.getFileSystem(conf);
	FSDataInputStream FSinput = fs.open(path);

	LineReader lineReader = new LineReader(FSinput, conf);
	Text line = new Text();
	int sampleID = 0, i,cnt;
	while ((lineReader.readLine(line)) != 0) {
		sampleID = report.addCount(line.toString());
		if(report.isPartitionNull())
			continue;

		for (i = 0; i < FastqQualityControlReport.BASE_STATIC_COUNT; i++) {
			cnt = lineReader.readLine(line);
			if(cnt == 0)
				continue;
			report.addBaseByPosition(sampleID, i, line.toString());
		}
	}

	lineReader.close();
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
/**
 * Creates a mappings file from the contents of a flat text file containing docid to docno
 * mappings. This method is used by {@link WikipediaDocnoMappingBuilder} internally.
 *
 * @param inputFile flat text file containing docid to docno mappings
 * @param outputFile output mappings file
 * @throws IOException
 */
static public void writeDocnoMappingData(FileSystem fs, String inputFile, int n, String outputFile) throws IOException {
	LOG.info("Writing " + n + " docids to " + outputFile);
	LineReader reader = new LineReader(fs.open(new Path(inputFile)));

	int cnt = 0;
	Text line = new Text();

	FSDataOutputStream out = fs.create(new Path(outputFile), true);
	out.writeInt(n);
	for (int i = 0; i < n; i++) {
		reader.readLine(line);
		String[] arr = line.toString().split("\\t");
		out.writeInt(Integer.parseInt(arr[0]));
		cnt++;
		if (cnt % 100000 == 0) {
			LOG.info(cnt + " articles");
		}
	}
	out.close();
	reader.close();
	LOG.info("Done!");
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
@Test
public void testNewLines() throws Exception {
  LineReader in = null;
  try {
    in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
    Text out = new Text();
    in.readLine(out);
    assertEquals("line1 length", 1, out.getLength());
    in.readLine(out);
    assertEquals("line2 length", 2, out.getLength());
    in.readLine(out);
    assertEquals("line3 length", 0, out.getLength());
    in.readLine(out);
    assertEquals("line4 length", 3, out.getLength());
    in.readLine(out);
    assertEquals("line5 length", 4, out.getLength());
    in.readLine(out);
    assertEquals("line5 length", 5, out.getLength());
    assertEquals("end of file", 0, in.readLine(out));
  } finally {
    if (in != null) {
      in.close();
    }
  }
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
@Test
public void testUTF8() throws Exception {
  LineReader in = null;

  try {
    in = makeStream("abcd\u20acbdcd\u20ac");
    Text line = new Text();
    in.readLine(line);
    assertEquals("readLine changed utf8 characters",
                 "abcd\u20acbdcd\u20ac", line.toString());
    in = makeStream("abc\u200axyz");
    in.readLine(line);
    assertEquals("split on fake newline", "abc\u200axyz", line.toString());
  } finally {
    if (in != null) {
      in.close();
    }
  }
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
/**
 * Parse the command line arguments into lines and display the result.
 * @param args
 * @throws Exception
 */
public static void main(String[] args) throws Exception {
  for(String arg: args) {
    System.out.println("Working on " + arg);
    LineReader reader = makeStream(unquote(arg));
    Text line = new Text();
    int size = reader.readLine(line);
    while (size > 0) {
      System.out.println("Got: " + line.toString());
      size = reader.readLine(line);
    }
    reader.close();
  }
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
  // generate splits
  List<InputSplit> splitList = new ArrayList<InputSplit>();

  for (FileStatus file: listStatus(job)) {
    if (file.isDirectory()) {
      continue;
    }
    Path path = file.getPath();
    FileSystem fs = path.getFileSystem(job.getConfiguration());
    FSDataInputStream fileIn = fs.open(path);
    LineReader in = new LineReader(fileIn, job.getConfiguration());
    int lineLen = 0;
    while(true) {
      Text lineText = new Text();
      lineLen = in.readLine(lineText);
      if(lineLen <= 0) {
      break;
      }
      Matcher m = LINE_PATTERN.matcher(lineText.toString());
      if((m != null) && m.matches()) {
        TableName tableName = TableName.valueOf(m.group(1));
        int startRow = Integer.parseInt(m.group(2));
        int rows = Integer.parseInt(m.group(3));
        int totalRows = Integer.parseInt(m.group(4));
        int clients = Integer.parseInt(m.group(5));
        boolean flushCommits = Boolean.parseBoolean(m.group(6));
        boolean writeToWAL = Boolean.parseBoolean(m.group(7));
        boolean useTags = Boolean.parseBoolean(m.group(8));
        int noOfTags = Integer.parseInt(m.group(9));

        LOG.debug("tableName=" + tableName +
                  " split["+ splitList.size() + "] " +
                  " startRow=" + startRow +
                  " rows=" + rows +
                  " totalRows=" + totalRows +
                  " clients=" + clients +
                  " flushCommits=" + flushCommits +
                  " writeToWAL=" + writeToWAL +
                  " useTags=" + useTags +
                  " noOfTags=" + noOfTags);

        PeInputSplit newSplit =
          new PeInputSplit(tableName, startRow, rows, totalRows, clients,
              flushCommits, writeToWAL, useTags, noOfTags);
        splitList.add(newSplit);
      }
    }
    in.close();
  }

  LOG.info("Total # of splits: " + splitList.size());
  return splitList;
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
/**
 * 循环调用loadChr方法，分别映射染色体文件，获得染色体长度
 * 
 * @param chrList
 *            包含所有染色体文件路径的list文件
 * @throws IOException
 */
public void loadBasicInformation(String chrList) throws IOException {
	Configuration conf = new Configuration();
	Path refPath = new Path(chrList);
	FileSystem fs = refPath.getFileSystem(conf);
	FSDataInputStream refin = fs.open(refPath);
	LineReader in = new LineReader(refin);
	Text line = new Text();

	String chrFile = "";
	String[] chrs = new String[3];
	while ((in.readLine(line)) != 0) {
		chrFile = line.toString();
		chrs = chrFile.split("\t");

		// insert chr
		if (!addChromosome(chrs[0])) {
			in.close();
			throw new RuntimeException("map Chromosome " + chrs[1] + " Failed.");
		}
		if (chromosomeInfoMap.containsKey(chrs[0])) {
			// map chr and get length
			chromosomeInfoMap.get(chrs[0]).setLength(Integer.parseInt(chrs[2]));
			chromosomeInfoMap.get(chrs[0]).setChromosomeName(chrs[0]);
		}
	}
	in.close();
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
public FastqMultipleSample(String list, boolean keepFile) throws IOException {
	Configuration conf = new Configuration();
	Path file = new Path(list);
	FileSystem fs = file.getFileSystem(conf);
	FSDataInputStream fsdata = fs.open(file);
	LineReader reader = new LineReader(fsdata);
	
	Text text = new Text();
	while(reader.readLine(text) != 0) {
		String line = text.toString();
		if(line.length() == 0) {
			continue;
		}
		
		FastqSample slist = new FastqSample();
		
		if(slist.setSampleList(line, keepFile)) {
			if(slist.getIndex() != null){
				sequenceType = true;
				if(slist.getFastq1() != null)
						sampleList.put(slist.getIndex(), slist);
				if(slist.getFastq2() != null)
						sampleList.put(slist.getIndex(), slist);
			}
			else
				sampleList.put(String.valueOf(slist.getId()), slist);
			sampleNumber++;
		}
	}
	reader.close();
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
private void readFile(Path p) {
	StringBuilder sb=null;
	if(!headerHasWrite) {
		sb=new StringBuilder();
	}
	try {
		FSDataInputStream table=fs.open(p);
		LineReader lineReader = new LineReader(table, conf);
		Text line = new Text();
		String tempString=null;
		while(lineReader.readLine(line) > 0 && line.getLength() != 0) {
			tempString=line.toString();
			if(tempString.startsWith(VCFHeaderStartTag)) {
				if(headerHasWrite) {
					continue;
				}
				sb.append(tempString.trim());
				sb.append("\n");
				if(tempString.startsWith(VCFHeaderEndLineTag)) {
					writeHeader(sb.toString().trim());
					headerHasWrite=true;
				}
			} else if (tempString.startsWith(SampleTag)) {
				String sampleName=tempString.split(":")[1];
				currentOutput=sample.get(sampleName);
			} else {
				write("\n");
				if(tempString.startsWith("chr1\t179462149")) {
					System.out.println("debug:"+tempString);
				}
				write(tempString);
			}
		}
		lineReader.close();
		table.close();
	} catch (IOException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
public static void getOutput(BamQualityControlOptions options, Configuration conf, Path oPath) throws IOException {
	ReportBuilder reportBuilder = new ReportBuilder();
	ResultReport reportType;
	ReferenceShare genome = new ReferenceShare();
	genome.loadChromosomeList(options.getReferenceSequencePath());
	Map <String, ResultReport> reports = new ConcurrentHashMap<>();

	FileSystem fs = oPath.getFileSystem(conf);
	FileStatus filelist[] = fs.listStatus(oPath);
	for(int i = 0; i < filelist.length; i++) {
		if(!filelist[i].isDirectory() && !filelist[i].getPath().toString().startsWith("_")) {
			FSDataInputStream reader = fs.open(filelist[i].getPath());
			LineReader lineReader = new LineReader(reader, conf);
			Text line = new Text();
			while(lineReader.readLine(line) > 0) {
				String lineString = line.toString();
				if(line.getLength() == 0)
					continue;

				if(lineString.contains("sample:")) {
					String sample = line.toString().split(":")[1];
					if(!reports.containsKey(sample)) {
						if ((options.getRegion() != null) || (options.getBedfile() != null))
							reportType = new RegionResultReport(options, conf);
						else
							reportType = new WholeGenomeResultReport(options);
						reports.put(sample, reportType);
						reportBuilder.setReportChoice(reportType);
						reportBuilder.initReports(sample);
					} else {
						reportType = reports.get(sample);
						reportBuilder.setReportChoice(reportType);
					}
				}
				reportBuilder.parseReport(lineReader, line, genome);					

			}
			lineReader.close();
			reader.close();
		}	
	}
	
	for(String sampleName:reports.keySet()) {
		System.err.println("sample:" + sampleName);
		ResultReport report = reports.get(sampleName);
		report.write(fs, sampleName);
	}	
	
	fs.close();
}