Java LineReader.readLine方法代码示例

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
public void testUTF8() throws Exception {
  LineReader in = null;

  try {
    in = makeStream("abcd\u20acbdcd\u20ac");
    Text line = new Text();
    in.readLine(line);
    assertEquals("readLine changed utf8 characters",
                 "abcd\u20acbdcd\u20ac", line.toString());
    in = makeStream("abc\u200axyz");
    in.readLine(line);
    assertEquals("split on fake newline", "abc\u200axyz", line.toString());
  } finally {
    if (in != null) {
      in.close();
    }
  }
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
public void testNewLines() throws Exception {
  LineReader in = null;
  try {
    in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
    Text out = new Text();
    in.readLine(out);
    assertEquals("line1 length", 1, out.getLength());
    in.readLine(out);
    assertEquals("line2 length", 2, out.getLength());
    in.readLine(out);
    assertEquals("line3 length", 0, out.getLength());
    in.readLine(out);
    assertEquals("line4 length", 3, out.getLength());
    in.readLine(out);
    assertEquals("line5 length", 4, out.getLength());
    in.readLine(out);
    assertEquals("line5 length", 5, out.getLength());
    assertEquals("end of file", 0, in.readLine(out));
  } finally {
    if (in != null) {
      in.close();
    }
  }
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
/**
 * Test readLine for correct interpretation of maxLineLength
 * (returned string should be clipped at maxLineLength, and the
 * remaining bytes on the same line should be thrown out).
 * Also check that returned value matches the string length.
 * Varies buffer size to stress test.
 *
 * @throws Exception
 */
@Test (timeout=5000)
public void testMaxLineLength() throws Exception {
  final String STR = "a\nbb\n\nccc\rdddd\r\neeeee";
  final int STRLENBYTES = STR.getBytes().length;
  Text out = new Text();
  for (int bufsz = 1; bufsz < STRLENBYTES+1; ++bufsz) {
    LineReader in = makeStream(STR, bufsz);
    int c = 0;
    c += in.readLine(out, 1);
    assertEquals("line1 length, bufsz: "+bufsz, 1, out.getLength());
    c += in.readLine(out, 1);
    assertEquals("line2 length, bufsz: "+bufsz, 1, out.getLength());
    c += in.readLine(out, 1);
    assertEquals("line3 length, bufsz: "+bufsz, 0, out.getLength());
    c += in.readLine(out, 3);
    assertEquals("line4 length, bufsz: "+bufsz, 3, out.getLength());
    c += in.readLine(out, 10);
    assertEquals("line5 length, bufsz: "+bufsz, 4, out.getLength());
    c += in.readLine(out, 8);
    assertEquals("line5 length, bufsz: "+bufsz, 5, out.getLength());
    assertEquals("end of file, bufsz: " +bufsz, 0, in.readLine(out));
    assertEquals("total bytes, bufsz: "+bufsz, c, STRLENBYTES);
  }
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
@Test
public void testNewLines() throws Exception {
  LineReader in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
  Text out = new Text();
  in.readLine(out);
  assertEquals("line1 length", 1, out.getLength());
  in.readLine(out);
  assertEquals("line2 length", 2, out.getLength());
  in.readLine(out);
  assertEquals("line3 length", 0, out.getLength());
  in.readLine(out);
  assertEquals("line4 length", 3, out.getLength());
  in.readLine(out);
  assertEquals("line5 length", 4, out.getLength());
  in.readLine(out);
  assertEquals("line5 length", 5, out.getLength());
  assertEquals("end of file", 0, in.readLine(out));
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
public CombineFileLineRecordReader(CombineFileSplit split,
    TaskAttemptContext context, Integer index) throws IOException {
  
  this.path = split.getPath(index);
  fs = this.path.getFileSystem(context.getConfiguration());
  this.startOffset = split.getOffset(index);
  this.end = startOffset + split.getLength(index);
  boolean skipFirstLine = false;
  
  //open the file
  fileIn = fs.open(path);
  if (startOffset != 0) {
    skipFirstLine = true;
    --startOffset;
    fileIn.seek(startOffset);
  }
  reader = new LineReader(fileIn);
  if (skipFirstLine) {  // skip first line and re-establish "startOffset".
    startOffset += reader.readLine(new Text(), 0,
                (int)Math.min((long)Integer.MAX_VALUE, end - startOffset));
  }
  this.pos = startOffset;
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
/**
 * Returns a split for each store files directory using the block location
 * of each file as locality reference.
 */
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
  List<InputSplit> splits = new ArrayList<InputSplit>();
  List<FileStatus> files = listStatus(job);

  Text key = new Text();
  for (FileStatus file: files) {
    Path path = file.getPath();
    FileSystem fs = path.getFileSystem(job.getConfiguration());
    LineReader reader = new LineReader(fs.open(path));
    long pos = 0;
    int n;
    try {
      while ((n = reader.readLine(key)) > 0) {
        String[] hosts = getStoreDirHosts(fs, path);
        splits.add(new FileSplit(path, pos, n, hosts));
        pos += n;
      }
    } finally {
      reader.close();
    }
  }

  return splits;
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
public static boolean distributeCache(String chrList, Job job, String cacheName)
		throws IOException, URISyntaxException {
	job.addCacheFile(new URI(chrList + "#" + cacheName));

	Configuration conf = job.getConfiguration();
	Path refPath = new Path(chrList);
	FileSystem fs = refPath.getFileSystem(conf);
	FSDataInputStream refin = fs.open(refPath);
	LineReader in = new LineReader(refin);
	Text line = new Text();

	String chrFile = "";
	String[] chrs = new String[3];
	while ((in.readLine(line)) != 0) {
		chrFile = line.toString();
		chrs = chrFile.split("\t");
		File fileTest = new File(chrs[1]);
		if (fileTest.isFile()) {
			chrs[1] = "file://" + chrs[1];
		}
		job.addCacheFile(new URI(chrs[1] + "#" + chrs[0]));
	}
	in.close();
	refin.close();
	return true;
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
/**
 * Creates a mappings file from the contents of a flat text file containing docid to docno
 * mappings. This method is used by {@link WikipediaDocnoMappingBuilder} internally.
 *
 * @param inputFile flat text file containing docid to docno mappings
 * @param outputFile output mappings file
 * @throws IOException
 */
static public void writeDocnoMappingData(FileSystem fs, String inputFile, int n, String outputFile) throws IOException {
	LOG.info("Writing " + n + " docids to " + outputFile);
	LineReader reader = new LineReader(fs.open(new Path(inputFile)));

	int cnt = 0;
	Text line = new Text();

	FSDataOutputStream out = fs.create(new Path(outputFile), true);
	out.writeInt(n);
	for (int i = 0; i < n; i++) {
		reader.readLine(line);
		String[] arr = line.toString().split("\\t");
		out.writeInt(Integer.parseInt(arr[0]));
		cnt++;
		if (cnt % 100000 == 0) {
			LOG.info(cnt + " articles");
		}
	}
	out.close();
	reader.close();
	LOG.info("Done!");
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
@Test
public void testUTF8() throws Exception {
  LineReader in = null;

  try {
    in = makeStream("abcd\u20acbdcd\u20ac");
    Text line = new Text();
    in.readLine(line);
    assertEquals("readLine changed utf8 characters",
                 "abcd\u20acbdcd\u20ac", line.toString());
    in = makeStream("abc\u200axyz");
    in.readLine(line);
    assertEquals("split on fake newline", "abc\u200axyz", line.toString());
  } finally {
    if (in != null) {
      in.close();
    }
  }
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
public void testNewLines() throws Exception {
  LineReader in = makeStream("a\nbb\n\nccc\rdddd\r\neeeee");
  Text out = new Text();
  in.readLine(out);
  assertEquals("line1 length", 1, out.getLength());
  in.readLine(out);
  assertEquals("line2 length", 2, out.getLength());
  in.readLine(out);
  assertEquals("line3 length", 0, out.getLength());
  in.readLine(out);
  assertEquals("line4 length", 3, out.getLength());
  in.readLine(out);
  assertEquals("line5 length", 4, out.getLength());
  in.readLine(out);
  assertEquals("line5 length", 5, out.getLength());
  assertEquals("end of file", 0, in.readLine(out));
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
/**
 * Test readLine for correct interpretation of maxLineLength
 * (returned string should be clipped at maxLineLength, and the
 * remaining bytes on the same line should be thrown out).
 * Also check that returned value matches the string length.
 * Varies buffer size to stress test.
 *
 * @throws Exception
 */
public void testMaxLineLength() throws Exception {
  final String STR = "a\nbb\n\nccc\rdddd\r\neeeee";
  final int STRLENBYTES = STR.getBytes().length;
  Text out = new Text();
  for (int bufsz = 1; bufsz < STRLENBYTES+1; ++bufsz) {
    LineReader in = makeStream(STR, bufsz);
    int c = 0;
    c += in.readLine(out, 1);
    assertEquals("line1 length, bufsz: "+bufsz, 1, out.getLength());
    c += in.readLine(out, 1);
    assertEquals("line2 length, bufsz: "+bufsz, 1, out.getLength());
    c += in.readLine(out, 1);
    assertEquals("line3 length, bufsz: "+bufsz, 0, out.getLength());
    c += in.readLine(out, 3);
    assertEquals("line4 length, bufsz: "+bufsz, 3, out.getLength());
    c += in.readLine(out, 10);
    assertEquals("line5 length, bufsz: "+bufsz, 4, out.getLength());
    c += in.readLine(out, 8);
    assertEquals("line5 length, bufsz: "+bufsz, 5, out.getLength());
    assertEquals("end of file, bufsz: " +bufsz, 0, in.readLine(out));
    assertEquals("total bytes, bufsz: "+bufsz, c, STRLENBYTES);
  }
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
/**
 * Parse the command line arguments into lines and display the result.
 * @param args
 * @throws Exception
 */
public static void main(String[] args) throws Exception {
  for(String arg: args) {
    System.out.println("Working on " + arg);
    LineReader reader = makeStream(unquote(arg));
    Text line = new Text();
    int size = reader.readLine(line);
    while (size > 0) {
      System.out.println("Got: " + line.toString());
      size = reader.readLine(line);
    }
    reader.close();
  }
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
@Test (timeout=5000)
public void testUTF8() throws Exception {
  LineReader in = makeStream("abcd\u20acbdcd\u20ac");
  Text line = new Text();
  in.readLine(line);
  assertEquals("readLine changed utf8 characters", 
               "abcd\u20acbdcd\u20ac", line.toString());
  in = makeStream("abc\u200axyz");
  in.readLine(line);
  assertEquals("split on fake newline", "abc\u200axyz", line.toString());
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
/**
 * Test readLine for various kinds of line termination sequneces.
 * Varies buffer size to stress test.  Also check that returned
 * value matches the string length.
 *
 * @throws Exception
 */
@Test (timeout=5000)
public void testNewLines() throws Exception {
  final String STR = "a\nbb\n\nccc\rdddd\r\r\r\n\r\neeeee";
  final int STRLENBYTES = STR.getBytes().length;
  Text out = new Text();
  for (int bufsz = 1; bufsz < STRLENBYTES+1; ++bufsz) {
    LineReader in = makeStream(STR, bufsz);
    int c = 0;
    c += in.readLine(out); //"a"\n
    assertEquals("line1 length, bufsz:"+bufsz, 1, out.getLength());
    c += in.readLine(out); //"bb"\n
    assertEquals("line2 length, bufsz:"+bufsz, 2, out.getLength());
    c += in.readLine(out); //""\n
    assertEquals("line3 length, bufsz:"+bufsz, 0, out.getLength());
    c += in.readLine(out); //"ccc"\r
    assertEquals("line4 length, bufsz:"+bufsz, 3, out.getLength());
    c += in.readLine(out); //dddd\r
    assertEquals("line5 length, bufsz:"+bufsz, 4, out.getLength());
    c += in.readLine(out); //""\r
    assertEquals("line6 length, bufsz:"+bufsz, 0, out.getLength());
    c += in.readLine(out); //""\r\n
    assertEquals("line7 length, bufsz:"+bufsz, 0, out.getLength());
    c += in.readLine(out); //""\r\n
    assertEquals("line8 length, bufsz:"+bufsz, 0, out.getLength());
    c += in.readLine(out); //"eeeee"EOF
    assertEquals("line9 length, bufsz:"+bufsz, 5, out.getLength());
    assertEquals("end of file, bufsz: "+bufsz, 0, in.readLine(out));
    assertEquals("total bytes, bufsz: "+bufsz, c, STRLENBYTES);
  }
}
 

import org.apache.hadoop.util.LineReader; //导入方法依赖的package包/类
@Test
public void testUTF8() throws Exception {
  LineReader in = makeStream("abcd\u20acbdcd\u20ac");
  Text line = new Text();
  in.readLine(line);
  assertEquals("readLine changed utf8 characters", 
               "abcd\u20acbdcd\u20ac", line.toString());
  in = makeStream("abc\u200axyz");
  in.readLine(line);
  assertEquals("split on fake newline", "abc\u200axyz", line.toString());
}