Java DumpContentType类代码示例

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
/**
 * Constructor.
 * 
 * @param dateStamp
 *            dump date in format YYYYMMDD
 * @param projectName
 *            project name string
 * @param dumpfileDirectoryManager
 *            the directory manager for the directory where dumps should be
 *            downloaded to
 * @param dumpContentType
 *            the type of dump this represents
 */
public WmfLocalDumpFile(String dateStamp, String projectName,
		DirectoryManager dumpfileDirectoryManager,
		DumpContentType dumpContentType) {
	super(dateStamp, projectName);

	String subdirectoryName = WmfDumpFile.getDumpFileDirectoryName(
			dumpContentType, dateStamp);
	if (!dumpfileDirectoryManager.hasSubdirectory(subdirectoryName)) {
		throw new IllegalArgumentException(
				"There is no local dump file directory at the specified location.");
	}
	try {
		this.localDumpfileDirectoryManager = dumpfileDirectoryManager
				.getSubdirectoryManager(subdirectoryName);
	} catch (IOException e) {
		throw new IllegalArgumentException(
				"Could not change to dump file directory", e);
	}

	this.dumpContentType = dumpContentType;
}
 

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
/**
 * Finds out which dump files of the given type are available for download.
 * The result is a list of objects that describe the available dump files,
 * in descending order by their date. Not all of the dumps included might be
 * actually available.
 *
 * @return list of objects that provide information on available full dumps
 */
List<MwDumpFile> findDumpsOnline(DumpContentType dumpContentType) {
	List<String> dumpFileDates = findDumpDatesOnline(dumpContentType);

	List<MwDumpFile> result = new ArrayList<MwDumpFile>();

	for (String dateStamp : dumpFileDates) {
		if (dumpContentType == DumpContentType.DAILY) {
			result.add(new WmfOnlineDailyDumpFile(dateStamp,
					this.projectName, this.webResourceFetcher,
					this.dumpfileDirectoryManager));
		} else if (dumpContentType == DumpContentType.JSON) {
			result.add(new JsonOnlineDumpFile(dateStamp, this.projectName,
					this.webResourceFetcher, this.dumpfileDirectoryManager));
		} else {
			result.add(new WmfOnlineStandardDumpFile(dateStamp,
					this.projectName, this.webResourceFetcher,
					this.dumpfileDirectoryManager, dumpContentType));
		}
	}

	logger.info("Found " + result.size() + " online dumps of type "
			+ dumpContentType + ": " + result);

	return result;
}
 

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
/**
 * Returns the absolute directory on the Web site where dumpfiles of the
 * given type can be found.
 *
 * @param dumpContentType
 *            the type of dump
 * @return relative web directory for the current dumpfiles
 * @throws IllegalArgumentException
 *             if the given dump file type is not known
 */
public static String getDumpFileWebDirectory(
		DumpContentType dumpContentType, String projectName) {
	if (dumpContentType == DumpContentType.JSON) {
		if ("wikidatawiki".equals(projectName)) {
			return WmfDumpFile.DUMP_SITE_BASE_URL
					+ WmfDumpFile.WEB_DIRECTORY.get(dumpContentType)
					+ "wikidata" + "/";
		} else {
			throw new RuntimeException(
					"Wikimedia Foundation uses non-systematic directory names for this type of dump file."
							+ " I don't know where to find dumps of project "
							+ projectName);
		}
	} else if (WmfDumpFile.WEB_DIRECTORY.containsKey(dumpContentType)) {
		return WmfDumpFile.DUMP_SITE_BASE_URL
				+ WmfDumpFile.WEB_DIRECTORY.get(dumpContentType)
				+ projectName + "/";
	} else {
		throw new IllegalArgumentException("Unsupported dump type "
				+ dumpContentType);
	}
}
 

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
@Test
public void validDumpProperties() throws IOException {
	String dateStamp = "20140220";
	wrf.setWebResourceContents(
			"https://dumps.wikimedia.org/other/incr/wikidatawiki/"
					+ dateStamp + "/status.txt", "done");
	wrf.setWebResourceContents(
			"https://dumps.wikimedia.org/other/incr/wikidatawiki/"
					+ dateStamp + "/wikidatawiki-" + dateStamp
					+ "-pages-meta-hist-incr.xml.bz2", "Line1",
			CompressionType.BZ2);
	WmfOnlineDailyDumpFile dump = new WmfOnlineDailyDumpFile(dateStamp,
			"wikidatawiki", wrf, dm);

	BufferedReader br = dump.getDumpFileReader();

	assertEquals(br.readLine(), "Line1");
	assertEquals(br.readLine(), null);
	assertTrue(dump.isAvailable());
	assertTrue(dump.isAvailable()); // second time should use cached entry
	assertEquals(dateStamp, dump.getDateStamp());
	assertEquals("wikidatawiki", dump.getProjectName());
	assertEquals("wikidatawiki-daily-" + dateStamp, dump.toString());
	assertEquals(DumpContentType.DAILY, dump.getDumpContentType());
}
 

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
@Test
public void validCurrentDumpPropertiesOldFormat() throws IOException {
	wrf.setWebResourceContentsFromResource(
			"https://dumps.wikimedia.org/wikidatawiki/20140210/",
			"/wikidatawiki-20140210-index.html", this.getClass());
	wrf.setWebResourceContents(
			"https://dumps.wikimedia.org/wikidatawiki/20140210/wikidatawiki-20140210-pages-meta-current.xml.bz2",
			"Line1", CompressionType.BZ2);
	wrf.setWebResourceContentsFromResource(
			"https://dumps.wikimedia.org/wikidatawiki/20140210/wikidatawiki-20140210-md5sums.txt",
			"/wikidatawiki-20140210-md5sums.txt", this.getClass());
	MwDumpFile dump = new WmfOnlineStandardDumpFile("20140210",
			"wikidatawiki", wrf, dm, DumpContentType.CURRENT);

	BufferedReader br = dump.getDumpFileReader();

	assertEquals(br.readLine(), "Line1");
	assertEquals(br.readLine(), null);
	assertTrue(dump.isAvailable());
	assertEquals("20140210", dump.getDateStamp());
	assertEquals(DumpContentType.CURRENT, dump.getDumpContentType());
}
 

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
@Test
public void validCurrentDumpPropertiesNewFormat() throws IOException {
	wrf.setWebResourceContentsFromResource(
			"https://dumps.wikimedia.org/wikidatawiki/20140210/",
			"/wikidatawiki-20140508-index.html", this.getClass());
	wrf.setWebResourceContents(
			"https://dumps.wikimedia.org/wikidatawiki/20140210/wikidatawiki-20140210-pages-meta-current.xml.bz2",
			"Line1", CompressionType.BZ2);
	wrf.setWebResourceContentsFromResource(
			"https://dumps.wikimedia.org/wikidatawiki/20140210/wikidatawiki-20140210-md5sums.txt",
			"/wikidatawiki-20140210-md5sums.txt", this.getClass());
	MwDumpFile dump = new WmfOnlineStandardDumpFile("20140210",
			"wikidatawiki", wrf, dm, DumpContentType.CURRENT);

	BufferedReader br = dump.getDumpFileReader();

	assertEquals(br.readLine(), "Line1");
	assertEquals(br.readLine(), null);
	assertTrue(dump.isAvailable());
	assertEquals("20140210", dump.getDateStamp());
	assertEquals(DumpContentType.CURRENT, dump.getDumpContentType());
}
 

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
@Test
public void inaccessibleCurrentDumpProperties() throws IOException {
	wrf.setWebResourceContentsFromResource(
			"http://dumps.wikimedia.org/wikidatawiki/20140210/",
			"/wikidatawiki-20140210-index.html", this.getClass());
	wrf.setWebResourceContents(
			"http://dumps.wikimedia.org/wikidatawiki/20140210/wikidatawiki-20140210-pages-meta-current.xml.bz2",
			"Line1");
	wrf.setWebResourceContentsFromResource(
			"http://dumps.wikimedia.org/wikidatawiki/20140210/wikidatawiki-20140210-md5sums.txt",
			"/wikidatawiki-20140210-md5sums.txt", this.getClass());
	wrf.setReturnFailingReaders(true);

	MwDumpFile dump = new WmfOnlineStandardDumpFile("20140210",
			"wikidatawiki", wrf, dm, DumpContentType.CURRENT);

	assertFalse(dump.isAvailable());
}
 

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
/**
 * Helper method to create mocked local dump files.
 *
 * @param dateStamp
 * @param dumpContentType
 * @param isDone
 * @throws IOException
 */
void setLocalDump(String dateStamp, DumpContentType dumpContentType,
		boolean isDone) throws IOException {

	Path dumpFilePath = this.dmPath.resolve("dumpfiles").resolve(
			"wikidatawiki");
	Path thisDumpPath = dumpFilePath.resolve(dumpContentType.toString()
			.toLowerCase() + "-" + dateStamp);
	dm.setDirectory(thisDumpPath);
	if (isDone) {
		Path filePath = thisDumpPath.resolve(WmfDumpFile.getDumpFileName(
				dumpContentType, "wikidatawiki", dateStamp));
		dm.setFileContents(filePath,
				"Contents of " + dumpContentType.toString().toLowerCase() + " " + dateStamp,
				WmfDumpFile.getDumpFileCompressionType(filePath.toString()));
	}
}
 

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
@Test
public void getAllDailyDumpsOffline() throws IOException {
	setLocalDump("20140220", DumpContentType.DAILY, true);
	setLocalDump("20140205", DumpContentType.DAILY, true);

	WmfDumpFileManager dumpFileManager = new WmfDumpFileManager(
			"wikidatawiki", dm, null);

	List<? extends MwDumpFile> dumpFiles = dumpFileManager
			.findAllDumps(DumpContentType.DAILY);

	String[] dumpDates = { "20140220", "20140205" };

	assertEquals(dumpFiles.size(), dumpDates.length);
	for (int i = 0; i < dumpFiles.size(); i++) {
		assertEquals(dumpFiles.get(i).getDumpContentType(),
				DumpContentType.DAILY);
		assertEquals(dumpFiles.get(i).getDateStamp(), dumpDates[i]);
		assertTrue(dumpFiles.get(i) instanceof WmfLocalDumpFile);
	}
}
 

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
@Test
public void getAllCurrentDumpsOffline() throws IOException {
	setLocalDump("20140220", DumpContentType.CURRENT, true);
	setLocalDump("20140205", DumpContentType.CURRENT, true);

	WmfDumpFileManager dumpFileManager = new WmfDumpFileManager(
			"wikidatawiki", dm, null);

	List<? extends MwDumpFile> dumpFiles = dumpFileManager
			.findAllDumps(DumpContentType.CURRENT);

	String[] dumpDates = { "20140220", "20140205" };

	assertEquals(dumpFiles.size(), dumpDates.length);
	for (int i = 0; i < dumpFiles.size(); i++) {
		assertEquals(dumpFiles.get(i).getDumpContentType(),
				DumpContentType.CURRENT);
		assertEquals(dumpFiles.get(i).getDateStamp(), dumpDates[i]);
		assertTrue(dumpFiles.get(i) instanceof WmfLocalDumpFile);
	}
}
 

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
@Test
public void getAllFullDumpsOffline() throws IOException {
	setLocalDump("20140220", DumpContentType.FULL, true);
	setLocalDump("20140205", DumpContentType.FULL, true);

	WmfDumpFileManager dumpFileManager = new WmfDumpFileManager(
			"wikidatawiki", dm, null);

	List<? extends MwDumpFile> dumpFiles = dumpFileManager
			.findAllDumps(DumpContentType.FULL);

	String[] dumpDates = { "20140220", "20140205" };

	assertEquals(dumpFiles.size(), dumpDates.length);
	for (int i = 0; i < dumpFiles.size(); i++) {
		assertEquals(dumpFiles.get(i).getDumpContentType(),
				DumpContentType.FULL);
		assertEquals(dumpFiles.get(i).getDateStamp(), dumpDates[i]);
		assertTrue(dumpFiles.get(i) instanceof WmfLocalDumpFile);
	}
}
 

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
@Test
public void getAllRelevantDumpsMainDumpMissing() throws IOException {
	setLocalDump("20140220", DumpContentType.DAILY, true);
	setLocalDump("20140210", DumpContentType.CURRENT, true);

	WmfDumpFileManager dumpFileManager = new WmfDumpFileManager(
			"wikidatawiki", dm, wrf);

	List<? extends MwDumpFile> dumpFiles = dumpFileManager
			.findAllRelevantRevisionDumps(false);

	assertEquals(dumpFiles.size(), 1);
	assertEquals(dumpFiles.get(0).getDumpContentType(),
			DumpContentType.DAILY);
	assertEquals(dumpFiles.get(0).getDateStamp(), "20140220");
	assertTrue(dumpFiles.get(0) instanceof WmfLocalDumpFile);
}
 

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
@Test
public void processAllRelevantDumps() throws IOException {

	setLocalDump("20140221", DumpContentType.DAILY, true);
	setLocalDump("20140220", DumpContentType.DAILY, true);
	setLocalDump("20140219", DumpContentType.CURRENT, true);

	WmfDumpFileManager dumpFileManager = new WmfDumpFileManager(
			"wikidatawiki", dm, null);

	TestDumpfileProcessor dfp = new TestDumpfileProcessor();

	for (MwDumpFile dumpFile : dumpFileManager
			.findAllRelevantRevisionDumps(true)) {
		dfp.processDumpFileContents(dumpFile.getDumpFileStream(), dumpFile);
	}

	assertEquals(
			dfp.result,
			"Contents of daily 20140221\nContents of daily 20140220\nContents of current 20140219\n");
}
 

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
private static Stream<MwDumpFile> getPossibleDumpsToProcess(WmfDumpFileManager dumpFileManager) {
    MwDumpFile jsonDump = dumpFileManager.findMostRecentDump(DumpContentType.JSON);
    return Stream.concat(
            Stream.of(jsonDump),
            dumpFileManager.findAllRelevantRevisionDumps(true).stream()
                    .filter(dump -> Integer.parseInt(dump.getDateStamp()) > Integer.parseInt(jsonDump.getDateStamp()))
    ).sorted(new MwDumpFile.DateComparator());
}
 

import org.wikidata.wdtk.dumpfiles.DumpContentType; //导入依赖的package包/类
@Override
public InputStream getDumpFileStream() throws IOException {
	prepareDumpFile();

	String fileName = WmfDumpFile.getDumpFileName(DumpContentType.DAILY,
			this.projectName, this.dateStamp);
	DirectoryManager dailyDirectoryManager = this.dumpfileDirectoryManager
			.getSubdirectoryManager(WmfDumpFile.getDumpFileDirectoryName(
					DumpContentType.DAILY, this.dateStamp));

	return dailyDirectoryManager.getInputStreamForFile(fileName,
			WmfDumpFile.getDumpFileCompressionType(fileName));
}