Java PipedRDFIterator.hasNext方法代码示例

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
@Test
	public void testExternalDataProviders() {
		metric = new EstimatedLinkExternalDataProviders();
		metric.setDatasetURI("http://www.myexperiment.org");

		 PipedRDFIterator<?> iter =loader.streamParser("/Users/jeremy/Desktop/www.myexperiment.org.nt.gz");
		// Load quads...
//		List<Quad> streamingQuads = loader.getStreamingQuads();
//		int counter = 0;
		
//		for(Quad quad : streamingQuads){
//			metric.compute(quad);
//			counter++;
//		}
//		System.out.println(counter);
		
		while(iter.hasNext()){
			Object nxt = iter.next();
			Object2Quad quad = new Object2Quad(nxt);
			metric.compute(quad.getStatement());
		}
//		
		assertEquals(0.052,metric.metricValue(),0.001);
	}
 

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
protected void runExtraction(PipedRDFIterator<Triple> iter, Extractor extractors[]) {
    Triple triple;
    while (iter.hasNext()) {
        triple = iter.next();
        for (int i = 0; i < extractors.length; ++i) {
            extractors[i].handleTriple(triple);
        }
    }
}
 

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
/**
 * (Variation 3)
 * 
 * @param iter Iterator for RDF triple
 * @param extractors Extractor
 */
protected void runExtraction(PipedRDFIterator<Triple> iter, Extractor extractors[]) {
    Triple triple;
    
    // As long as we have a triple left, do..
    // (1) take a triple
    // (2) call "handleTriple()" on it
    while (iter.hasNext()) {
        triple = iter.next();
        for (int i = 0; i < extractors.length; ++i) {
            extractors[i].handleTriple(triple);
        }
    }
}
 

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
@Override
public void load( 
		final SolrQueryRequest request, 
		final SolrQueryResponse response,
		final ContentStream stream, 
		final UpdateRequestProcessor processor) throws Exception {
	
	final PipedRDFIterator<Quad> iterator = new PipedRDFIterator<Quad>();
	final StreamRDF inputStream = new PipedQuadsStream(iterator);
	
	executor.submit(new Runnable() {
		@Override
		public void run() {
			try {
				RDFDataMgr.parse(
						inputStream, 
						stream.getStream(), 
						RDFLanguages.contentTypeToLang(stream.getContentType()));
			} catch (final IOException exception) {
				throw new SolrException(ErrorCode.SERVER_ERROR, exception);
			}					
		}
	});
	
	final DatasetGraph dataset = new LocalDatasetGraph(request, response);
	while (iterator.hasNext()) {
		dataset.add(iterator.next());
	}									
}
 

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
public static void main(String[] args) {
		final String fileName = "data/Earthquakes-Spain-2013.ttl";		

		Sink<Triple> output = new SinkTripleOutput(System.out, null, SyntaxLabels.createNodeToLabel());
		PipedRDFIterator<Triple> iterator = new PipedRDFIterator<Triple>();
        final PipedRDFStream<Triple> inputStream = new PipedTriplesStream(iterator);

		//RDFDataMgr.parse(inputStream, fileName);
		
		// PipedRDFStream and PipedRDFIterator need to be on different threads
        ExecutorService executor = Executors.newSingleThreadExecutor();

        // Create a runnable for our parser thread
        Runnable parser = new Runnable() {

            @Override
            public void run() {
                // Call the parsing process.
                RDFDataMgr.parse(inputStream, fileName);
            }
        };

        // Start the parser on another thread
        executor.submit(parser);
		
//		
//		// Iterate over the data
		while (iterator.hasNext()) {
			Triple next = iterator.next();
			System.out.println("STRING: "+ next.toString());
			//output.send(next);
		}
		
		
	}
 

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
public static void main(String[] args) {

      final String filename = "category_labels_en.ttl";

      // Create a PipedRDFStream to accept input and a PipedRDFIterator to
      // consume it
      // You can optionally supply a buffer size here for the
      // PipedRDFIterator, see the documentation for details about recommended
      // buffer sizes
      PipedRDFIterator<Triple> iter = new PipedRDFIterator<Triple>();
      final PipedRDFStream<Triple> inputStream = new PipedTriplesStream(iter);

      // PipedRDFStream and PipedRDFIterator need to be on different threads
      ExecutorService executor = Executors.newSingleThreadExecutor();

      // Create a runnable for our parser thread
      Runnable parser = new Runnable() {

          public void run() {
              // Call the parsing process.
              RDFDataMgr.parse(inputStream, filename);
          }
      };

      // Start the parser on another thread
      executor.submit(parser);

      // We will consume the input on the main thread here

      // We can now iterate over data as it is parsed, parsing only runs as
      // far ahead of our consumption as the buffer size allows
      while (iter.hasNext()) {
          Triple next = iter.next();
          if (next.getObject().isLiteral()){
          	System.out.println(next.getObject().getLiteralLexicalForm());
          }
      }
  }
 

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
public static void main(String... argv) {
    final String filename = "data.ttl";

    // Create a PipedRDFStream to accept input and a PipedRDFIterator to
    // consume it
    // You can optionally supply a buffer size here for the
    // PipedRDFIterator, see the documentation for details about recommended
    // buffer sizes
    PipedRDFIterator<Triple> iter = new PipedRDFIterator<Triple>();
    final PipedRDFStream<Triple> inputStream = new PipedTriplesStream(iter);

    // PipedRDFStream and PipedRDFIterator need to be on different threads
    ExecutorService executor = Executors.newSingleThreadExecutor();

    // Create a runnable for our parser thread
    Runnable parser = new Runnable() {

        @Override
        public void run() {
            // Call the parsing process.
            RDFDataMgr.parse(inputStream, filename);
        }
    };

    // Start the parser on another thread
    executor.submit(parser);

    // We will consume the input on the main thread here

    // We can now iterate over data as it is parsed, parsing only runs as
    // far ahead of our consumption as the buffer size allows
    while (iter.hasNext()) {
        Triple next = iter.next();
        // Do something with each triple
    }
}
 

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
public static void nonEstimate(String dataset){

		System.out.println("Evaluating Non-Estimate");
		long tMin = Long.MAX_VALUE;
		long tMax = Long.MIN_VALUE;
		long tAvg = 0;

		for (int i = -1; i < 2; i++){
//			List<Quad> streamingQuads = loader.getStreamingQuads();
			PipedRDFIterator<?> iter = loader.streamParser(dataset);

			long tStart = System.currentTimeMillis();
			LinkExternalDataProviders m = new LinkExternalDataProviders();
			m.setDatasetURI(dsToURI.get(dataset));

			while(iter.hasNext()){
				Object nxt = iter.next();
				Object2Quad quad = new Object2Quad(nxt);
				m.compute(quad.getStatement());
			}

			
			if (i < 0){
				m.metricValue();
			} else {
				System.out.println("Count : " + i + " Value : "+ m.metricValue());
			}
			
			long tEnd = System.currentTimeMillis();
			if (i >= 0){
				long difference = tEnd - tStart;
				tAvg += difference;
				tMax = (tMax < difference) ? difference : tMax;
				tMin = (tMin > difference) ? difference : tMin;
			}
		}
		tAvg = tAvg/2;
		System.out.println("Min: "+ (tMin/1000.0) + " Max: "+ (tMax/1000.0) + " Avg: "+ (tAvg/1000.0));
	}
 

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
@Test
	public void testMachineReadableLicense(){
		HumanReadableLicense mrl = new HumanReadableLicense();
		mrl.setDatasetURI("http://lod.taxonconcept.org");

//		List<Quad> quads = l.getStreamingQuads();
		
//		for (Quad q : quads){
//			mrl.compute(q);
//		}
		
		PipedRDFIterator<?> iter = l.streamParser("/Users/jeremy/Desktop/lod.taxonconcept.org.nt.gz");
		while(iter.hasNext()){
			Object nxt = iter.next();
			Object2Quad quad = new Object2Quad(nxt);
			mrl.compute(quad.getStatement());
		}
		
//		assertEquals(0.0, mrl.metricValue(), 0.00001);
		System.out.println(mrl.metricValue());
		
		// Problem report
		ProblemList<?> pl = mrl.getQualityProblems();
		QualityReport qr = new QualityReport();
		String plModelURI = qr.createQualityProblem(mrl.getMetricURI(), pl);
		Model plModel = qr.getProblemReportFromTBD(plModelURI);
		
		try {
			plModel.write(new FileOutputStream(new File("/Users/jeremy/Desktop/preport_test.ttl")), "TURTLE");
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		}
	}
 

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
@Override
    public File fetch(CrawleableUri uri/*, Sink sink*/) {
        int tripleCount = 0;
        String filePath = this.downloadFile(uri, "/tmp/");
        if (filePath == null) {
            return null;
//            return 0;
        }
        LOGGER.debug("{} is saved to {}", uri.toString(), filePath);
        String fileType = this.detectFileType(uri, filePath);
        LOGGER.debug("Detected file type: {}", fileType);

        if (fileType.contains("application/zip")) {
            String unzipPath = filePath + "__extracted";
            File[] files = ZipArchiver.unzip(filePath, unzipPath, "emptypassword");
            File firstFile = files[0];
            filePath = firstFile.getPath().toString();
        }

        Lang hint;
        if (matchesSerialization(uri.toString(), "rdf")) {
            LOGGER.debug("Detected rdf/xml serialization.");
            hint = Lang.RDFXML;
        } else if (matchesSerialization(uri.toString(), "ttl")) {
            LOGGER.debug("Detected ttl serialization.");
            hint = Lang.TURTLE;
        } else if (matchesSerialization(uri.toString(), "nt")) {
            LOGGER.debug("Detected nt serialization.");
            hint = Lang.NT;
        } else if (matchesSerialization(uri.toString(), "n3")) {
            hint = Lang.N3;
            LOGGER.debug("Detected n3 serialization.");
        } else {
            LOGGER.error("Could not detect serialization for {}. Skipping.", uri.toString());
//            return tripleCount;
            return null;
        }
        PipedRDFIterator<Triple> iterator = new PipedRDFIterator<>();
        final PipedRDFStream<Triple> inStream = new PipedTriplesStream(iterator);

        ExecutorService executor = Executors.newSingleThreadExecutor();
        final String finalFilePath = filePath;
        Runnable parser = new Runnable() {
            @Override
            public void run() {
                LOGGER.debug("Parsing {}", finalFilePath);
                RDFDataMgr.parse(inStream, finalFilePath, hint, null);
            }
        };

        executor.submit(parser);

//        LOGGER.debug("Opening sink for {}", filePath);
//        sink.openSinkForUri(uri);
        while (iterator.hasNext()) {
            Triple next = iterator.next();
//            sink.addTriple(uri, next);
            ++tripleCount;
        }
//        sink.closeSinkForUri(uri);
//        return tripleCount;

        return null;
    }
 

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
public void convertToBoundingBox(InputStream in, OutputStream out, String baseUri,boolean mvbb){
	setRdfWriter(new WriterStreamRDFBlocks(out));
	getRdfWriter().base(baseUri);
	getRdfWriter().prefix("inst", baseUri);
	getRdfWriter().prefix("rdf", Namespace.RDF);
	getRdfWriter().prefix("xsd", Namespace.XSD);
	getRdfWriter().prefix("owl", Namespace.OWL);
	getRdfWriter().prefix("geom", GEOM.getURI());
	getRdfWriter().start();

	getRdfWriter().finish();
		PipedRDFIterator<Triple> iter = new PipedRDFIterator<Triple>();
		final PipedRDFStream<Triple> stream = new PipedTriplesStream(iter);
		ExecutorService executor = Executors.newSingleThreadExecutor();
		Runnable parser = new Runnable() {
			public void run() {
				RDFDataMgr.parse(stream, in, RDFLanguages.TTL);
			}
		};
		executor.submit(parser);
		while (iter.hasNext()) {
			Triple t = iter.next();
			Node subject = t.getSubject();
			Node predicate = t.getPredicate();
			Node object = t.getObject();
			if(predicate.equals(GEOM.asBody.asNode())){
				Geometry g=EwktReader.parseGeometry((String)object.getLiteralValue());
				if(!g.isEmpty()){
				AABBVisitor av=new AABBVisitor();					
				g.accept(av);
				AABB aabb=av.getAABB();
				getRdfWriter().triple(new Triple(subject,GEOM.asAABB.asNode(),NodeFactory.createLiteral(EwktWriter.writeGeometry(aabb.toPolyhedralSurface()))));
				if(mvbb){
					MVBBVisitor mv=new MVBBVisitor();
					g.accept(mv);
					Box mvbbb=mv.getMVBB();
					getRdfWriter().triple(new Triple(subject,GEOM.asMVBB.asNode(),NodeFactory.createLiteral(EwktWriter.writeGeometry(mvbbb.toPolyhedralSurface()))));
				}
				}
			}

		}
		executor.shutdown();
		System.out.println("Model parsed!");	
		System.out.println("Finished!");
}
 

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
public void extract(PipedRDFIterator<Triple> iter) {
	while (iter.hasNext()) {
		handleTriple(iter.next());
	}
}
 

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
/**
 * "extract()" method will call "handleTriple()" method 
 * repeatedly.
 */
public void extract(PipedRDFIterator<Triple> iter) {
	while (iter.hasNext()) {
		handleTriple(iter.next());
	}
}
 

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
public void runRiot() {
	final String filename = "article_categories_en.ttl";

	// Create a PipedRDFStream to accept input and a PipedRDFIterator to
	// consume it
	// You can optionally supply a buffer size here for the
	// PipedRDFIterator, see the documentation for details about recommended
	// buffer sizes

	PipedRDFIterator<Triple> iter = new PipedRDFIterator<Triple>();
	
	final PipedRDFStream<Triple> inputStream = new PipedTriplesStream(iter);

	// PipedRDFStream and PipedRDFIterator need to be on different threads
	ExecutorService executor = Executors.newSingleThreadExecutor();

	// Create a runnable for our parser thread
	Runnable parser = new Runnable() {

		public void run() {
			// Call the parsing process.
			RDFDataMgr.parse(inputStream, filename);
		}
	};

	// Start the parser on another thread
	executor.submit(parser);

	// We will consume the input on the main thread here

	// We can now iterate over data as it is parsed, parsing only runs as
	// far ahead of our consumption as the buffer size allows
	while (iter.hasNext()) {
		Triple next = iter.next();
		if (next.getObject().isLiteral()) {
			String literalLoxicalForm = next.getObject().getLiteralLexicalForm();
			literalsListDuplicates.add(literalLoxicalForm);
			literalsListNODuplicates.add(literalLoxicalForm);
		}
	}
}
 

import org.apache.jena.riot.lang.PipedRDFIterator; //导入方法依赖的package包/类
private DGETInfo checkForVoid(String url, String operation, Robots rob) {
	DGETInfo info = new DGETInfo();
	info.setOperation(operation);
	info.setURL(url);
	info.setResponseServer("missing");
	
	boolean isRobotsAllowed = checkRobotsTxt(rob, url);
	info.setAllowedByRobotsTXT(isRobotsAllowed);
	
	HashMap<CharSequence, Object> voidPred = new HashMap<CharSequence, Object>();
	HashMap<CharSequence, Object> spdsPred = new HashMap<CharSequence, Object>();
	info.setSPARQLDESCpreds( spdsPred );
	info.setVoiDpreds( voidPred );
	
	HttpGet request = new HttpGet(url);
	request.addHeader("accept", "application/rdf+xml, application/x-turtle, application/rdf+n3, application/xml, text/turtle, text/rdf, text/plain;q=0.1");
	request.addHeader("User-Agent", CONSTANTS.USER_AGENT);
	log.info("GET {}",request);
	HttpResponse resp;
	try {
		resp = cm.connect(request);
		log.debug("checkForVoid: Connected to {}", request);
		String type = getType(resp);

		String status = ""+resp.getStatusLine().getStatusCode();
		info.setResponseCode(status);
		
		Header [] header = resp.getAllHeaders();
		
		// 1) CHeck the header for information
		parseHeaders(info,  header);
		if(status.startsWith("2")){
			String content = EntityUtils.toString(resp.getEntity());	
			content= content.replaceAll("\t",	" ");
			info.setContent(content);
			PipedRDFIterator<Triple> iter = new PipedRDFIterator<Triple>();
			final PipedRDFStream<Triple> inputStream = new PipedTriplesStream(iter);
			ByteArrayInputStream bais = new ByteArrayInputStream(content.getBytes());
			RDFDataMgr.parse(inputStream, bais,url, getLangFromType(type));

			while(iter.hasNext()){
				Triple t = iter.next();
				String pred  = t.getPredicate().toString();
				if(pred.startsWith(sparqDescNS)){
					update(pred.replace(sparqDescNS, ""), spdsPred );
				}else if(pred.startsWith(voidNS)){
					update( pred.replace(voidNS, ""), voidPred );
				}
			}
		}
	}catch(Exception e ){
		log.warn("failed checking for VOID "+url+" for "+_epURI, ExceptionHandler.logAndtoString(e,true));
		info.setException(ExceptionHandler.logAndtoString(e));
	}finally{
		request.releaseConnection();
	}
	return info;
}