Java ProgressLogger.lightUpdate方法代码示例

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
@Override
public Void call() throws Exception {
	final ProgressLogger pl = new ProgressLogger(LOGGER, 1, TimeUnit.MINUTES, "records");

	pl.itemsName = "records";
	pl.displayFreeMemory = true;
	pl.displayLocalSpeed = true;
	pl.start("Scanning...");

	try {
		for(;;) {
			final Result<?>[] result = queue.take();
			if (result == Result.END_OF_RESULTS) {
				pl.done();
				return null;
			}
			for (final Result<?> r : result) if (r != null) r.write();
			pl.lightUpdate();
		}
	}
	catch(final Exception e) {
		LOGGER.error("Exception in flushing thread", e);
		throw e;
	}
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
/**
 * Returns a list of pointers to a GZIP archive entries positions (including the end of file).
 *
 * @param in the stream from which to read the GZIP archive.
 * @param pl a progress logger.
 * @return a list of longs where the <em>i</em>-th long is the offset in the stream of the first byte of the <em>i</em>-th archive entry.
 */
public static LongBigArrayBigList index(final InputStream in, final ProgressLogger pl) throws IOException {
	final LongBigArrayBigList pointers = new LongBigArrayBigList();
	long current = 0;
	final GZIPArchiveReader gzar = new GZIPArchiveReader(in);
	GZIPArchive.ReadEntry re;
	for (;;) {
		re = gzar.skipEntry();
		if (re == null) break;
		pointers.add(current);
		current += re.compressedSkipLength;
		if (pl != null) pl.lightUpdate();
	}
	in.close();
	return pointers;
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
@SuppressWarnings({ "rawtypes", "unchecked" })
public static Map invert(Map inputMap) throws InstantiationException,
		IllegalAccessException, InvocationTargetException,
		NoSuchMethodException, ClassNotFoundException {
	LOGGER.info("Inverting map...");
	Map outputMap = (Map) invertMapType(inputMap.getClass()).getConstructor(new Class[] {}).newInstance(new Object[] {});
	
	ProgressLogger pl = new ProgressLogger(LOGGER, "entries");
	pl.expectedUpdates = inputMap.size();
	pl.start();
	
	for (Object entryObj : inputMap.entrySet()) {
		Map.Entry entry = (Map.Entry) entryObj;
		Object oldValue = outputMap.put(entry.getValue(), entry.getKey());
		if (oldValue != null)
			throw new IllegalArgumentException(
					"The value " + entry.getValue() + " is associated to both '" +
					oldValue + "' and '" + entry.getKey() + "'. The map is not" +
					"bijective"
			);
		pl.lightUpdate();
	}
	pl.done();
	return outputMap;
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
public static SummaryStatistics computeStatistics(UnexpectednessScorer scorer) {
	int numNodes = scorer.graph.numNodes();
	SummaryStatistics stats = new SummaryStatistics();
	
	ProgressLogger pl = new ProgressLogger(LOGGER, "docs");
	pl.expectedUpdates = numNodes;
	pl.start("Finding statistics for values of " + scorer + "...");

	for (int i = 0; i < numNodes; i++) {
		for (double x : scorer.scores(i).values()) {
			stats.addValue(x);
			if (Double.isInfinite(x) || Double.isNaN(x))
				throw new ArithmeticException(
						"Scorer " + scorer + " has returned value "
						+ x + " for a result of document " + i);
		}
		pl.lightUpdate();
	}
	pl.done();
	
	return stats;
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
public static SummaryStatistics computeApproxStatistics(UnexpectednessScorer scorer) {
	int numNodes = scorer.graph.numNodes();
	int sampleSize = 10000;
	if (sampleSize > numNodes) sampleSize = numNodes;

	SummaryStatistics stats = new SummaryStatistics();
	
	ProgressLogger pl = new ProgressLogger(LOGGER, "docs");
	pl.expectedUpdates = sampleSize;
	pl.start("Finding statistics for values of " + scorer + " (sample of "+sampleSize+")...");

	for (int i: RandomSingleton.get().ints(sampleSize, 0, numNodes).toArray()) {
		for (double x : scorer.scores(i).values()) {
			stats.addValue(x);
			if (Double.isInfinite(x) || Double.isNaN(x))
				throw new ArithmeticException(
						"Scorer " + scorer + " has returned value "
						+ x + " for a results of document " + i);
		}
		pl.lightUpdate();
	}
	pl.done();
	
	LOGGER.info(scorer + " -- sample of " + numNodes + " -- " +  stats.toString());
	return stats;
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
public void compute() {
	ProgressLogger pl = new ProgressLogger(LOGGER, "pages");
	pl.expectedUpdates = page2cat.size();
	pl.start("Moving old categories to closest milestones...");
	for (IntSet entry : page2cat.values()) {
		IntSet newCategories = new IntOpenHashSet();
		int milestone;
		for (int cat : entry) {
			milestone = closestMilestones[cat];
			if (milestone != -1)
				newCategories.add(milestone);
		}
		entry.clear();
		entry.addAll(newCategories);
		pl.lightUpdate();
	}
	pl.done();
	
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
public static void run( final FastBufferedReader fbr, final DataOutputStream dos, ProgressLogger pl ) throws IOException {
	final MutableString s = new MutableString();
	Object2IntOpenHashMap<MutableString> map = new Object2IntOpenHashMap<MutableString>();
	map.defaultReturnValue( -1 );
	int slash, hostIndex = -1;

	if ( pl != null ) pl.start( "Reading URLS..." );
	while( fbr.readLine( s ) != null ) {
		slash = s.indexOf( '/', 8 );
		// Fix for non-BURL URLs
		if ( slash != -1 ) s.length( slash );
		if ( ( hostIndex = map.getInt( s ) ) == -1 ) map.put( s.copy(), hostIndex = map.size() );
		dos.writeInt( hostIndex );
		if ( pl != null ) pl.lightUpdate();
	}
	
	if ( pl != null ) pl.done();
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
public static void main(String[] arg) throws IOException {
	if (arg.length == 0) {
		System.err.println("Usage: " + BuildRepetitionSet.class.getSimpleName() + " REPETITIONSET");
		System.exit(1);
	}

	final FastBufferedReader fastBufferedReader = new FastBufferedReader(new InputStreamReader(System.in, Charsets.US_ASCII));
	final MutableString s = new MutableString();
	final LongOpenHashSet repeatedSet = new LongOpenHashSet();
	final String outputFilename = arg[0];
	final ProgressLogger pl = new ProgressLogger();

	MutableString lastUrl = new MutableString();
	pl.itemsName = "lines";
	pl.start("Reading... ");
	while(fastBufferedReader.readLine(s) != null) {
		final int firstTab = s.indexOf('\t');
		final int secondTab = s.indexOf('\t', firstTab + 1);
		MutableString url = s.substring(secondTab + 1);
		if (url.equals(lastUrl)) {
			final int storeIndex = Integer.parseInt(new String(s.array(), 0, firstTab));
			final long storePosition = Long.parseLong(new String(s.array(), firstTab + 1, secondTab - firstTab - 1));
			repeatedSet.add((long)storeIndex << 48 | storePosition);
			System.out.print(storeIndex);
			System.out.print('\t');
			System.out.print(storePosition);
			System.out.print('\t');
			System.out.println(url);
		}

		lastUrl = url;
		pl.lightUpdate();
	}

	pl.done();

	fastBufferedReader.close();
	BinIO.storeObject(repeatedSet, outputFilename);
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
@SuppressWarnings("resource")
public static int[] writeRecords(final String path, final int numRecords, final WarcRecord[] randomRecords, final int parallel) throws IOException, InterruptedException {
	final ProgressLogger pl = new ProgressLogger(LOGGER, "records");
	if (parallel <= 1) pl.expectedUpdates = numRecords;
	final ProgressLogger plb = new ProgressLogger(LOGGER, "KB");
	final CountingOutputStream cos = new CountingOutputStream(new FastBufferedOutputStream(new FileOutputStream (path)));
	final WarcWriter ww;
	if (parallel == 0) {
		ww = new UncompressedWarcWriter(cos);
		pl.start("Writing records…");
	} else if (parallel == 1) {
		ww = new CompressedWarcWriter(cos);
		pl.start("Writing records (compressed)…");
	} else {
		ww = null;
		pl.start("SHOULD NOT HAPPEN");
		throw new IllegalStateException();
	}
	plb.start();
	long written = 0;
	final int[] position = new int[numRecords];
	for (int i = 0; i < numRecords; i++) {
		final int pos = RandomTestMocks.RNG.nextInt(randomRecords.length);
		position[i] = pos;
		ww.write(randomRecords[pos]);
		if (parallel <= 0) {
			pl.lightUpdate();
			plb.update((cos.getCount() - written) / 1024);
		}
		written = cos.getCount();
	}
	ww.close();
	pl.done(numRecords);
	plb.done(cos.getCount());
	return position;
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
public void toMatrixMarketFormat(Writer writer) throws IOException {
	writer.write("%%MatrixMarket matrix coordinate real general\n");
	
	int entriesToWrite = numberOfNonZeroEntries();
	
	if (entriesToWrite == 0)
		throw new RuntimeException("Empty matrix. Aborting Matrix Market export.");
	
	int size = computeSize();
	
	writer.write(size + " " + size + " " + entriesToWrite + "\n");
	
	ObjectIterator<Element> elements = entries();
	
	ProgressLogger pl = new ProgressLogger(LOGGER, "elements");
	pl.expectedUpdates = entriesToWrite;
	pl.start("Exporting matrix to Matrix Market format...");
	
	while (elements.hasNext()) {
			Element e = elements.next();
			if (e.value != 0) {
				writer.write((e.i+1) + " " + (e.j+1) + " " + e.value + "\n");
				entriesToWrite--;
				pl.lightUpdate();
			}
		}
	
	pl.done();
		
	if (entriesToWrite != 0)
		throw new IllegalStateException();
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
public static void store( final ArcLabelledImmutableGraph graph, final CharSequence basename, final CharSequence underlyingBasename, final ProgressLogger pl ) throws IOException {
	final OutputBitStream labels = new OutputBitStream( basename + LABELS_EXTENSION, STD_BUFFER_SIZE ); 
	final OutputBitStream offsets = new OutputBitStream( basename + LABEL_OFFSETS_EXTENSION, STD_BUFFER_SIZE );

	if ( pl != null ) {
		pl.itemsName = "nodes";
		pl.expectedUpdates = graph.numNodes();
		pl.start( "Saving labels..." );
	}

	final ArcLabelledNodeIterator nodeIterator = graph.nodeIterator();
	offsets.writeGamma( 0 );
	int curr;
	long count;
	LabelledArcIterator successors;

	while( nodeIterator.hasNext() ) {
		curr = nodeIterator.nextInt();
		successors = nodeIterator.successors();
		count = 0;
		while( successors.nextInt() != -1 ) count += successors.label().toBitStream( labels, curr );
		offsets.writeLongGamma( count );
		if ( pl != null ) pl.lightUpdate();
	}
	
	if ( pl != null ) pl.done();
	labels.close();
	offsets.close();
	
	final PrintWriter properties = new PrintWriter( new FileOutputStream( basename + ImmutableGraph.PROPERTIES_EXTENSION ) );
	properties.println( ImmutableGraph.GRAPHCLASS_PROPERTY_KEY + " = " + BitStreamArcLabelledImmutableGraph.class.getName() );
	properties.println( ArcLabelledImmutableGraph.UNDERLYINGGRAPH_PROPERTY_KEY + " = " + underlyingBasename );
	properties.println( BitStreamArcLabelledImmutableGraph.LABELSPEC_PROPERTY_KEY + " = " + graph.prototype().toSpec() );
	properties.close();
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
public static void gatherExamples(SmaphAnnotator smaph, A2WDataset ds,
        ExampleGatherer<Tag, HashSet<Tag>> entityFilterGatherer,
        ExampleGatherer<Annotation, HashSet<Annotation>> annotationRegressorGatherer,
        ExampleGatherer<HashSet<Annotation>, HashSet<Annotation>> collectiveGatherer,
        List<HashSet<Annotation>> greedyPartialSolution,
        int greedyStep,
        ExampleGatherer<Annotation, HashSet<Annotation>> greedyGatherer,
        boolean keepNEOnly,
		Map<String, QueryInformation> qiCache)
        throws Exception {
	
	// On first step, initialize greedy partial solutions
	if (greedyGatherer != null && greedyStep == 0 && greedyPartialSolution.isEmpty())
		for (int i = 0; i < ds.getSize(); i++)
			greedyPartialSolution.add(new HashSet<>());
			
	
	ProgressLogger plog = new ProgressLogger(logger, "document");
	plog.start("Collecting examples.");
	for (int i = 0; i < ds.getSize(); i++) {
		String query = ds.getTextInstanceList().get(i);
		HashSet<Tag> goldStandard = ds.getC2WGoldStandardList().get(i);
		HashSet<Annotation> goldStandardAnn = ds.getA2WGoldStandardList().get(i);
		List<Pair<FeaturePack<Tag>, Boolean>> efVectorsToPresence = null;
		List<Tag> efCandidates = null;
		List<Pair<FeaturePack<Annotation>, Boolean>> arVectorsToPresence = null;
		List<Annotation> arCandidates = null;
		List<Pair<FeaturePack<HashSet<Annotation>>, Double>> collVectorsToF1 = null;
		List<HashSet<Annotation>> collCandidates = null;
		List<Pair<FeaturePack<Annotation>, Double>> greedyVectorsToF1Incr = null;
		List<Annotation> greedyCandidates = null;
		HashSet<Annotation> greedyPartialSolutionI = null;

		if (entityFilterGatherer != null){
			efVectorsToPresence = new Vector<>();
			efCandidates = new Vector<>();
		}
		if (annotationRegressorGatherer != null) {
			arVectorsToPresence = new Vector<>();
			arCandidates = new Vector<>();
		}
		if (collectiveGatherer != null){
			collVectorsToF1 = new Vector<>();
			collCandidates = new Vector<>();
		}
		if (greedyGatherer != null){
			greedyVectorsToF1Incr = new Vector<>();
			greedyCandidates = new Vector<>();
			greedyPartialSolutionI = greedyPartialSolution.get(i);
		}

		smaph.generateExamples(query, goldStandard, goldStandardAnn, efVectorsToPresence, efCandidates, arVectorsToPresence,
		        arCandidates, collVectorsToF1, collCandidates, greedyPartialSolutionI, greedyStep, greedyVectorsToF1Incr, greedyCandidates, keepNEOnly, qiCache, null);

		if (entityFilterGatherer != null)
			entityFilterGatherer.addExample(goldBoolToDouble(efVectorsToPresence), efCandidates, goldStandard);
		if (collectiveGatherer != null)
			collectiveGatherer.addExample(collVectorsToF1, collCandidates, goldStandardAnn);
		if (annotationRegressorGatherer != null)
			annotationRegressorGatherer.addExample(goldBoolToDouble(arVectorsToPresence), arCandidates, goldStandardAnn);
		if (greedyGatherer != null)
			greedyGatherer.addExample(greedyVectorsToF1Incr, greedyCandidates, goldStandardAnn);

		plog.lightUpdate();
	}
	plog.done();
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
public static void main(String arg[]) throws IOException, InterruptedException, JSAPException {

	SimpleJSAP jsap = new SimpleJSAP(WarcCompressor.class.getName(),
		"Given a store uncompressed, write a compressed store.",
		new Parameter[] { new FlaggedOption("output", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, 'o', "output", "The output filename  (- for stdout)."),
		new UnflaggedOption("store", JSAP.STRING_PARSER, JSAP.NOT_REQUIRED, "The name of the store (if omitted, stdin)."),
	});

	JSAPResult jsapResult = jsap.parse(arg);
	if (jsap.messagePrinted()) return;

	final InputStream in = jsapResult.userSpecified("store") ? new FastBufferedInputStream(new FileInputStream(jsapResult.getString("store"))) : System.in;

	final WarcReader reader = new UncompressedWarcReader(in);
	final ProgressLogger pl = new ProgressLogger(LOGGER, 1, TimeUnit.MINUTES, "records");
	final String output = jsapResult.getString("output");

	PrintStream out = "-".equals(output) ? System.out : new PrintStream(new FastBufferedOutputStream(new FileOutputStream(output)), false, "UTF-8");
	final WarcWriter writer = new CompressedWarcWriter(out);

	pl.itemsName = "records";
	pl.displayFreeMemory = true;
	pl.displayLocalSpeed = true;
	pl.start("Scanning...");

	for (long storePosition = 0;; storePosition++) {
	    LOGGER.trace("STOREPOSITION " + storePosition);
	    WarcRecord record = null;
	    try {
		record = reader.read();
	    } catch (Exception e) {
		LOGGER.error("Exception while reading record " + storePosition + " ");
		LOGGER.error(e.getMessage());
		e.printStackTrace();
		continue;
	    }
	    if (record == null)
		break;
	    writer.write(record);
	    pl.lightUpdate();
	}
	pl.done();
	writer.close();
    }
 

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
/**
 * Use this method to compute entity embeddings on a single machine. See --help
 * @param args command line arguments
 * @throws JSAPException
 * @throws ClassNotFoundException
 * @throws IOException
 */
public static void standalonemain( String args[] ) throws JSAPException, ClassNotFoundException, IOException {
    SimpleJSAP jsap = new SimpleJSAP( EntityEmbeddings.class.getName(), "Learns entity embeddings", new Parameter[]{
            new FlaggedOption( "input", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, 'i', "input", "Entity description files" ),
            new FlaggedOption( "vectors", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, 'v', "vectors", "Word2Vec file" ),
            new FlaggedOption( "rho", JSAP.INTEGER_PARSER, "-1", JSAP.NOT_REQUIRED, 'r', "rho", "rho negative sampling parameters (if it's <0 we use even sampling)" ),
            new FlaggedOption( "max", JSAP.INTEGER_PARSER, "-1", JSAP.NOT_REQUIRED, 'm', "max", "Max words per entity (<0 we use all the words)" ),
            new FlaggedOption( "output", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, 'o', "output", "Compressed version" ), }
    );
    JSAPResult jsapResult = jsap.parse( args );
    if( jsap.messagePrinted() ) return;
    CompressedW2V vectors = new CompressedW2V( jsapResult.getString( "vectors" ) );
    ProgressLogger pl = new ProgressLogger();
    final int rho = jsapResult.getInt( "rho" );
    final int nwords = vectors.getSize();
    final int d = vectors.N;
    final int maxWords = jsapResult.getInt( "max" ) > 0? jsapResult.getInt( "max" ):Integer.MAX_VALUE;
    final BufferedReader br = new BufferedReader( new InputStreamReader( new FileInputStream( new File( jsapResult.getString( "input" ) ) ) ) );
    int count = 0;
    pl.count = count;
    pl.itemsName = "entities";
    while( br.readLine() != null ) count++;
    br.close();
    final PrintWriter pw = new PrintWriter( new BufferedWriter( new FileWriter( jsapResult.getString( "output" ), false ) ) );
    pw.println( count + " " + d );

    float alpha = 10;
    EntityEmbeddings eb = new EntityEmbeddings();
    final BufferedReader br2 = new BufferedReader( new InputStreamReader( new FileInputStream( new File( jsapResult.getString( "input" ) ) ) , "UTF-8") );
    pl.start();
    String line;
    while( ( line = br2.readLine() ) != null ) {
        String[] parts = line.split( "\t" );
        if( parts.length > 1 ) {
            TrainingExamples ex = eb.getVectors( parts[ 1 ], vectors, rho, nwords, maxWords );
            float[] w = eb.trainLR2( ex.x, d, ex.y, alpha );
            pw.print( parts[ 0 ] + " " );
            for( int i = 0; i < d; i++ ) {
                pw.print( w[ i ] + " " );
            }
            pw.println();
            pl.lightUpdate();
            pw.flush();

            for( int i = 0; i < ex.y.length; i++ ) {
                if( ex.y[ i ] > 0 ) {
                    double v = eb.scoreLR( ex.x[ i ], w );
                }
            }
        }
    }
    br2.close();
    pw.close();
    pl.stop();
}
 

import it.unimi.dsi.logging.ProgressLogger; //导入方法依赖的package包/类
public static void main(final String[] arg) throws JSAPException, IOException {

		final SimpleJSAP jsap = new SimpleJSAP(GenerateRandom64BitIntegers.class.getName(), "Generates a list of sorted 64-bit random integers in DataOutput format.",
				new Parameter[] {
					new FlaggedOption("gap", JSAP.INTSIZE_PARSER, "1", JSAP.NOT_REQUIRED, 'g', "gap", "Impose a minimum gap."),
					new UnflaggedOption("n", JSAP.LONG_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.NOT_GREEDY, "The number of integers (too small values might cause overflow)."),
					new UnflaggedOption("output", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.REQUIRED, JSAP.NOT_GREEDY, "The output file.")
		});

		final JSAPResult jsapResult = jsap.parse(arg);
		if (jsap.messagePrinted()) return;

		final long n = jsapResult.getLong("n");
		final int gap = jsapResult.getInt("gap");
		final String output = jsapResult.getString("output");

		final RandomGenerator r = new XoRoShiRo128PlusRandomGenerator();

		final ProgressLogger pl = new ProgressLogger(LOGGER);
		pl.expectedUpdates = n;
		pl.start("Generating... ");

		BigInteger l = BigInteger.ZERO;
		final BigInteger limit = BigInteger.valueOf(256).pow(8);
		final long incr = (long)Math.floor(1.99 * (limit.divide(BigInteger.valueOf(n)).longValue())) - 1;

		@SuppressWarnings("resource")
		final DataOutputStream dos = new DataOutputStream(new FileOutputStream(output));

		LOGGER.info("Increment: " + incr);

		for(long i = 0; i < n; i++) {
			l = l.add(BigInteger.valueOf((r.nextLong() & 0x7FFFFFFFFFFFFFFFL) % incr + gap));
			if (l.compareTo(limit) > 0) throw new AssertionError(Long.toString(i));
			dos.writeLong(l.longValue());
			pl.lightUpdate();
		}


		pl.done();
		dos.close();

		LOGGER.info("Last/limit: " + (l.doubleValue() / limit.doubleValue()));
	}