C++ path_type::empty方法代码示例

int main(int argc, char** argv)
{
  utils::mpi_world  mpi_world(argc, argv);
  
  const int mpi_rank = MPI::COMM_WORLD.Get_rank();
  const int mpi_size = MPI::COMM_WORLD.Get_size();  
  
  try {
    if (getoptions(argc, argv) != 0) 
      return 1;

    if (! temporary_dir.empty())
      ::setenv("TMPDIR_SPEC", temporary_dir.string().data(), 1);
    
    if (ngram_file.empty() || ! boost::filesystem::exists(ngram_file))
      throw std::runtime_error("no ngram file?");
    if (output_file.empty())
      throw std::runtime_error("no output file?");
    if (ngram_file == output_file)
      throw std::runtime_error("dump to the same directory?");
    
    ngram_type ngram(debug);
    ngram.open_shard(ngram_file, mpi_rank);

    if (static_cast<int>(ngram.index.size()) != mpi_size)
      throw std::runtime_error("MPI universe size do not match with ngram shard size");
    
    utils::resource start;

    ngram_quantize(ngram);

    utils::resource end;
    
    if (debug && mpi_rank == 0)
      std::cerr << "quantize language model"
		<< " cpu time:  " << end.cpu_time() - start.cpu_time() 
		<< " user time: " << end.user_time() - start.user_time()
		<< std::endl;
    
    if (mpi_rank == 0)
      ngram.write_prepare(output_file);
    
    MPI::COMM_WORLD.Barrier();
    ngram.write_shard(output_file, mpi_rank);
  }
  catch (std::exception& err) {
    std::cerr << "error: "  << err.what() << std::endl;
    MPI::COMM_WORLD.Abort(1);
    return 1;
  }
  return 0;
}

int main(int argc, char** argv)
{
  try {
    if (getoptions(argc, argv) != 0) 
      return 1;

    if (! temporary_dir.empty())
      ::setenv("TMPDIR_SPEC", temporary_dir.string().data(), 1);
    
    expgram::NGram ngram(ngram_file, shards, debug);
    
    utils::compress_ostream os(output_file);
    os << "ngram order: " << ngram.index.order() << '\n';
    for (int order = 1; order <= ngram.index.order(); ++ order)
      os << order << "-gram: " << std::setw(16) << ngram.index.ngram_size(order) << '\n';
    
    dump(os, "index",    ngram.stat_index()) << '\n';
    dump(os, "pointer",  ngram.stat_pointer()) << '\n';
    dump(os, "vocab",    ngram.stat_vocab()) << '\n';
    dump(os, "logprob",  ngram.stat_logprob()) << '\n';
    dump(os, "backoff",  ngram.stat_backoff()) << '\n';
    dump(os, "logbound", ngram.stat_logbound()) << '\n';
  }
  catch (std::exception& err) {
    std::cerr << "error: " << err.what() << std::endl;
    return 1;
  }
  return 0;
}

  void NGram::open(const path_type& path)
  {
    typedef utils::repository repository_type;
    
    clear();

    if (path.empty())
      throw std::runtime_error("no ngram?");
    else if (! boost::filesystem::exists(path))
      throw std::runtime_error("no ngram? " + path.string());
    
    repository_type rep(path, repository_type::read);
    
    index.open(rep.path("index"));
    
    if (boost::filesystem::exists(rep.path("logprob")))
      open_shards(rep.path("logprob"), logprobs);
    
    if (boost::filesystem::exists(rep.path("backoff")))
      open_shards(rep.path("backoff"), backoffs);
    
    if (boost::filesystem::exists(rep.path("logbound")))
      open_shards(rep.path("logbound"), logbounds);
    
    repository_type::const_iterator siter = rep.find("smooth");
    if (siter == rep.end())
      throw std::runtime_error("no smoothing parameter...?");
    smooth = utils::lexical_cast<logprob_type>(siter->second);
    
    if (debug)
      std::cerr << "ngram: " << path
		<< " # of shards: " << index.size()
		<< " smooth: " << smooth
		<< std::endl;
  }

int main(int argc, char ** argv)
{
  try {
    options(argc, argv);

    cicada::optimize::LineSearch::value_min = value_lower;
    cicada::optimize::LineSearch::value_max = value_upper;

    if (scorer_list) {
      std::cout << cicada::eval::Scorer::lists();
      return 0;
    }    
    
    if (regularize_l1 && regularize_l2)
      throw std::runtime_error("you cannot use both of L1 and L2...");
    
    if (regularize_l1 || regularize_l2) {
      if (C <= 0.0)
	throw std::runtime_error("the scaling for L1/L2 must be positive");
    }
    
    if (weight_normalize_l1 && weight_normalize_l2)
      throw std::runtime_error("you cannot use both of L1 and L2 for weight normalization...");


    threads = utils::bithack::max(threads, 1);
    
    // read reference set
    scorer_document_type scorers(scorer_name);
    read_refset(refset_files, scorers);

    const size_t scorers_size = scorers.size();
    
    if (iterative && tstset_files.size() > 1) {
      scorer_document_type scorers_iterative(scorer_name);
      scorers_iterative.resize(scorers.size() * tstset_files.size());
      
      for (size_t i = 0; i != tstset_files.size(); ++ i)
	std::copy(scorers.begin(), scorers.end(), scorers_iterative.begin() + scorers.size() * i);
      
      scorers.swap(scorers_iterative);
    }

    if (debug)
      std::cerr << "# of references: " << scorers.size() << std::endl;


    if (debug)
      std::cerr << "reading kbests" << std::endl;

    hypothesis_map_type kbests(scorers.size());
    
    read_tstset(tstset_files, kbests, scorers_size);

    initialize_score(kbests, scorers);

    // collect initial weights
    weight_set_collection_type weights;
    
    if (! feature_weights_files.empty()) {

      for (path_set_type::const_iterator fiter = feature_weights_files.begin(); fiter != feature_weights_files.end(); ++ fiter) {
	if (*fiter != "-" && ! boost::filesystem::exists(*fiter))
	  throw std::runtime_error("no file? " + fiter->string());
	
	utils::compress_istream is(*fiter);
	
	weights.push_back(weight_set_type());
	is >> weights.back();
      }
      
      if (initial_average && weights.size() > 1) {
	weight_set_type weight;
	
	weight_set_collection_type::const_iterator witer_end = weights.end();
	for (weight_set_collection_type::const_iterator witer = weights.begin(); witer != witer_end; ++ witer)
	  weight += *witer;
	
	weight *= (1.0 / weights.size());
	
	weights.push_back(weight);
      }
      
      std::set<weight_set_type, std::less<weight_set_type>, std::allocator<weight_set_type> > uniques;
      uniques.insert(weights.begin(), weights.end());
      weights.clear();
      weights.insert(weights.end(), uniques.begin(), uniques.end());
    } else {
      weights.push_back(weight_set_type());
      
      // all one weight...
      for (feature_type::id_type id = 0; id < feature_type::allocated(); ++ id)
	if (! feature_type(id).empty())
	  weights.back()[feature_type(id)] = 1.0;
    }
    
    // collect lower/upper bounds
    weight_set_type bound_lower;
    weight_set_type bound_upper;
    
//.........这里部分代码省略.........

int main(int argc, char ** argv)
{
  utils::mpi_world mpi_world(argc, argv);
  
  const int mpi_rank = MPI::COMM_WORLD.Get_rank();
  const int mpi_size = MPI::COMM_WORLD.Get_size();
  
  try {
    options(argc, argv);

    cicada::optimize::LineSearch::value_min = value_lower;
    cicada::optimize::LineSearch::value_max = value_upper;
    
    if (scorer_list) {
      std::cout << cicada::eval::Scorer::lists();
      return 0;
    }
    
    if (int(yield_sentence) + yield_alignment + yield_span > 1)
      throw std::runtime_error("specify either sentence|alignment|span yield");
    if (int(yield_sentence) + yield_alignment + yield_span == 0)
      yield_sentence = true;

    if (weights_file.empty() || ! boost::filesystem::exists(weights_file))
      throw std::runtime_error("no weight file? " + weights_file.string());
    if (direction_name.empty())
      throw std::runtime_error("no direction?");
    
    // read reference set
    scorer_document_type scorers(scorer_name);
    
    read_refset(refset_files, scorers);
    
    if (mpi_rank == 0 && debug)
      std::cerr << "# of references: " << scorers.size() << std::endl;

    // read test set
    
    if (mpi_rank == 0 && debug)
      std::cerr << "reading hypergraphs" << std::endl;

    hypergraph_set_type graphs(scorers.size());
    
    read_tstset(tstset_files, graphs);
    
    weight_set_type weights;
    {
      utils::compress_istream is(weights_file, 1024 * 1024);
      is >> weights;
    }
    
    weight_set_type direction;
    direction[direction_name] = 1.0;
    
    segment_document_type segments(graphs.size());
    
    compute_envelope(scorers, graphs, weights, direction, segments);

    if (mpi_rank == 0) {
      line_search_type line_search(debug);
      
      utils::compress_ostream os(output_file, 1024 * 1024);
      
      line_search(segments, value_lower, value_upper, OutputIterator(os, weights[direction_name]));
    }
  }
  catch (const std::exception& err) {
    std::cerr << "error: " << err.what() << std::endl;
    return 1;
  }
  return 0;
}

int main(int argc, char** argv)
{
  try {
    options(argc, argv);

    if (int(leaf_mode) + treebank_mode > 1)
      throw std::runtime_error("multiple output options specified: leaf/treebank(default: treebank)");
    if (int(leaf_mode) + treebank_mode == 0)
      treebank_mode = true;
    
    typedef boost::spirit::istream_iterator iter_type;

    const bool flush_output = (output_file == "-"
			       || (boost::filesystem::exists(output_file)
				   && ! boost::filesystem::is_regular_file(output_file)));
    
    penntreebank_grammar<iter_type> grammar;
    
    treebank_type parsed;
    sentence_type sent;
    pos_set_type  pos;
    std::string   line;
    
    utils::compress_istream is(input_file, 1024 * 1024);
    utils::compress_ostream os(output_file, 1024 * 1024);

    std::auto_ptr<utils::compress_istream> ms;
    
    if (! pos_file.empty()) {
      if (! boost::filesystem::exists(pos_file))
	throw std::runtime_error("no map file: " + pos_file.string());
      
      ms.reset(new utils::compress_istream(pos_file, 1024 * 1024));
    }
    
    is.unsetf(std::ios::skipws);
    iter_type iter(is);
    iter_type iter_end;
        
    while (iter != iter_end) {
      parsed.clear();
      
      if (! boost::spirit::qi::phrase_parse(iter, iter_end, grammar, boost::spirit::standard::space, parsed)) {
	std::string buffer;
	for (int i = 0; i != 64 && iter != iter_end; ++ i, ++iter)
	  buffer += *iter;
	
	throw std::runtime_error("parsing failed: " + buffer);
      }

      if (! root_symbol.empty())
	parsed.cat_ = root_symbol;
      else if (parsed.cat_.empty())
	parsed.cat_ = "ROOT";

      if (validate)
	if (! treebank_validate(parsed))
	  throw std::runtime_error("invalid tree");

      if (ms.get()) {
	typedef boost::tokenizer<utils::space_separator, utils::piece::const_iterator, utils::piece> tokenizer_type;

	if (! utils::getline(*ms, line))
	  throw std::runtime_error("# of lines do not match with POS");
	
	utils::piece line_piece(line);
	tokenizer_type tokenizer(line_piece);
	
	pos.clear();
	pos.insert(pos.end(), tokenizer.begin(), tokenizer.end());

	pos_set_type::iterator iter = pos.begin();
	
	transform_pos(parsed, iter, pos.end());

	if (iter != pos.end())
	  throw std::runtime_error("too long POS sequence?");
      }
      
      if (remove_none)
	transform_remove_none(parsed);
      
      if (normalize)
	transform_normalize(parsed);
      
      if (remove_cycle)
	transform_cycle(parsed);

      if (unescape_terminal)
	transform_unescape(parsed);
      
      if (leaf_mode) {
	sent.clear();
	
	transform_leaf(parsed, sent);
	
	if (! sent.empty()) {
	  std::copy(sent.begin(), sent.end() - 1, std::ostream_iterator<std::string>(os, " "));
	  os << sent.back();
	}
//.........这里部分代码省略.........

int main(int argc, char ** argv)
{
  try {
    options(argc, argv);
    
    feature_list_type features_confidence;
    feature_list_type features_count;

    if (! confidence_feature_file.empty()) {
      if (confidence_feature_file != "-" && ! boost::filesystem::exists(confidence_feature_file))
	throw std::runtime_error("no confidence feature file? " + confidence_feature_file.string());
      
      utils::compress_istream is(confidence_feature_file);
      std::string feature;
      while (is >> feature)
	features_confidence.push_back(feature);
    }
    
    if (! count_feature_file.empty()) {
      if (count_feature_file != "-" && ! boost::filesystem::exists(count_feature_file))
	throw std::runtime_error("no count feature file? " + count_feature_file.string());
      
      utils::compress_istream is(count_feature_file);
      std::string feature;
      while (is >> feature)
	features_count.push_back(feature);
    }

    const bool flush_output = (output_file == "-"
                               || (boost::filesystem::exists(output_file)
                                   && ! boost::filesystem::is_regular_file(output_file)));
    
    hypergraph_type merged;
    hypergraph_type hypergraph;
    
    cicada::Feature feature_confidence(confidence);
    cicada::Feature feature_count(count);
    
    if (input_files.empty())
      input_files.push_back("-");
    
    if (multiple_mode) {
      namespace qi = boost::spirit::qi;
      namespace standard = boost::spirit::standard;
      
      // forest ||| forest ||| forest

      utils::compress_ostream os(output_file, 1024 * 1024 * (! flush_output));
      
      for (path_set_type::const_iterator iter = input_files.begin(); iter != input_files.end(); ++ iter) {
	utils::compress_istream is(input_files.front(), 1024 * 1024);
	std::string line;
	
	while (utils::getline(is, line)) {
	  int rank = 1;
	  int id = 0;
	  
	  merged.clear();
	  hypergraph.clear();
	  
	  std::string::const_iterator iter = line.begin();
	  std::string::const_iterator end = line.end();
	  
	  for (/**/; iter != end; ++ id, ++ rank) {
	    if (id != 0)
	      if (! qi::phrase_parse(iter, end, "|||", standard::space))
		break;
	    
	    if (! hypergraph.assign(iter, end))
	      throw std::runtime_error("invalid hypergraph format");
	    if (! hypergraph.is_valid()) continue;
	    
	    const double conf = 1.0 / (1.0 + rank);
	      
	    feature_set_type features;
	      
	    if (! features_confidence.empty()) {
	      if (id >= static_cast<int>(features_confidence.size()))
		throw std::runtime_error("# of confidence features do not match");
	      features[features_confidence[id]] = conf;
	    }
	    if (! features_count.empty()) {
	      if (id >= static_cast<int>(features_count.size()))
		throw std::runtime_error("# of count features do not match");
	      features[features_count[id]] = count_weight;
	    }
	    if (! feature_confidence.empty())
	      features[feature_confidence] = conf;
	    if (! feature_count.empty())
	      features[feature_count] = count_weight;
	      
	    if (! features.empty()) {
	      hypergraph_type::edge_set_type::iterator eiter_end = hypergraph.edges.end();
	      for (hypergraph_type::edge_set_type::iterator eiter = hypergraph.edges.begin(); eiter != eiter_end; ++ eiter)
		eiter->features += features;
	    }
	    
	    merged.unite(hypergraph);
	  }
	  
//.........这里部分代码省略.........

int main(int argc, char** argv)
{
  try {
    options(argc, argv);
    
    if (grammar_list) {
      std::cout << grammar_type::lists();
      return 0;
    }
    
    if (tree_grammar_list) {
      std::cout << tree_grammar_type::lists();
      return 0;
    }
    
    threads = utils::bithack::max(1, threads);

    // read grammars...
    grammar_type grammar(grammar_files.begin(), grammar_files.end());
    if (debug)
      std::cerr << "grammar: " << grammar.size() << std::endl;
    
    tree_grammar_type tree_grammar(tree_grammar_files.begin(), tree_grammar_files.end());
    if (debug)
      std::cerr << "tree grammar: " << tree_grammar.size() << std::endl;

    typedef Task task_type;
    typedef std::vector<task_type, std::allocator<task_type> > task_set_type;
    
    task_type::queue_type queue(threads);
    task_set_type tasks(threads, task_type(queue, tree_grammar, grammar));
    
    boost::thread_group workers;
    for (int i = 0; i != threads; ++ i)
      workers.add_thread(new boost::thread(boost::ref(tasks[i])));
    
    utils::compress_istream is(input_file, 1024 * 1024);
    
    std::string line;
    while (utils::getline(is, line))
      if (! line.empty())
	queue.push_swap(line);
    
    for (int i = 0; i != threads; ++ i)
      queue.push(std::string());
    
    workers.join_all();

    tree_rule_pair_unique_type tree_rules_unique;
    rule_pair_unique_type      rules_unique;
    
    for (int i = 0; i != threads; ++ i) {
      if (tree_rules_unique.empty())
	tree_rules_unique.swap(tasks[i].tree_rules_unique);
      else
	tree_rules_unique.insert(tasks[i].tree_rules_unique.begin(), tasks[i].tree_rules_unique.end());
      
      tasks[i].tree_rules_unique.clear();
      
      if (rules_unique.empty())
	rules_unique.swap(tasks[i].rules_unique);
      else
	rules_unique.insert(tasks[i].rules_unique.begin(), tasks[i].rules_unique.end());
      
      tasks[i].rules_unique.clear();
    }
    tasks.clear();

    typedef std::ostream_iterator<char> oiter_type;

    features_generator<oiter_type>   generate_features;
    attributes_generator<oiter_type> generate_attributes;
    
    if (! output_tree_file.empty()) {
      namespace karma = boost::spirit::karma;
      namespace standard = boost::spirit::standard;
      
      utils::compress_ostream os(output_tree_file, 1024 * 1024);
      
      tree_rule_pair_unique_type::const_iterator iter_end = tree_rules_unique.end();
      for (tree_rule_pair_unique_type::const_iterator iter = tree_rules_unique.begin(); iter != iter_end; ++ iter) {
	os << iter->source.decode() << " ||| " << iter->target.decode();
	
	if (! iter->features.empty()) {
	  feature_set_type features(iter->features.begin(), iter->features.end());
	  karma::generate(oiter_type(os), generate_features, features);
	}
	
	if (! iter->attributes.empty())
	  karma::generate(oiter_type(os), generate_attributes, iter->attributes);
	os << '\n';
      }
    }
    
    if (! output_rule_file.empty()) {
      namespace karma = boost::spirit::karma;
      namespace standard = boost::spirit::standard;
      
      utils::compress_ostream os(output_rule_file, 1024 * 1024);
      
//.........这里部分代码省略.........