C++ scanner函数代码示例

rc_t generateFakeArchive(LogArchiver::ArchiveDirectory* dir,
        unsigned bytesPerRun, unsigned runCount, unsigned& total)
{
    total = 0;

    LogFactory factory(true, // sorted
            1, // start with this page ID
            10, // new page ID every 10 logrecs
            1 // increment max pade ID one by one
    );
    logrec_t lr;
    unsigned runsGen = 0;

    // create LogScanner to know which logrecs are ignored
    LogScanner scanner(BLOCK_SIZE);
    LogArchiver::initLogScanner(&scanner);

    LogArchiver::BlockAssembly assemb(dir);

    while (runsGen < runCount) {
        unsigned bytesGen = 0;
        assemb.start(runsGen);
        while (bytesGen < bytesPerRun) {
            factory.next(&lr);

            if (scanner.isIgnored(lr.type())) {
                continue;
            }
            assert(lr.valid_header(lr.lsn_ck()));

            if (!assemb.add(&lr)) {
                assemb.finish();
                assemb.start(runsGen);
                bool worked = assemb.add(&lr);
                EXPECT_TRUE(worked);
            }

            bytesGen += lr.length();
            total++;
        }

        assemb.finish();
        runsGen++;
    }

    assemb.shutdown();

    return RCOK;
}

int ParseUMapInfo(int lumpnum)
{
	FScanner scanner(lumpnum);
	unsigned int i;

	while (scanner.GetToken())
	{
		scanner.TokenMustBe(TK_Map);

		UMapEntry parsed;
		ParseMapEntry(scanner, &parsed);

		// Endpic overrides level exits.
		if (parsed.endpic[0])
		{
			parsed.nextmap[0] = parsed.nextsecret[0] = 0;
			if (parsed.endpic[0] == '!') parsed.endpic[0] = 0;
		}
		/*
		else if (!parsed.nextmap[0] && !parsed.endpic[0])
		{
			if (!parsed.MapName.CompareNoCase("MAP30")) uppercopy(parsed.endpic, "$CAST");
			else if (!parsed.MapName.CompareNoCase("E1M8"))  uppercopy(parsed.endpic, gameinfo.creditPages.Last());
			else if (!parsed.MapName.CompareNoCase("E2M8"))  uppercopy(parsed.endpic, "VICTORY");
			else if (!parsed.MapName.CompareNoCase("E3M8"))  uppercopy(parsed.endpic, "$BUNNY");
			else if (!parsed.MapName.CompareNoCase("E4M8"))  uppercopy(parsed.endpic, "ENDPIC");
			else if (gameinfo.gametype == GAME_Chex && !parsed.MapName.CompareNoCase("E1M5"))  uppercopy(parsed.endpic, "CREDIT");
			else
			{
				parsed.nextmap[0] = 0;	// keep previous setting
			}
		}
		*/

		// Does this property already exist? If yes, replace it.
		for(i = 0; i < Maps.Size(); i++)
		{
			if (!parsed.MapName.Compare(Maps[i].MapName))
			{
				Maps[i] = parsed;
				return 1;
			}
		}
		// Not found so create a new one.
		Maps.Push(parsed);
		
	}
	return 1;
}

QVariant Driver::parse(const QString &filename) {
	QString filePath = QFileInfo(filename).canonicalFilePath();
	Scanner scanner(filePath);
	Driver driver(filePath, scanner);
	Parser parser(driver);
	parser.parse();
	if (!driver.rootCtx()->m_errors.isEmpty()) {
		throw ErrorParserException(driver.rootCtx()->m_errors.join("\n"));
	}
	QVariantMap resProps;
	resProps[NODE_KEY_VERSION] = 1;
	resProps[NODE_KEY_MEMBERS] = driver.m_localElts;
	resProps[NODE_KEY_NODE_TYPE] = TYPE_NODE_FRAGMENT;
	return resProps;
}

Reaction & IndigoCdxReaction::getReaction ()
{
   if (!_loaded)
   {
      Indigo &self = indigoGetInstance();

      BufferScanner scanner(_data);
      ReactionCdxLoader loader(scanner);
      loader.stereochemistry_options = self.stereochemistry_options;
      loader.ignore_bad_valence = self.ignore_bad_valence;
      loader.loadReaction(_rxn);
      _loaded = true;
   }
   return _rxn;
}

void lista_declaraciones_init() {

    if (sbol->codigo == CIDENT) {
        strcpy(inf_id->nbre,sbol->lexema);
        scanner();
    } else {
        error_handler(16);    // falta identificador
    }
    tipo_global = inf_id->ptr_tipo;
    inf_id->clase = CLASVAR;

    declarador_init();
    if (!insertarTS()) {
        error_handler(9);
    }

    while (sbol->codigo == CCOMA) {

        scanner();

        if (sbol->codigo == CIDENT) {
            strcpy(inf_id->nbre,sbol->lexema);
            inf_id->ptr_tipo = tipo_global;
            inf_id->clase = CLASVAR;
            scanner();
        } else {
            error_handler(16);    // falta identificador
        }

        declarador_init();
        if (!insertarTS()) {
            error_handler(9);
        }
    }

}

PluralFormsCalculator* PluralFormsCalculator::make(const char* s)
{
    PluralFormsCalculator* calculator = new PluralFormsCalculator;
    if (s != NULL)
    {
        PluralFormsScanner scanner(s);
        PluralFormsParser p(scanner);
        if (!p.parse(*calculator))
        {
            delete calculator;
            return NULL;
        }
    }
    return calculator;
}

void RingoFastIndex::_decompressRowid (const Array<char> &stored, OraRowidText &rid)
{
   BufferScanner scanner(stored.ptr() + 2, stored[1]);

   RowIDLoader loader(_context.context().context().rid_dict, scanner);
   QS_DEF(Array<char>, rowid);

   loader.loadRowID(rowid);

   if (rowid.size() != 18)
      throw Error("rowid size=%d?", rowid.size());

   memcpy(rid.ptr(), rowid.ptr(), 18);
   rid.ptr()[18] = 0;
}

cmark_bufsize_t _ext_scan_at(cmark_bufsize_t (*scanner)(const unsigned char *),
	const char *s, cmark_bufsize_t offset)
{
	cmark_bufsize_t res;
	cmark_bufsize_t len = strlen(s);
	unsigned char *ptr = (unsigned char *)s;

        if (ptr == NULL || offset > len) {
          return 0;
        } else {
	  res = scanner(ptr + offset);
        }

	return res;
}

//<stat>           ->      <in> | <out> | <block> | <if> | <loop> | <assign>
node_t* stat_f() {
    node_t *p = getNode("<stat>");

    if (tk.tokenId == SCAN_tk) { // <in>
        p->child1 = getNode("SCAN");
        tk = scanner(fp, lineNum);
        p->child2 = in_f();
        return p;
    } else if (tk.tokenId == PRINT_tk) { // <out>
        p->child1 = getNode("PRINT");
        tk = scanner(fp, lineNum);
        p->child2 = out_f();
        return p;
    } else if (tk.name == "{") { // <block>
        //don't need to put delimiters in the tree
        p->child1 = block_f();
        return p;
    } else if (tk.tokenId == IF_tk) { // <if>
        p->child1 = getNode("IF");
        tk = scanner(fp, lineNum);
        p->child2 = if_f();
        return p;
    } else if (tk.tokenId == LOOP_tk) { // <loop>
        p->child1 = getNode("LOOP");
        tk = scanner(fp, lineNum);
        p->child2 = loop_f();
        return p;
    } else if (tk.tokenId == IDENT_tk) { // <assign>
        p->child1 = assign_f();
        return p;
    } else {
        error("Got token " + tk.name + ", expected SCAN, PRINT, {, IF, LOOP, or IDENTIFIER token\n");
    }

    return p;
}

//<program>  ->     PROGRAM <var> <block>
node_t* prog_f() {
    node_t *p = getNode("<program>");

    if(tk.tokenId == PROG_tk) { // PROGRAM
        p->child1 = getNode("PROGRAM");
        tk = scanner(fp, lineNum);
        p->child2 = var_f();
        p->child3 = block_f();
        return p;
    } else {
        error("Got token " + tk.name + ", expected PROGRAM token\n");
    }

    return p;
}

  sim::Llvm_library compile(ast::Node_if const& ast_root,
      std::string const& defaultname) {
    ir::Library<Llvm_impl> rv;

    rv.name = defaultname;
    rv.ns = std::make_shared<sim::Llvm_namespace>();

    ast::Ast_printer printer(std::cout);
    ast_root.accept(printer);

    Llvm_namespace_scanner scanner(*rv.ns);
    ast_root.accept(scanner);

    return rv;
  }

TEST(HttpMultipartScannerTest, Reset) {
  HttpMultipartScanner scanner("foo", true);

  scanner.Execute("abcdefg---\r\n--foo\r\n");
  EXPECT_TRUE(scanner.IsDone());
  EXPECT_EQ(scanner.GetSkipped(), "abcdefg---\r\n--foo\r\n");

  scanner.Reset(true);
  EXPECT_FALSE(scanner.IsDone());
  scanner.SetBoundary("bar");

  scanner.Execute("--foo\r\n--bar\r\n");
  EXPECT_TRUE(scanner.IsDone());
  EXPECT_EQ(scanner.GetSkipped(), "--foo\r\n--bar\r\n");
}

void KNNTest::run() {
	Scanner scanner("cpp/ml/test/TestKNNInput.txt");
	Writer writer("cpp/ml/test/TestKNNOutput.txt");
	KNN kNN(3);
	DatasetClassification dataset("cpp/ml/test/TestKNNData.txt");
    CrossValidatorParameter parameter;
    while (scanner.hasNext()) {
    	parameter.testingDataRatio = scanner.nextDouble();
    	CrossValidatorClassification validator(&kNN, &dataset, &parameter);
    	validator.validate();
    	double testingErr = validator.getTestingError();
    	writer.println(testingErr);
    }
    writer.close();
}

bool _ringoRegisterReaction (OracleEnv &env, const char *rowid,
                             const Array<char> &reaction_buf,
                             RingoOracleContext &context,
                             RingoIndex &index,
                             BingoFingerprints &fingerprints)
{
   QS_DEF(Array<char>, data);
   QS_DEF(Array<char>, compressed_rowid);
   ArrayOutput output(data);

   output.writeChar(0); // 0 -- present, 1 -- removed from index
   
   ArrayOutput rid_output(compressed_rowid);
   RowIDSaver rid_saver(context.context().rid_dict, rid_output);

   rid_saver.saveRowID(rowid);

   output.writeByte((byte)compressed_rowid.size());
   output.writeArray(compressed_rowid);

   TRY_READ_TARGET_RXN
   {
      BufferScanner scanner(reaction_buf);

      try
      {
         index.prepare(scanner, output, NULL);
      }
      catch (CmfSaver::Error &e)
      { 
         if (context.context().reject_invalid_structures)
            throw; // Rethrow this exception further
         env.dbgPrintf(bad_reaction_warning_rowid, rowid, e.message());
         return false;
      }
      catch (CrfSaver::Error &e)
      { 
         if (context.context().reject_invalid_structures)
            throw; // Rethrow this exception further
         env.dbgPrintf(bad_reaction_warning_rowid, rowid, e.message());
         return false;
      }
   }
   CATCH_READ_TARGET_RXN_ROWID(rowid, {
      if (context.context().reject_invalid_structures)
         throw; // Rethrow this exception further
      return false;
   });

int main(int argc, const char * argv[])
{
    /******************************************
     This is not a correct implementation, you will need to modfy this
     so that it satisfies the project problem.  Currently, this just
     prints every token and then deletes every token.
     *****************************************/
    Token *token = NULL;
    char source_name[MAX_FILE_NAME_LENGTH];
    char date[DATE_STRING_LENGTH];
    FILE *source_file = init_lister(argv[1], source_name, date);
    Print print(source_name, date);
    Scanner scanner(source_file, source_name, date, print);

	////
	Token* identifierSearchTreeHead = NULL;
	////
    
    do
    {
		//token: "Token* token"
        token = scanner.getToken();
        print.printToken(token);
        if (token->getCode() != PERIOD && token->getCode() != END_OF_FILE)
        {
			//?????????????????TODO: add identifiers to binary tree and add line numbers where
			//						the identifier shows up to a linked list stored in the identifier token
			if(token->getCode() == IDENTIFIER){
				Token::addTokenNodeToBinarySearchTree(identifierSearchTreeHead, token, scanner.getLineNumber());
			}else if(token->getCode() != PERIOD && token->getCode() != END_OF_FILE){
				delete token;
			}
        }
    }
    while (token->getCode() != PERIOD && token->getCode() != END_OF_FILE);


	//print line number info for identifier tokens
	cout << "\nCross Reference Information\n";
	cout << "Identifier\t\tLine Numbers\n";
	cout << "-----------\t\t------------\n";
	cout << Token::getBinarySearchTreeLinesStringsInOrder(identifierSearchTreeHead);

    delete token;
	delete identifierSearchTreeHead;
    fclose(source_file);
    return 0;
}