C++ CUrl::ParseUrl方法代码示例

void CCrawl::Fetch(void *arg)
{
	string str_url,host;

	int nGsock = -1;//之前的套接字文件描述符
	string strGHost;//之前的主机号

	//生成一个PSE file来存放网页数据
	//string ofs_name = DATA_PSE_FILE + "." + CStrFunction::itos(GetCurrentThreadId());//PSE.raw+当前线程号

	string ofs_name = DATA_PSE_FILE + CStrFunction::itos(GetCurrentThreadId())+ ".txt";//PSE+当前线程号+.txt
	CPSEFile pse_file(ofs_name);//创建一个PSE格式的文件，保存为原始网页库

	//生成一个link_for_pse file来存放链接数据
	ofs_name = DATA_LINK_FOR_PSE_FILE  + CStrFunction::itos(GetCurrentThreadId())+ ".txt";//PSE+当前线程号+.txt
	CLinkForPSEFile link_for_pse_file(ofs_name);//创建一个网页结构库

	int isleep_cnt = 0;//线程运行控制参数

	for(;;)
	{
		WaitForSingleObject(mutex_collection,INFINITE);//互斥锁

		int cnt = map_urls.size();
		if(cnt > 0)
		{
			//已经收集的没有访问的url
			cout<<"collection has "<<cnt<<" unvisited urls"<<endl;
			multimap<string,string>::iterator it = map_urls.begin();
			if(it != map_urls.end())
			{
				//从带访问的url队列中得到一个url进行访问
				str_url = (*it).second;
				map_urls.erase(it);

				ReleaseMutex(mutex_collection);

				//分解url
				CUrl iurl;
				//看看url是否有http://，没有则返回

				if(iurl.ParseUrl(str_url) == false)
				{
					cout<<"parse url false in Fetch"<<str_url<<endl;
					continue;
				}

				//表明现在抓取的网页所在的主机，同之前抓取的网页所在的主机不同
				//我们不能利用之前的套接字文件描述符进行CS通信,必须创建新的
				//套接字文件描述符进行通信,这是由于循环导致的
				if(strGHost != iurl.host_name)
				{
					closesocket(nGsock);
					nGsock = -1;
					strGHost = iurl.host_name;
				}

				//根据URL以及套接字文件描述符抓取URL对应的网页,并保存为原始网页库和网页结构库
				((CCrawl *)arg)->DownroadFile(&pse_file,&link_for_pse_file,iurl,nGsock);

				cnt = 0;
			}else	
			{
				ReleaseMutex(mutex_collection);
				
			}
		}else
		{
			//等待访问的url队列map_urls已经没有url了，这是我们需要挂起线程进行等待
			ReleaseMutex(mutex_collection);
			Sleep(1000);
			isleep_cnt++;
		}

		if(b_f_over == true && isleep_cnt == 200)//当线程挂起的次数达到两百的时候，结束调用fetch
		{
			break;
		}
	}

	pse_file.Close();
	link_for_pse_file.Close();

}

//将url放入map_urls到容器中
void CCrawl::AddUrl(const char * url)
{
	string str_url = url;
	if(str_url.empty() || str_url.length() < 8)
	{
		cout<<"the url is empty or too short"<<endl;
		return ;
	}

	CPage ipage;
	if(ipage.NormalizeUrl(str_url) == false)
		return ;

	CUrl iurl;

	//图片类型的网页，存放到历史网页链接库中
	if(iurl.IsImageUrl(str_url))
	{
		if(ofs_link_for_history_file)
		{
			WaitForSingleObject(mutex_link_for_history_file,INFINITE);

			ofs_link_for_history_file<<str_url<<endl;

			ReleaseMutex(mutex_link_for_history_file);
		}

		return ;
	}

	if(iurl.ParseUrl(str_url) == false)
	{
		cout<<"parse url error in AddUrl"<<endl;
		return ;
	}

	if(iurl.IsValidHost(iurl.host_name.c_str()) == false)
	{
		cout<<"not the valid host in AddUrl"<<endl;
		return ;
	}

	if(iurl.IsForeignHost(iurl.host_name.c_str()) )
	{
		cout<<"foreign host in AddUrl"<<endl;
		return ;
	}

	//如果是阻塞的ip地址，剔除掉
	unsigned long inaddr = 0;
	char *ip = NULL;

	inaddr =(unsigned long) inet_addr(iurl.host_name.c_str());

	if(inaddr != INADDR_NONE)
	{
		ip = new char[iurl.host_name.size() + 1];
		memset(ip,0,iurl.host_name.size() + 1);
		memcpy(ip,iurl.host_name.c_str(),iurl.host_name.size());
		if(!iurl.IsValidIp(ip))
		{
			delete []ip;
			ip = NULL;
			return ;
		}

		delete []ip;
		ip = NULL;
	}


	CStrFunction::StrToLower(iurl.host_name,iurl.host_name.size());

	CMD5 imd5;
	imd5.GenerateMd5((unsigned char *)str_url.c_str(),str_url.size());

	string str_digest = imd5.ToString();

	if(set_visited_url_md5.find(str_digest) != set_visited_url_md5.end())
	{
		return ;
	}

	if(set_unvisited_url_md5.find(str_digest) != set_unvisited_url_md5.end())
	{
		return ;
	}
	else
	{
		WaitForSingleObject(mutex_unvisited_url_md5,INFINITE);
		set_unvisited_url_md5.insert(str_digest);
		ReleaseMutex(mutex_unvisited_url_md5);
	}

	//确保同一个线程在一个网站上爬取
	int cnt = 0;

	for(;;)
	{
//.........这里部分代码省略.........