C++ Url::getHost方法代码示例

int wmtor::saveNewPage(Url& url,HttpContent &content,HttpHeader &header){

	struct stat st;
	if(stat(_filePath.c_str(),&st)!=0){
		cout<<"Creating directory "<<_filePath<<endl;
		if(mkdir(_filePath.c_str(),0755)== -1){
			cerr<<"Error:Cannot create directory"<<_filePath<<endl;
			return -1;
		}
	}
	string HostPath = _filePath+"/"+url.getHost();
	if(stat(HostPath.c_str(),&st)!=0){
		cout<<"Creating directory "<< HostPath <<endl;
		if(mkdir(HostPath.c_str(),0755)== -1){
			cerr<<"Error:Cannot create directory"<< HostPath <<endl;
			return -1;
		}
	}

	string filehead = url.getUrlStr();
	string::size_type pos= 0;
	while((pos=filehead.find('/',pos))!=string::npos)
		filehead[pos] = '_';
	string filename = HostPath + "/" + filehead + "_v" + CStrFun::itos(url.getUpdateCount());
	url.incUpdateCount();
	ofstream conFile(filename.c_str());
	if(conFile.is_open()){
		conFile<<content.getContentStr()<<endl;
		conFile.close();
	}else{
		cerr<<"Cannot open file: "<< filename <<endl;
		return -1;
	}
	return 0;
}

// returns false and sets g_errno on error
bool Title::setTitle ( Xml *xml, Words *words, int32_t maxTitleLen, Query *query,
                       LinkInfo *linkInfo, Url *firstUrl, const char *filteredRootTitleBuf, int32_t filteredRootTitleBufSize,
                       uint8_t contentType, uint8_t langId, int32_t niceness ) {
	// make Msg20.cpp faster if it is just has
	// Msg20Request::m_setForLinkInfo set to true, no need to extricate a title.
	if ( maxTitleLen <= 0 ) {
		return true;
	}

	m_niceness = niceness;
	m_maxTitleLen = maxTitleLen;

	// if this is too big the "first line" algo can be huge!!!
	// and really slow everything way down with a huge title candidate
	int32_t maxTitleWords = 128;

	// assume no title
	reset();

	int32_t NW = words->getNumWords();

	//
	// now get all the candidates
	//

	// . allow up to 100 title CANDIDATES
	// . "as" is the word # of the first word in the candidate
	// . "bs" is the word # of the last word IN the candidate PLUS ONE
	int32_t n = 0;
	int32_t as[MAX_TIT_CANDIDATES];
	int32_t bs[MAX_TIT_CANDIDATES];
	float scores[MAX_TIT_CANDIDATES];
	Words *cptrs[MAX_TIT_CANDIDATES];
	int32_t types[MAX_TIT_CANDIDATES];
	int32_t parent[MAX_TIT_CANDIDATES];

	// record the scoring algos effects
	float  baseScore        [MAX_TIT_CANDIDATES];
	float  noCapsBoost      [MAX_TIT_CANDIDATES];
	float  qtermsBoost      [MAX_TIT_CANDIDATES];
	float  inCommonCandBoost[MAX_TIT_CANDIDATES];

	// reset these
	for ( int32_t i = 0 ; i < MAX_TIT_CANDIDATES ; i++ ) {
		// assume no parent
		parent[i] = -1;
	}

	// xml and words class for each link info, rss item
	Xml   tx[MAX_TIT_CANDIDATES];
	Words tw[MAX_TIT_CANDIDATES];
	int32_t  ti = 0;

	// restrict how many link texts and rss blobs we check for titles
	// because title recs like www.google.com have hundreds and can
	// really slow things down to like 50ms for title generation
	int32_t kcount = 0;
	int32_t rcount = 0;

	//int64_t x = gettimeofdayInMilliseconds();

	// . get every link text
	// . TODO: repeat for linkInfo2, the imported link text
	for ( Inlink *k = NULL; linkInfo && (k = linkInfo->getNextInlink(k)) ; ) {
		// breathe
		QUICKPOLL(m_niceness);
		// fast skip check for link text
		if ( k->size_linkText >= 3 && ++kcount >= 20 ) continue;
		// fast skip check for rss item
		if ( k->size_rssItem > 10 && ++rcount >= 20 ) continue;

		// set Url
		Url u;
		u.set( k->getUrl(), k->size_urlBuf );

		// is it the same host as us?
		bool sh = true;

		// skip if not from same host and should be
		if ( firstUrl->getHostLen() != u.getHostLen() ) {
			sh = false;
		}

		// skip if not from same host and should be
		if ( strncmp( firstUrl->getHost(), u.getHost(), u.getHostLen() ) ) {
			sh = false;
		}

		// get the link text
		if ( k->size_linkText >= 3 ) {
			char *p    = k->getLinkText();
			int32_t  plen = k->size_linkText - 1;
			if ( ! verifyUtf8 ( p , plen ) ) {
				log("title: set4 bad link text from url=%s", k->getUrl());
				continue;
			}

			// now the words.
			if ( !tw[ti].set( k->getLinkText(), k->size_linkText - 1, true, 0 ) ) {
//.........这里部分代码省略.........

/*
  Forward the HTTP requesto to another server.

  \param td The HTTP thread context.
  \param scriptpath Not used.
  \param exec The remote server Url.
  \param execute Not used.
  \param onlyHeader Specify if send only the HTTP header.
 */
int Proxy::send (HttpThreadContext *td, const char* scriptpath,
                 const char* exec, bool execute, bool onlyHeader)
{
  Url destUrl (exec, 80);
  ConnectionPtr con = NULL;
  Socket *sock;
  FiltersChain chain;
  HttpRequestHeader req;
  size_t nbw;
  bool keepalive = false;

  for (HashMap<string, HttpRequestHeader::Entry*>::Iterator it =
         td->request.begin (); it != td->request.end (); it++)
    {
      HttpRequestHeader::Entry *e = *it;
      req.setValue (e->name.c_str (), e->value.c_str ());
    }

  if (stringcmpi (destUrl.getProtocol (), "http"))
    {
      td->connection->host->warningsLogWrite
        ("Proxy: %s is not a supported protocol",
         destUrl.getProtocol ().c_str ());
      return td->http->raiseHTTPError (500);
    }

  try
    {
      req.ver.assign ("HTTP/1.1");
      req.cmd.assign (td->request.cmd);

      if (destUrl.getResource ()[0] == '\0' && td->pathInfo[0] == '\0')
        req.uri = "/";
      else
        {
          req.uri = destUrl.getResource ();
          req.uri.append (td->pathInfo);
        }
      if (td->request.uriOpts.length ())
        {
          req.uri.append ("?");
          req.uri.append (td->request.uriOpts);
        }

      req.setValue ("Connection", "keep-alive");
      if (td->request.uriOptsPtr)
        {
          char buffer[32];
          size_t size = td->inputData.getFileSize ();
          sprintf (buffer, "%u", size);
          req.setValue ("Content-Length", buffer);
        }

      ostringstream host;
      host << destUrl.getHost ();
      if (destUrl.getPort () != 80 )
        host << ":" << destUrl.getPort ();

      req.setValue ("Host", host.str ().c_str ());

      string xForwardedFor;
      td->request.getValue ("X-Forwarded-For", &xForwardedFor);
      if (xForwardedFor.size ())
        xForwardedFor.append (", ");
      xForwardedFor.append (td->connection->getIpAddr ());
      req.setValue ("X-Forwarded-For", xForwardedFor.c_str ());

      con = getConnection (destUrl.getHost ().c_str (), destUrl.getPort ());
      if (! con)
        return td->http->raiseHTTPError (500);

      sock = con->socket;

      u_long hdrLen =
        HttpHeaders::buildHTTPRequestHeader (td->auxiliaryBuffer->getBuffer (),
                                             &req);


      sock->write (td->auxiliaryBuffer->getBuffer (), hdrLen, &nbw);

      if (td->request.uriOptsPtr)
        td->inputData.fastCopyToSocket (sock, 0, td->auxiliaryBuffer, &nbw);

      chain.setStream (td->connection->socket);
      if (td->mime)
        Server::getInstance ()->getFiltersFactory ()->chain (&chain,
                                                             td->mime->filters,
                                                             td->connection->socket,
                                                             &nbw,
                                                             1);

//.........这里部分代码省略.........

int HttpClient::requestContent(Url &url, string &contentStr, int expectContentLength) {
    // construct a request
    string requestStr;
    string path = url.getPath();
    if (path.empty())
        path = "/";
    requestStr = "GET " + path + " HTTP/1.0\r\nHost: " + url.getHost()
            + "\r\nUser-Agent: openSE/1.0 (Ubuntu11.04)\r\nAccept-Language: zh,en-us\r\nAccept-Charset: gb2312,utf-8\r\nConnection: Keep-Alive\r\n\r\n";

    cout << "requestStr:\n" << requestStr << endl;

    // send request:
    if (url.getHost() != _preHost) {
        if (_preSockFd != -1) {
            closesocket(_preSockFd);
            _preSockFd = -1;
        }
    }
    int sockFd;
    bool sendSuccess = false;

    // try to use previous connection
    if (_preSockFd != -1) {
        sockFd = _preSockFd;
        if (rio_writen(sockFd, requestStr.c_str(), requestStr.size()) == -1) {
            cerr << "use previous connection:rio_writen error !" << endl;
            closesocket(_preSockFd);
            _preSockFd = -1;
        } else
            sendSuccess = true;
    }

    if (!sendSuccess) {
        // try to creat a new connection
        sockFd = tcpConnect(url.getIp(), url.getPort());
        if (sockFd == -1) {
            cerr << "tcpConnect error" << endl;
            return -1;
        }
        // try to use new connection
        if (rio_writen(sockFd, requestStr.c_str(), requestStr.size()) == -1) {
            cerr << "rio_writen error for requestStr:" << requestStr << endl;
            closesocket(sockFd);
            return -1;
        }
    }

    //// receive http header
    //string headerStr;
    //headerStr.reserve(1024);
    //if(receiveHeader(sockFd, headerStr, DEFAULT_TIMEOUT_SECONDS) <= 0){
    //	cerr << "receiveHeader error" << endl;
    //	closesocket(sockFd);
    //	_preSockFd = -1;
    //	return -1;
    //	}

    //cout << "headerStr:\n" << headerStr << endl;

    //// parser http header
    //httpHeader.setHeaderStr(headerStr);

    //// check StatusCode
    //int stausCode = httpHeader.getStatusCode();
    //if(stausCode == -1){
    //	cerr << "not find status code in httpHeader: " << httpHeader.getHeaderStr() << endl;
    //	}

    //if(stausCode == 301 || stausCode == 302){
    //	closesocket(sockFd);
    //	_preSockFd = -1;
    //	string locationUrlStr = httpHeader.getLocation();
    //	if(locationUrlStr.empty()){
    //		cerr << "error location in httpHeader: " << httpHeader.getHeaderStr() << endl;
    //		}
    //	//locationStr = location;
    //	Url locationUrl(locationUrlStr);
    //	return requestWebPage(locationUrl, httpHeader, httpContent);
    //	}

    //if(stausCode < 200 || stausCode > 299){
    //	closesocket(sockFd);
    //	_preSockFd = -1;
    //	cerr << "status code beyond [200-300) in httpHeader: " << httpHeader.getHeaderStr() << endl;
    //	return -1;
    //	}

    //// check content type
    //string contentType = httpHeader.getContentType();
    //if(contentType.find("image") != string::npos){
    //	closesocket(sockFd);
    //	_preSockFd = -1;
    //	cerr << "contentType is image in httpHeader: " << httpHeader.getHeaderStr() << endl;
    //	return -1;
    //	}

    //// check ContentLength
    //int contentLength = httpHeader.getContentLength();

    //if(contentLength == -1){
//.........这里部分代码省略.........

int HttpClient::requestHeader(Url &url, string&headerStr) {
    // construct a request
    string requestStr;
    string path = url.getPath();
    if (path.empty())
        path = "/";
    requestStr = "HEAD " + path + " HTTP/1.0\r\nHost: " + url.getHost()
            + "\r\nUser-Agent: openSE/1.0 (Ubuntu11.04)\r\nAccept-Language: zh,en-us\r\nAccept-Charset: gb2312,utf-8\r\nConnection: Keep-Alive\r\n\r\n";

    cout << "requestStr:\n" << requestStr << endl;

    // send request:
    if (url.getHost() != _preHost) {
        if (_preSockFd != -1) {
            closesocket(_preSockFd);
            _preSockFd = -1;
        }
    }
    int sockFd;
    bool sendSuccess = false;

    // try to use previous connection
    if (_preSockFd != -1) {
        sockFd = _preSockFd;
        if (rio_writen(sockFd, requestStr.c_str(), requestStr.size()) == -1) {
            cerr << "use previous connection:rio_writen error !" << endl;
            closesocket(_preSockFd);
            _preSockFd = -1;
        } else
            sendSuccess = true;
    }

    if (!sendSuccess) {
        // try to creat a new connection
        sockFd = tcpConnect(url.getIp(), url.getPort());
        if (sockFd == -1) {
            cerr << "tcpConnect error" << endl;
            return -1;
        }
        // try to use new connection
        if (rio_writen(sockFd, requestStr.c_str(), requestStr.size()) == -1) {
            cerr << "rio_writen error for requestStr:" << requestStr << endl;
            closesocket(sockFd);
            return -1;
        }
    }


    headerStr.reserve(1024);
    int timeoutSeconds = DEFAULT_TIMEOUT_SECONDS;
    int bytesRead = 0;
    // set socket to no block type
#ifdef _MSC_VER

#else
    int flags;
    flags = fcntl(sockFd, F_GETFL, 0);
    if (flags < 0) {
        cerr << "1.fcntl() error in receiveHeader()< 0" << endl;
        return -1;
    }
    flags |= O_NONBLOCK;
    if (fcntl(sockFd, F_SETFL, flags) < 0) {
        cerr << "2.fcntl() error in receiveHeader" << endl;
        return -1;
    }
#endif

    int newlines = 0;
    while (newlines < 2) {
        fd_set rfds;
        FD_ZERO(&rfds);
        FD_SET(sockFd, &rfds);

        struct timeval tv;
        tv.tv_sec = timeoutSeconds;
        tv.tv_usec = 0;

        int selectRet;
        if (timeoutSeconds >= 0) // wait DEFAULT_TIMEOUT_SECONDS seconds
            selectRet = select(sockFd + 1, &rfds, NULL, NULL, &tv);
        else // wait infinitely
            selectRet = select(sockFd + 1, &rfds, NULL, NULL, NULL);

        if (selectRet == 0 && timeoutSeconds < 0) {
            cerr << "select should wait infinitely" << endl;
            bytesRead = -1;
            break;
        } else if (selectRet == -1) {
            cerr << "select error" << endl;
            bytesRead = -1;
            break;
        } else if (selectRet == 0) {
            cerr << "select timeout after " << timeoutSeconds << " seconds" << endl;
            bytesRead = -1;
            break;
        }

        char c;
        int ret = recv(sockFd, &c, 1, 0);
//.........这里部分代码省略.........

int main ( int argc , char *argv[] ) {
	bool addWWW = true;
	bool stripSession = true;
	// check for arguments
	for (int32_t i = 1; i < argc; i++) {
		if (strcmp(argv[i], "-w") == 0)
			addWWW = false;
		else if (strcmp(argv[i], "-s") == 0)
			stripSession = false;
	}
	// initialize
	//g_mem.init(100*1024);
	hashinit();
	//g_conf.m_tfndbExtBits = 23;
 loop:
	// read a url from stddin
	char sbuf[1024];
	if ( ! fgets ( sbuf , 1024 , stdin ) ) exit(1);
	char *s = sbuf;
	char fbuf[1024];
	// decode if we should
	if ( strncmp(s,"http%3A%2F%2F",13) == 0 ||
	     strncmp(s,"https%3A%2F%2F",13) == 0 ) {
		urlDecode(fbuf,s,gbstrlen(s));
		s = fbuf;
	}
	// old url
	printf("###############\n");
	printf("old: %s",s);
	int32_t slen = gbstrlen(s);
	// remove any www. if !addWWW
	if (!addWWW) {
		if (slen >= 4 &&
		    strncasecmp(s, "www.", 4) == 0) {
			slen -= 4;
			memmove(s, &s[4], slen);
		}
		else {
			// get past a ://
			int32_t si = 0;
			while (si < slen &&
			       ( s[si] != ':' ||
				 s[si+1] != '/' ||
				 s[si+2] != '/' ) )
				si++;
			// remove the www.
			if (si + 7 < slen) {
				si += 3;
				if (strncasecmp(&s[si], "www.", 4) == 0) {
					slen -= 4;
					memmove(&s[si], &s[si+4], slen-si);
				}
			}
		}
	}
	// set it
	Url u;
	u.set ( s , slen ,
		addWWW   ,      /*add www?*/
		stripSession ); /*strip session ids?*/
	// print it
	char out[1024*4];
	char *p = out;
	p += sprintf(p,"tld: ");
	gbmemcpy ( p, u.getTLD(),u.getTLDLen());
	p += u.getTLDLen();
	char c = *p;
	*p = '\0';
	printf("%s\n",out);
	*p = c;
	

	// dom
	p = out;
	sprintf ( p , "dom: ");
	p += gbstrlen ( p );
	gbmemcpy ( p , u.getDomain() , u.getDomainLen() );
	p += u.getDomainLen();
	c = *p;
	*p = '\0';
	printf("%s\n",out);
	*p = c;
	// host
	p = out;
	sprintf ( p , "host: ");
	p += gbstrlen ( p );
	gbmemcpy ( p , u.getHost() , u.getHostLen() );
	p += u.getHostLen();
	c = *p;
	*p = '\0';
	printf("%s\n",out);
	*p = c;
	// then the whole url
	printf("url: %s\n", u.getUrl() );

	/*
	int32_t  siteLen;
	char *site = u.getSite ( &siteLen , NULL , false );
	if ( site ) {
		c = site[siteLen];
//.........这里部分代码省略.........

bool HttpMime::addCookieHeader(const char *cookieJar, const char *url, SafeBuf *sb) {
	Url tmpUrl;
	tmpUrl.set(url);

	SafeBuf tmpSb;

	size_t cookieJarLen = strlen(cookieJar);

	const char *lineStartPos = cookieJar;
	const char *lineEndPos = NULL;
	while ((lineEndPos = (const char*)memchr(lineStartPos, '\n', cookieJarLen - (lineStartPos - cookieJar))) != NULL) {
		const char *currentPos = lineStartPos;
		const char *tabPos = NULL;
		unsigned fieldCount = 0;

		bool skipCookie = false;
		const char *domain = NULL;
		int32_t domainLen = 0;
		while (fieldCount < 5 && (tabPos = (const char*)memchr(currentPos, '\t', lineEndPos - currentPos)) != NULL) {
			switch (fieldCount) {
				case 0:
					// domain
					domain = currentPos;
					domainLen = tabPos - currentPos;
					break;
				case 1:
					// flag
					if (memcmp(currentPos, "TRUE", 4) == 0) {
						// allow subdomain
						if (tmpUrl.getHostLen() >= domainLen) {
							if (!endsWith(tmpUrl.getHost(), tmpUrl.getHostLen(), domain, domainLen)) {
								// doesn't end with domain - ignore cookie
								skipCookie = true;
								break;
							}
						} else {
							skipCookie = true;
							break;
						}
					} else {
						// only specific domain
						if (tmpUrl.getHostLen() != domainLen || strncasecmp(domain, tmpUrl.getHost(), domainLen) != 0) {
							// non-matching domain - ignore cookie
							skipCookie = true;
							break;
						}
					}
					break;
				case 2: {
					// path
					const char *path = currentPos;
					int32_t pathLen = tabPos - currentPos;
					if (strncasecmp(path, tmpUrl.getPath(), pathLen) == 0) {
						if (tmpUrl.getPathLen() != pathLen) {
							if (path[pathLen - 1] != '/' && tmpUrl.getPath()[tmpUrl.getPathLen() - 1] != '/') {
								// non-matching path - ignore cookie
								skipCookie = true;
								break;
							}
						}
					} else {
						// non-matching path - ignore cookie
						skipCookie = true;
						break;
					}
				} break;
				case 3:
					// secure

					break;
				case 4:
					// expiration

					break;
			}

			currentPos = tabPos + 1;
			++fieldCount;
		}

		if (!skipCookie) {
			tmpSb.safeMemcpy(currentPos, lineEndPos - currentPos);
			tmpSb.pushChar(';');
		}

		lineStartPos = lineEndPos + 1;
	}
	// we don't need to care about the last line (we always end on \n)

	if (tmpSb.length() > 0) {
		sb->safeStrcpy("Cookie: ");
		sb->safeMemcpy(&tmpSb);
		sb->safeStrcpy("\r\n");
	}

	return true;
}

void HttpMime::addCookie(const httpcookie_t &cookie, const Url &currentUrl, SafeBuf *cookieJar) {
	// don't add expired cookie into cookie jar
	if (cookie.m_expired) {
		return;
	}

	if (cookie.m_domain) {
		cookieJar->safeMemcpy(cookie.m_domain, cookie.m_domainLen);
		cookieJar->pushChar('\t');
		cookieJar->safeStrcpy(cookie.m_defaultDomain ? "FALSE\t" : "TRUE\t");
	} else {
		cookieJar->safeMemcpy(currentUrl.getHost(), currentUrl.getHostLen());
		cookieJar->pushChar('\t');

		cookieJar->safeStrcpy("FALSE\t");
	}

	if (cookie.m_path) {
		cookieJar->safeMemcpy(cookie.m_path, cookie.m_pathLen);
		cookieJar->pushChar('\t');
	} else {
		if (currentUrl.getPathLen()) {
			cookieJar->safeMemcpy(currentUrl.getPath(), currentUrl.getPathLen());
		} else {
			cookieJar->pushChar('/');
		}
		cookieJar->pushChar('\t');
	}

	if (cookie.m_secure) {
		cookieJar->safeStrcpy("TRUE\t");
	} else {
		cookieJar->safeStrcpy("FALSE\t");
	}

	// we're not using expiration field
	cookieJar->safeStrcpy("0\t");

	int32_t currentLen = cookieJar->length();
	cookieJar->safeMemcpy(cookie.m_cookie, cookie.m_cookieLen);

	// cater for multiline cookie
	const char *currentPos = cookieJar->getBufStart() + currentLen;
	const char *delPosStart = NULL;
	int32_t delLength = 0;
	while (currentPos < cookieJar->getBufPtr() - 1) {
		if (delPosStart) {
			if (is_wspace_a(*currentPos) || *currentPos == '\n' || *currentPos == '\r') {
				++delLength;
			} else {
				break;
			}
		} else {
			if (*currentPos == '\n' || *currentPos == '\r') {
				delPosStart = currentPos;
				++delLength;
			}
		}

		++currentPos;
	}
	cookieJar->removeChunk1(delPosStart, delLength);

	/// @todo ALC handle httpOnly attribute

	cookieJar->pushChar('\n');
}