C++ Package::ParseFromString方法代码示例

void get_map_1(vector<string> &mqueue, uint32_t num_nodes) {
        //map<uint32_t, NodeList> update_map;
        //uint32_t num_nodes = svrclient.memberList.size();
        for(vector<string>::iterator it = mqueue.begin(); it != mqueue.end(); ++it) {
                Package package;
                package.ParseFromString(*it); //cout << "key = " << package.virtualpath() << endl;
                uint32_t serverid = myhash(package.virtualpath().c_str(), num_nodes);
                string str(*it); str.append("#");
                if(update_map_zht.find(serverid) == update_map_zht.end()) {
                        str.append("$");
                        NodeList new_list;
                        new_list.push_back(str);
                        update_map_zht.insert(make_pair(serverid, new_list));
                }
                else {
                        NodeList &exist_list = update_map_zht[serverid];
                        string last_str(exist_list.back());
                        if((last_str.size() + str.size()) > STRING_THRESHOLD) {
                                str.append("$");
                                exist_list.push_back(str);
                        }
                        else {
                                exist_list.pop_back();
                                str.append(last_str);
                                exist_list.push_back(str);
                        }
                }
        }
        //return update_map;
}

int Worker::zht_insert(string str) {
	Package package;
	package.ParseFromString(str);
	package.set_operation(3);
	str = package.SerializeAsString();

	int index;
	svrclient.str2Host(str, index);

	if (index != selfIndex) { //cout << "NOOOOO..index = " << index << endl;
		pthread_mutex_lock(&msg_lock);
		int ret = svrclient.insert(str);
		pthread_mutex_unlock(&msg_lock);
		return ret;
	} else {
		string key = package.virtualpath(); //cout << "key = " << key << endl;
		//pthread_mutex_lock(&zht_lock);
		int ret = pmap->put(key, str);
		//pthread_mutex_unlock(&zht_lock);
		if (ret != 0) {
			cerr << "insert error: key = " << key << " ret = " << ret << endl;
			return -3;
		} else
			return 0;
	}
}

void* notQueue(void* args) {
	Worker *worker = (Worker*) args;
	string *st;
	Package package;

	while (ON) {
		while (notifyq.size() > 0) {
			pthread_mutex_lock(&notq_lock);
			if (notifyq.size() > 0) {
				try {
					st = notifyq.front();
					notifyq.pop();
				} catch (exception& e) {
					cout << "void* notifyq: " << e.what() << endl;
					exit(1);
				}
			} else {
				pthread_mutex_unlock(&notq_lock);
				continue;
			}
			pthread_mutex_unlock(&notq_lock);
			package.ParseFromString(*st);
			delete st;
			worker->update(package);
		}
	}
}

int benchmarkInsert() {

    double start = 0;
    double end = 0;
    start = TimeUtil::getTime_msec();
    int errCount = 0;

    int c = 0;
    vector<string>::iterator it;
    for (it = pkgList.begin(); it != pkgList.end(); it++) {

        c++;

        string pkg_str = *it;
        Package pkg;
        pkg.ParseFromString(pkg_str);

        int ret = zc.insert(pkg.virtualpath(), pkg_str);

        if (ret < 0) {
            errCount++;
        }
    }

    end = TimeUtil::getTime_msec();

    char buf[200];
    sprintf(buf, "Inserted packages, %d, %d, cost(ms), %f", numOfOps - errCount,
            numOfOps, end - start);
    cout << buf << endl;

    return 0;
}

//transfer a key to a host index where it should go
struct HostEntity ZHTClient::str2Host(string str) {
	Package pkg;
	pkg.ParseFromString(str);
	int index = myhash(pkg.virtualpath().c_str(), this->memberList.size());
	struct HostEntity host = this->memberList.at(index);

	return host;
}

int benchmarkAppend() {

    vector<string> pkgList_append;

    vector<string>::iterator it;
    for (it = pkgList.begin(); it != pkgList.end(); it++) {

        Package package;
        package.ParseFromString((*it));

        package.add_listitem("item-----6-append");

        pkgList_append.push_back(package.SerializeAsString());
    }

    double start = 0;
    double end = 0;
    start = TimeUtil::getTime_msec();
    int errCount = 0;

    int c = 0;
    for (it = pkgList_append.begin(); it != pkgList_append.end(); it++) {

        c++;

        string pkg_str = *it;
        Package pkg;
        pkg.ParseFromString(pkg_str);

        int ret = zc.append(pkg.virtualpath(), pkg_str);

        if (ret < 0) {
            errCount++;
        }
    }

    end = TimeUtil::getTime_msec();

    char buf[200];
    sprintf(buf, "Appended packages, %d, %d, cost(ms), %f", numOfOps - errCount,
            numOfOps, end - start);
    cout << buf << endl;

    return 0;
}

struct HostEntity str2Host(string str, vector<struct HostEntity> memberList,
		int &index) {
	Package pkg;
	pkg.ParseFromString(str);
	int index_inner = myhash(pkg.virtualpath().c_str(), memberList.size());
	struct HostEntity host = memberList.at(index_inner);
	index = index_inner;
	return host;
}

void submittaskszht(ZHTClient &clientRet, string client_id)
{
	int ret = 0;
	Package package;
	package.set_virtualpath(client_id);
	package.set_operation(21);
	cout << "Ok, what is the hell!\n" << endl;

	for (map<uint32_t, NodeList>::iterator map_it = update_map_zht.begin(); map_it != update_map_zht.end(); ++map_it)
	{
		uint32_t index = map_it->first;
		NodeList &update_list = map_it->second;
		while (!update_list.empty())
		{
			int num_vector_count, per_vector_count;
			string alltasks;
			vector< vector<string> > tokenize_string = tokenize_1(update_list.front(), '$', '#', num_vector_count, per_vector_count);
			//cout << " num_vector_count = " << num_vector_count << " per_vector_count = " << per_vector_count << endl;
			for(int i = 0; i < per_vector_count; i++)
			{
				Package tmpPackage;
				tmpPackage.ParseFromString(tokenize_string.at(0).at(i));
				string key = tmpPackage.virtualpath();
				alltasks.append(key); alltasks.append("\'\"");
			} //cout << "Serve
			package.set_realfullpath(alltasks);
			string str = package.SerializeAsString();
			ret += clientRet.svrtosvr(str, str.length(), index);
			update_list.pop_front();
		}
	}

//	Package package; string alltasks;
//			package.set_virtualpath(client_id); // Here key is just the client ID
//			package.set_operation(21);
//			//package.set_operation(22);
//			num_packages++;
//
//			int num_tasks_this_package = max_tasks_per_package;
//			int num_tasks_left = num_tasks - total_submitted1;
//			if (num_tasks_left < max_tasks_per_package) {
//				num_tasks_this_package = num_tasks_left;
//			}
//			for(int j = 0; j < num_tasks_this_package; j++) {
//				int task_id = it->first; ++it;
//
//	        	        stringstream task_id_ss;
//	                	task_id_ss << task_id << client_id; // Task ID + Client ID
//				alltasks.append(task_id_ss.str()); alltasks.append("\'\""); // Task ID
//			}
//			total_submitted1 = total_submitted1 + num_tasks_this_package;
//			package.set_realfullpath(alltasks);
//			string str = package.SerializeAsString();
//			//cout << "String size = " << str.size() << " str length = " << strlen(str.c_str());
//			int32_t ret = clientRet.svrtosvr(str, str.length(), toserver);
}

int ZHTClient::remove(string str) {

    Package package;
    package.ParseFromString(str);

    char *c_str;
    int size = str.length();
    c_str = (char*) malloc((size + 5 + 1) * sizeof(char));
    if (c_str == NULL) {
        cout << "ZHTClient::svrtosvr: " << strerror(errno) << endl;
        exit(1);
    }
    int len = copystring(c_str, str);

    if (package.virtualpath().empty()) //empty key not allowed.
        return -1;
    /*if (package.realfullpath().empty()) //coup, to fix ridiculous bug of protobuf!
     package.set_realfullpath(" ");*/

    package.set_operation(2); //1 for look up, 2 for remove, 3 for insert
    package.set_replicano(3); //5: original, 3 not original
    str = package.SerializeAsString();

    int sock = this->str2SockLRU(str, TCP);
    reuseSock(sock);
//	cout<<"sock = "<<sock<<endl;
    struct HostEntity dest = this->str2Host(str);
    sockaddr_in recvAddr;
    //pthread_mutex_lock(&msg_lock);
    int sentSize = generalSendTo(dest.host.data(), dest.port, sock, c_str, len,
                                 TCP);
    //pthread_mutex_unlock(&msg_lock);
    //int sentSize = generalSendTo(dest.host.data(), dest.port, sock, str.c_str(), str.size(), TCP);
//		cout<<"remove sentSize "<< sentSize <<endl;
    int32_t* ret_buf = (int32_t*) malloc(sizeof(int32_t));

    //	generalReveiveTCP(sock, (void*) ret_buf, sizeof(int32_t), 0);
    //pthread_mutex_lock(&msg_lock);
    generalReceive(sock, (void*) ret_buf, sizeof(int32_t), recvAddr, 0, TCP);
    //pthread_mutex_unlock(&msg_lock);

//	generalSendTCP(sock, str.c_str());

//	int32_t* ret = (int32_t*) malloc(sizeof(int32_t));

//	generalReveiveTCP(sock, (void*) ret, sizeof(int32_t), 0);
    int ret_1 = *(int32_t*) ret_buf;
//	cout<<"remove got: "<< ret_1 <<endl;
//cout <<"Returned status: "<< *(int32_t*) ret<<endl;
//	d3_closeConnection(sock);
    free(ret_buf);

    return ret_1;
}

// general server to server send (for load information and task stealing)
int32_t ZHTClient::svrtosvr(string str, int size, int index) {

    Package package;
    package.ParseFromString(str);

    if (package.virtualpath().empty()) 	//empty key not allowed.
    {
        return -1;
    }
    str = package.SerializeAsString();

    char *c_str;
    c_str = (char*) malloc((size + 5 + 1) * sizeof(char));
    if (c_str == NULL) {
        cout << "ZHTClient::svrtosvr: " << strerror(errno) << endl;
        exit(1);
    }
    int len = copystring(c_str, str);

    if (package.virtualpath().empty()) //empty key not allowed.
    {
        return -1;
    }
    //cout << "C++ string len = " << str.length() << endl; cout << "C++ string: \n" << str << endl;
    /*cout << "C string: " << endl;
     for(int i = 0; i < len; i++) cout << c_str[i]; cout << endl;
     return 1;*/

    int sock = this->index2SockLRU(index, TCP);
    reuseSock(sock);

    struct HostEntity dest = this->index2Host(index);
    sockaddr_in recvAddr; //cout << "going to acquire lock" << endl;
    //pthread_mutex_lock(&msg_lock); //cout << "lock acquired" << endl;
    //int sentSize = generalSendTo(dest.host.data(), dest.port, sock, str.c_str(), str.size(), TCP);
    //cout << "what is the fuck!" << dest.host.data() << endl;
    int sentSize = generalSendTo(dest.host.data(), dest.port, sock, c_str, len,
                                 TCP); //cout << "svrsvr sent" << endl;
    //pthread_mutex_unlock(&msg_lock);
    int32_t* ret_buf = (int32_t*) malloc(sizeof(int32_t));
    //pthread_mutex_lock(&msg_lock);
    generalReceive(sock, (void*) ret_buf, sizeof(int32_t), recvAddr, 0, TCP); //cout << "svrsvr recv" << endl;
    //pthread_mutex_unlock(&msg_lock); //cout << "lock released" << endl;
    int32_t ret_1 = *(int32_t*) ret_buf;
    free(ret_buf);

    //if(package.operation() == 14) {
    //	cout << "ZHTClient::svrtosvr num_task = " << ret_1 << endl;
    //}

    return ret_1;
}

//send a plain string to destination, receive return state.
int ZHTClient::insert(string str) {

	/*	int sock = -1;

	 struct HostEntity dest = this->str2Host(str);

	 //	cout<<"Client: will send to "<<dest.host<<endl;
	 //int ret = simpleSend(str, dest, sock);

	 sock = makeClientSocket(dest.host.c_str(), dest.port, 1);
	 //	cout<<"client sock = "<< sock<<endl;
	 reuseSock(sock);
	 generalSendTCP(sock, str.c_str());
	 */
	Package package;
	package.ParseFromString(str);

	if (package.virtualpath().empty()) //empty key not allowed.
		return -1;
	if (package.realfullpath().empty()) //coup, to fix ridiculous bug of protobuf!
		package.set_realfullpath(" ");

	package.set_operation(3); //1 for look up, 2 for remove, 3 for insert
	package.set_replicano(5); //5: original, 3 not original
	str = package.SerializeAsString();

	int sock = this->str2SockLRU(str, TCP);
	reuseSock(sock);
//	cout<<"sock = "<<sock<<endl;
//	int sentSize = generalSendTCP(sock, str.c_str());
	struct HostEntity dest = this->str2Host(str);
	sockaddr_in recvAddr;
	int sentSize = generalSendTo(dest.host.data(), dest.port, sock, str.c_str(),
			TCP);
//	cout <<"Client inseret sent: "<<sentSize<<endl;
	int32_t* ret_buf = (int32_t*) malloc(sizeof(int32_t));

//	generalReveiveTCP(sock, (void*) ret_buf, sizeof(int32_t), 0);
	generalReceive(sock, (void*) ret_buf, 4, recvAddr, 0, TCP);
	int ret = *(int32_t*) ret_buf;
	if (ret < 0) {
//		cerr << "zht_util.h: Failed to insert." << endl;
	}
//cout <<"Returned status: "<< *(int32_t*) ret<<endl;
//	d3_closeConnection(sock);
	free(ret_buf);

	//return ret_1;
//	cout<<"insert got: "<< *ret <<endl;
	return ret;
}

//DEPRECATED
struct metadata* zht_unparse_meta(const char* text) {

	Package package;
	package.ParseFromString(text);
	
	struct metadata* meta = (struct metadata*)malloc(sizeof(struct metadata));
	meta->loc = (struct comLocations *) malloc(sizeof(struct comLocations));
	
	meta->key = package.virtualpath().c_str();
	meta->k = package.k();
	meta->m = package.m();
	meta->bufsize = package.bufsize();
	meta->fileSize = package.filesize();
	meta->encodingLib = package.encodinglib();
	
	return meta;
}

int Worker::zht_ins_mul(Package &package) {
	int num_vector_count, per_vector_count;
	vector<vector<string> > tokenize_string = tokenize(package.realfullpath(),
			'$', '#', num_vector_count, per_vector_count);
	//cout << " num_vector_count = " << num_vector_count << " per_vector_count = " << per_vector_count << endl;
	for (int i = 0; i < per_vector_count; i++) {
		zht_insert(tokenize_string.at(0).at(i));
	} //cout << "Server: " << selfIndex << " no. tasks inserted = " << per_vector_count << endl;
	TaskQueue_Item *qi;
	pthread_mutex_lock(&w_lock);
	for (int i = 0; i < per_vector_count; i++) {
		Package pkg;
		pkg.ParseFromString(tokenize_string.at(0).at(i));
		qi = new TaskQueue_Item();
		qi->task_id = pkg.virtualpath();
		(&wqueue)->push_back(qi);
	}
	pthread_mutex_unlock(&w_lock);
	return per_vector_count;
}

int32_t ZHTClient::send(string str, int size) {

    Package package;
    package.ParseFromString(str);

    char *c_str;
    c_str = (char*) malloc((size + 5 + 1) * sizeof(char));
    if (c_str == NULL) {
        cout << "ZHTClient::send: " << strerror(errno) << endl;
        exit(1);
    }
    int len = copystring(c_str, str);

    if (package.virtualpath().empty()) //empty key not allowed.
        return -1;
    //cout << "C++ string len = " << str.length() << endl; cout << "C++ string: \n" << str << endl;
    /*cout << "C string: " << endl;
     for(int i = 0; i < len; i++) cout << c_str[i]; cout << endl;
     return 1;*/

    int sock = this->str2SockLRU(str, TCP);
    reuseSock(sock);
    struct HostEntity dest = this->str2Host(str);
    sockaddr_in recvAddr;

    //pthread_mutex_lock(&msg_lock);
    //int sentSize = generalSendTo(dest.host.data(), dest.port, sock, str.c_str(), str.size(), TCP);
    int sentSize = generalSendTo(dest.host.data(), dest.port, sock, c_str, len,
                                 TCP);
    //pthread_mutex_unlock(&msg_lock);

    //cout << " Sent size = " << sentSize << endl;
    int32_t* ret_buf = (int32_t*) malloc(sizeof(int32_t));
    //pthread_mutex_lock(&msg_lock);
    generalReceive(sock, (void*) ret_buf, sizeof(int32_t), recvAddr, 0, TCP);
    //pthread_mutex_unlock(&msg_lock);
    int ret_1 = *(int32_t*) ret_buf;
    free(ret_buf);
    return ret_1;
}

float benchmarkRemove(vector<string> strList, ZHTClient &client) {
    vector<string>::iterator it;

    for (it = strList.begin(); it != strList.end(); it++) {
        Package package;
        package.ParseFromString((*it));
        package.set_operation(2); // 3 for insert, 1 for look up, 2 for remove
        package.set_replicano(5); //5: original, 3 not original

        strList.erase(it);
        string newStr = package.SerializeAsString();
        strList.push_back(newStr);
    }

    double start = 0;
    double end = 0;
    start = getTime_msec();
    int errCount = 0;
    int c=0;
    for (it = strList.begin(); it != strList.end(); it++) {
        string result;
        c++;
        //cout <<"Remove count "<< c << endl;

        //cout <<"Remove: "<< (*it).c_str() << endl;
        if (client.remove((*it)) < 0) {
            errCount++;
        }

    }

    end = getTime_msec();

    cout << "Remove " << strList.size() - errCount << " packages out of "
         << strList.size() << ", cost " << end - start << " ms" << endl;
    return 0;

    return 0;
}