C++ Slave::acceptLibrary方法代码示例

DWORD WINAPI Slave::SPEShuffler(LPVOID p)
#endif
{
   Slave* self = ((Param5*)p)->serv_instance;
   int transid = ((Param5*)p)->transid;
   string client_ip = ((Param5*)p)->client_ip;
   int client_port = ((Param5*)p)->client_ctrl_port;
   int client_data_port = ((Param5*)p)->client_data_port;
   string path = ((Param5*)p)->path;
   string localfile = ((Param5*)p)->filename;
   int bucketnum = ((Param5*)p)->bucketnum;
   CGMP* gmp = ((Param5*)p)->gmp;
   string function = ((Param5*)p)->function;
   int bucketid = ((Param5*)p)->bucketid;
   const int key = ((Param5*)p)->key;
   const int type = ((Param5*)p)->type;
   string master_ip = ((Param5*)p)->master_ip;
   int master_port = ((Param5*)p)->master_port;

   queue<Bucket>* bq = NULL;
   CMutex* bqlock = NULL;
   CCond* bqcond = NULL;
   int64_t* pendingSize = NULL;
   pthread_t shufflerex;

   bool init_success = true;

   //set up data connection, for keep-alive purpose
   if (self->m_DataChn.connect(client_ip, client_data_port) < 0)
   {
      init_success = false;
   }
   else
   {
      // read library files for MapReduce, no need for Sphere UDF
      if (type == 1)
         self->acceptLibrary(key, client_ip, client_data_port, transid);

      bq = new queue<Bucket>;
      bqlock = new CMutex;
      bqcond = new CCond;
      pendingSize = new int64_t;
      *pendingSize = 0;

      ((Param5*)p)->bq = bq;
      ((Param5*)p)->bqlock = bqlock;
      ((Param5*)p)->bqcond = bqcond;
      ((Param5*)p)->pending = pendingSize;

#ifndef WIN32
      pthread_create(&shufflerex, NULL, SPEShufflerEx, p);
#else
      DWORD ThreadID;
      shufflerex = CreateThread(NULL, 0, SPEShufflerEx, p, NULL, &ThreadID);
#endif

      self->m_SectorLog << LogStart(LogLevel::SCREEN) << "SPE Shuffler " << path << " " << localfile << " " << bucketnum << LogEnd();
   }

   while (init_success)
   {
      string speip;
      int speport;
      SectorMsg msg;
      int msgid;
      int r = gmp->recvfrom(speip, speport, msgid, &msg, false);

      // client releases the task or client has already been shutdown
      if (((r > 0) && (speip == client_ip) && (speport == client_port))
         || ((r < 0) && (!self->m_DataChn.isConnected(client_ip, client_data_port))))
      {
         Bucket b;
         b.totalnum = -1;
         b.totalsize = 0;
         bqlock->acquire();
         bq->push(b);
         bqcond->signal();
         bqlock->release();

         break;
      }

      if (r < 0)
         continue;

      if (*pendingSize > 256000000)
      {
         // too many incoming results, ask the sender to wait
         // the receiver buffer size threshold is set to 256MB. This prevents the shuffler from being overflowed
         // it also helps direct the traffic to less congested shuffler and leads to better load balance
         msg.setType(-msg.getType());
         gmp->sendto(speip, speport, msgid, &msg);
      }
      else
      {
         Bucket b;
         b.totalnum = *(int32_t*)(msg.getData() + 8);;
         b.totalsize = *(int32_t*)(msg.getData() + 12);
         b.src_ip = speip;
         b.src_dataport = *(int32_t*)msg.getData();
//.........这里部分代码省略.........

DWORD WINAPI Slave::SPEHandler(LPVOID p)
#endif
{
   Slave* self = ((Param4*)p)->serv_instance;
   const string ip = ((Param4*)p)->client_ip;
   const int ctrlport = ((Param4*)p)->client_ctrl_port;
   const int dataport = ((Param4*)p)->client_data_port;
   const int speid = ((Param4*)p)->speid;
   const int transid = ((Param4*)p)->transid;
   const int key = ((Param4*)p)->key;
   const string function = ((Param4*)p)->function;
   const int rows = ((Param4*)p)->rows;
   const char* param = ((Param4*)p)->param;
   const int psize = ((Param4*)p)->psize;
   const int type = ((Param4*)p)->type;
   const string master_ip = ((Param4*)p)->master_ip;
   const int master_port = ((Param4*)p)->master_port;
   delete (Param4*)p;

   SectorMsg msg;
   bool init_success = true;

   self->m_SectorLog << LogStart(LogLevel::LEVEL_3) << "SPE starts " << ip << " " << dataport << LogEnd();

   if (self->m_DataChn.connect(ip, dataport) < 0)
   {
      self->m_SectorLog << LogStart(LogLevel::LEVEL_2) << "failed to connect to spe client " << ip << ":" << ctrlport << " " << function << LogEnd();
      init_success = false;
   }

   self->m_SectorLog << LogStart(LogLevel::LEVEL_3) << "connected." << LogEnd();

   // read outupt parameters
   int buckets = 0;
   if (self->m_DataChn.recv4(ip, dataport, transid, buckets) < 0)
      init_success = false;

   SPEDestination dest;
   if (buckets > 0)
   {
      if (self->m_DataChn.recv4(ip, dataport, transid, dest.m_iLocNum) < 0)
         init_success = false;
      int len = dest.m_iLocNum * 80;
      if (self->m_DataChn.recv(ip, dataport, transid, dest.m_pcOutputLoc, len) < 0)
         init_success = false;
      len = buckets * 4;
      if (self->m_DataChn.recv(ip, dataport, transid, (char*&)dest.m_piLocID, len) < 0)
         init_success = false;
   }
   else if (buckets < 0)
   {
      int32_t len = 0;
      if (self->m_DataChn.recv(ip, dataport, transid, dest.m_pcOutputLoc, len) < 0)
         init_success = false;
      dest.m_strLocalFile = dest.m_pcOutputLoc;
   }
   dest.init(buckets);


   // initialize processing function
   self->acceptLibrary(key, ip, dataport, transid);
   SPHERE_PROCESS process = NULL;
   MR_MAP map = NULL;
   MR_PARTITION partition = NULL;
   void* lh = NULL;
   self->openLibrary(key, function, lh);
   if (NULL == lh)
   {
      self->m_SectorLog << LogStart(LogLevel::LEVEL_2) << "failed to open SPE library " << ip << ":" << ctrlport << " " << function << LogEnd();
      init_success = false;
   }

   if (type == 0)
   {
      if (self->getSphereFunc(lh, function, process) < 0)
         init_success = false;
   }
   else if (type == 1)
   {
      if (self->getMapFunc(lh, function, map, partition) < 0)
         init_success = false;
   }
   else
   {
      init_success = false;
   }

   timeval t1, t2, t3, t4;
   gettimeofday(&t1, 0);

   msg.setType(1); // success, return result
   msg.setData(0, (char*)&(speid), 4);

   SPEResult result;
   result.init(buckets);

   // processing...
   while (init_success)
   {
      char* dataseg = NULL;
//.........这里部分代码省略.........