C++ Slave::createDir方法代码示例

DWORD WINAPI Slave::copy(LPVOID p)
#endif
{
   Slave* self = ((Param3*)p)->serv_instance;
   int transid = ((Param3*)p)->transid;
   int dir = ((Param3*)p)->dir;
   string src = ((Param3*)p)->src;
   string dst = ((Param3*)p)->dst;
   string master_ip = ((Param3*)p)->master_ip;
   int master_port = ((Param3*)p)->master_port;
   delete (Param3*)p;

   if (src.c_str()[0] == '\0')
      src = "/" + src;
   if (dst.c_str()[0] == '\0')
      dst = "/" + dst;

   bool success = true;

   queue<string> tr;	// files to be replicated
   queue<string> td;	// directories to be explored

   if (dir > 0)
      td.push(src);
   else
      tr.push(src);

   while (!td.empty())
   {
      // If the file to be replicated is a directory, recursively list all files first

      string src_path = td.front();
      td.pop();

      // try list this path
      SectorMsg msg;
      msg.setType(101);
      msg.setKey(0);
      msg.setData(0, src_path.c_str(), src_path.length() + 1);

      Address addr;
      self->m_Routing.lookup(src_path, addr);

      if (self->m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0)
      {
         success = false;
         break;
      }

      // the master only returns positive if this is a directory
      if (msg.getType() >= 0)
      {
         // if this is a directory, create it, and put all files and sub-directories into the queue of files to be copied

         // create a local dir
         string dst_path = dst;
         if (src != src_path)
            dst_path += "/" + src_path.substr(src.length() + 1, src_path.length() - src.length() - 1);

         //create at .tmp first, then move to real location
         self->createDir(string(".tmp") + dst_path);

         string filelist = msg.getData();
         unsigned int s = 0;
         while (s < filelist.length())
         {
            int t = filelist.find(';', s);
            SNode sn;
            sn.deserialize(filelist.substr(s, t - s).c_str());
            if (sn.m_bIsDir)
               td.push(src_path + "/" + sn.m_strName);
            else
               tr.push(src_path + "/" + sn.m_strName);
            s = t + 1;
         }

         continue;
      }
   }

   while (!tr.empty())
   {
      string src_path = tr.front();
      tr.pop();

      SNode tmp;
      if (self->m_pLocalFile->lookup(src_path.c_str(), tmp) >= 0)
      {
         //if file is local, copy directly
         //note that in this case, src != dst, therefore this is a regular "cp" command, not a system replication

         //IMPORTANT!!!
         //local files must be read directly from local disk, and cannot be read via datachn due to its limitation

         string dst_path = dst;
         if (src != src_path)
            dst_path += "/" + src_path.substr(src.length() + 1, src_path.length() - src.length() - 1);

         //copy to .tmp first, then move to real location
         self->createDir(string(".tmp") + dst_path.substr(0, dst_path.rfind('/')));
//.........这里部分代码省略.........

DWORD WINAPI Slave::fileHandler(LPVOID p)
#endif
{
   Slave* self = ((Param2*)p)->serv_instance;
   string filename = self->m_strHomeDir + ((Param2*)p)->filename;
   string sname = ((Param2*)p)->filename;
   int key = ((Param2*)p)->key;
   int mode = ((Param2*)p)->mode;
   int transid = ((Param2*)p)->transid;
   string client_ip = ((Param2*)p)->client_ip;
   int client_port = ((Param2*)p)->client_port;
   unsigned char crypto_key[16];
   unsigned char crypto_iv[8];
   memcpy(crypto_key, ((Param2*)p)->crypto_key, 16);
   memcpy(crypto_iv, ((Param2*)p)->crypto_iv, 8);
   string master_ip = ((Param2*)p)->master_ip;
   int master_port = ((Param2*)p)->master_port;
   delete (Param2*)p;

   // uplink and downlink addresses for write, no need for read
   string src_ip = client_ip;
   int src_port = client_port;
   string dst_ip;
   int dst_port = -1;

   // IO permissions
   bool bRead = mode & 1;
   bool bWrite = mode & 2;
   bool trunc = mode & 4;
   bool bSecure = mode & 16;

   bool m_bChange = false;

   int last_timestamp = 0;

   self->m_SectorLog << LogStart(LogLevel::LEVEL_3) << "connecting to " << client_ip << " " << client_port << " " << filename << LogEnd();

   if ((!self->m_DataChn.isConnected(client_ip, client_port)) && (self->m_DataChn.connect(client_ip, client_port) < 0))
   {
      self->m_SectorLog << LogStart(LogLevel::LEVEL_2) << "failed to connect to file client " << client_ip << " " << client_port << " " << filename << LogEnd();

      // release transactions and file locks
      self->m_TransManager.updateSlave(transid, self->m_iSlaveID);
      self->m_pLocalFile->unlock(sname, key, mode);
      self->report(master_ip, master_port, transid, sname, +FileChangeType::FILE_UPDATE_NO);

      return NULL;
   }

   Crypto* encoder = NULL;
   Crypto* decoder = NULL;
   if (bSecure)
   {
      encoder = new Crypto;
      encoder->initEnc(crypto_key, crypto_iv);
      decoder = new Crypto;
      decoder->initDec(crypto_key, crypto_iv);      
   }

   //create a new directory or file in case it does not exist
   if (bWrite)
   {
      self->createDir(sname.substr(0, sname.rfind('/')));

      SNode s;
      if (LocalFS::stat(filename, s) < 0)
      {
         ofstream newfile(filename.c_str(), ios::out | ios::binary | ios::trunc);
         newfile.close();
      }
   }

   timeval t1, t2;
   gettimeofday(&t1, 0);
   int64_t rb = 0;
   int64_t wb = 0;

   WriteLog writelog;

   fstream fhandle;
   if (!trunc)
      fhandle.open(filename.c_str(), ios::in | ios::out | ios::binary);
   else
      fhandle.open(filename.c_str(), ios::in | ios::out | ios::binary | ios::trunc);

   // a file session is successful only if the client issue a close() request
   bool success = true;
   bool run = true;
   int32_t cmd = 0;

   while (run)
   {
      if (self->m_DataChn.recv4(client_ip, client_port, transid, cmd) < 0)
         break;

      switch (cmd)
      {
      case 1: // read
         {
            char* param = NULL;
//.........这里部分代码省略.........

DWORD WINAPI Slave::SPEShufflerEx(LPVOID p)
#endif
{
   Slave* self = ((Param5*)p)->serv_instance;
   int transid = ((Param5*)p)->transid;
   string path = ((Param5*)p)->path;
   string localfile = ((Param5*)p)->filename;
   int bucketnum = ((Param5*)p)->bucketnum;
   const int key = ((Param5*)p)->key;
   const int type = ((Param5*)p)->type;
   string function = ((Param5*)p)->function;
   string master_ip = ((Param5*)p)->master_ip;
   int master_port = ((Param5*)p)->master_port;
   queue<Bucket>* bq = ((Param5*)p)->bq;
   CMutex* bqlock = ((Param5*)p)->bqlock;
   CCond* bqcond = ((Param5*)p)->bqcond;
   int64_t* pendingSize = ((Param5*)p)->pending;
   delete (Param5*)p;

   self->createDir(path);

   // remove old result data files
   for (int i = 0; i < bucketnum; ++ i)
   {
      int size = self->m_strHomeDir.length() + path.length() + localfile.length() + 64;
      char* tmp = new char[size];
      snprintf(tmp, size, "%s.%d", (self->m_strHomeDir + path + "/" + localfile).c_str(), i);
      LocalFS::erase(tmp);
      snprintf(tmp, size, "%s.%d.idx", (self->m_strHomeDir + path + "/" + localfile).c_str(), i);
      LocalFS::erase(tmp);
      delete [] tmp;
   }

   // index file initial offset
   vector<int64_t> offset;
   offset.resize(bucketnum);
   for (vector<int64_t>::iterator i = offset.begin(); i != offset.end(); ++ i)
      *i = 0;
   set<int> fileid;

   while (true)
   {
      bqlock->acquire();
      while (bq->empty())
         bqcond->wait(*bqlock);
      Bucket b = bq->front();
      bq->pop();
      *pendingSize -= b.totalsize;
      bqlock->release();

      if (b.totalnum == -1)
         break;

      string speip = b.src_ip;
      int dataport = b.src_dataport;
      int session = b.session;

      for (int i = 0; i < b.totalnum; ++ i)
      {
         int bucket = 0;
         if (self->m_DataChn.recv4(speip, dataport, session, bucket) < 0)
            continue;

         fileid.insert(bucket);

         char* tmp = new char[self->m_strHomeDir.length() + path.length() + localfile.length() + 64];
         sprintf(tmp, "%s.%d", (self->m_strHomeDir + path + "/" + localfile).c_str(), bucket);
         fstream datafile(tmp, ios::out | ios::binary | ios::app);
         sprintf(tmp, "%s.%d.idx", (self->m_strHomeDir + path + "/" + localfile).c_str(), bucket);
         fstream indexfile(tmp, ios::out | ios::binary | ios::app);
         delete [] tmp;
         int64_t start = offset[bucket];
         if (0 == start)
            indexfile.write((char*)&start, 8);

         int32_t len;
         char* data = NULL;
         if (self->m_DataChn.recv(speip, dataport, session, data, len) < 0)
            continue;
         datafile.write(data, len);
         delete [] data;

         tmp = NULL;
         if (self->m_DataChn.recv(speip, dataport, session, tmp, len) < 0)
            continue;
         int64_t* index = (int64_t*)tmp;
         for (int j = 0; j < len / 8; ++ j)
            index[j] += start;
         offset[bucket] = index[len / 8 - 1];
         indexfile.write(tmp, len);
         delete [] tmp;

         datafile.close();
         indexfile.close();
      }

      // update total received data
      self->m_SlaveStat.updateIO(speip, b.totalsize, +SlaveStat::SYS_IN);
   }

//.........这里部分代码省略.........

void* Slave::copy(void* p)
{
   Slave* self = ((Param3*)p)->serv_instance;
   int transid = ((Param3*)p)->transid;
   string src = ((Param3*)p)->src;
   string dst = ((Param3*)p)->dst;
   string master_ip = ((Param3*)p)->master_ip;
   int master_port = ((Param3*)p)->master_port;
   delete (Param3*)p;

   if (src.c_str()[0] == '\0')
      src = "/" + src;
   if (dst.c_str()[0] == '\0')
      dst = "/" + dst;

   SNode tmp;
   if (self->m_pLocalFile->lookup(src.c_str(), tmp) >= 0)
   {
      //if file is local, copy directly
      //note that in this case, src != dst, therefore this is a regular "cp" command, not a system replication
      //TODO: check disk space

      self->createDir(dst.substr(0, dst.rfind('/')));
      string rhome = self->reviseSysCmdPath(self->m_strHomeDir);
      string rsrc = self->reviseSysCmdPath(src);
      string rdst = self->reviseSysCmdPath(dst);
      system(("cp " + rhome + rsrc + " " + rhome + rdst).c_str());

      // if the file has been modified during the replication, remove this replica
      int type = (src == dst) ? +FileChangeType::FILE_UPDATE_REPLICA : +FileChangeType::FILE_UPDATE_NEW;

      struct stat64 s;
      if (stat64((self->m_strHomeDir + dst).c_str(), &s) < 0)
         type = +FileChangeType::FILE_UPDATE_NO;

      if (self->report(master_ip, master_port, transid, dst, type) < 0)
         system(("rm " + rhome + rdst).c_str());

      // clear this transaction
      self->m_TransManager.updateSlave(transid, self->m_iSlaveID);

      return NULL;
   }

   bool success = true;

   queue<string> tr;
   tr.push(src);

   while (!tr.empty())
   {
      string src_path = tr.front();
      tr.pop();

      // try list this path
      SectorMsg msg;
      msg.setType(101);
      msg.setKey(0);
      msg.setData(0, src_path.c_str(), src_path.length() + 1);

      Address addr;
      self->m_Routing.lookup(src_path, addr);

      if (self->m_GMP.rpc(addr.m_strIP.c_str(), addr.m_iPort, &msg, &msg) < 0)
      {
         success = false;
         break;
      }

      if (msg.getType() >= 0)
      {
         // if this is a directory, put all files and sub-drectories into the queue of files to be copied

         string filelist = msg.getData();
         unsigned int s = 0;
         while (s < filelist.length())
         {
            int t = filelist.find(';', s);
            SNode sn;
            sn.deserialize(filelist.substr(s, t - s).c_str());
            tr.push(src_path + "/" + sn.m_strName);
            s = t + 1;
         }

         continue;
      }

      // open the file and copy it to local
      msg.setType(110);
      msg.setKey(0);

      int32_t mode = SF_MODE::READ;
      msg.setData(0, (char*)&mode, 4);
      int64_t reserve = 0;
      msg.setData(4, (char*)&reserve, 8);
      int32_t localport = self->m_DataChn.getPort();
      msg.setData(12, (char*)&localport, 4);
      msg.setData(16, "\0", 1);
      msg.setData(80, src_path.c_str(), src_path.length() + 1);

//.........这里部分代码省略.........

void* Slave::SPEShufflerEx(void* p)
{
   Slave* self = ((Param5*)p)->serv_instance;
   int transid = ((Param5*)p)->transid;
   string client_ip = ((Param5*)p)->client_ip;
   int client_port = ((Param5*)p)->client_ctrl_port;
   int client_data_port = ((Param5*)p)->client_data_port;
   string path = ((Param5*)p)->path;
   string localfile = ((Param5*)p)->filename;
   int bucketnum = ((Param5*)p)->bucketnum;
   int bucketid = ((Param5*)p)->bucketid;
   const int key = ((Param5*)p)->key;
   const int type = ((Param5*)p)->type;
   string function = ((Param5*)p)->function;
   queue<Bucket>* bq = ((Param5*)p)->bq;
   pthread_mutex_t* bqlock = ((Param5*)p)->bqlock;
   pthread_cond_t* bqcond = ((Param5*)p)->bqcond;
   int64_t* pendingSize = ((Param5*)p)->pending;
   string master_ip = ((Param5*)p)->master_ip;
   int master_port = ((Param5*)p)->master_port;
   delete (Param5*)p;

   self->createDir(path);

   // remove old result data files
   for (int i = 0; i < bucketnum; ++ i)
   {
      char* tmp = new char[self->m_strHomeDir.length() + path.length() + localfile.length() + 64];
      sprintf(tmp, "%s.%d", (self->m_strHomeDir + path + "/" + localfile).c_str(), i);
      unlink(tmp);
      sprintf(tmp, "%s.%d.idx", (self->m_strHomeDir + path + "/" + localfile).c_str(), i);
      unlink(tmp);
      delete [] tmp;
   }

   // index file initial offset
   vector<int64_t> offset;
   offset.resize(bucketnum);
   for (vector<int64_t>::iterator i = offset.begin(); i != offset.end(); ++ i)
      *i = 0;
   set<int> fileid;

   while (true)
   {
      pthread_mutex_lock(bqlock);
      while (bq->empty())
         pthread_cond_wait(bqcond, bqlock);
      Bucket b = bq->front();
      bq->pop();
      *pendingSize -= b.totalsize;
      pthread_mutex_unlock(bqlock);

      if (b.totalnum == -1)
         break;

      string speip = b.src_ip;
      int dataport = b.src_dataport;
      int session = b.session;

      for (int i = 0; i < b.totalnum; ++ i)
      {
         int bucket = 0;
         if (self->m_DataChn.recv4(speip, dataport, session, bucket) < 0)
            continue;

         fileid.insert(bucket);

         char* tmp = new char[self->m_strHomeDir.length() + path.length() + localfile.length() + 64];
         sprintf(tmp, "%s.%d", (self->m_strHomeDir + path + "/" + localfile).c_str(), bucket);
         fstream datafile(tmp, ios::out | ios::binary | ios::app);
         sprintf(tmp, "%s.%d.idx", (self->m_strHomeDir + path + "/" + localfile).c_str(), bucket);
         fstream indexfile(tmp, ios::out | ios::binary | ios::app);
         delete [] tmp;
         int64_t start = offset[bucket];
         if (0 == start)
            indexfile.write((char*)&start, 8);

         int32_t len;
         char* data = NULL;
         if (self->m_DataChn.recv(speip, dataport, session, data, len) < 0)
            continue;
         datafile.write(data, len);
         delete [] data;

         tmp = NULL;
         if (self->m_DataChn.recv(speip, dataport, session, tmp, len) < 0)
            continue;
         int64_t* index = (int64_t*)tmp;
         for (int j = 0; j < len / 8; ++ j)
            index[j] += start;
         offset[bucket] = index[len / 8 - 1];
         indexfile.write(tmp, len);
         delete [] tmp;

         datafile.close();
         indexfile.close();
      }

      // update total received data
      self->m_SlaveStat.updateIO(speip, b.totalsize, 0);
//.........这里部分代码省略.........