C++ Owned类代码示例

// This test verifies that the environment secrets are resolved when launching a
// task.
TEST_F(EnvironmentSecretIsolatorTest, ResolveSecret)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  mesos::internal::slave::Flags flags = CreateSlaveFlags();

  Fetcher fetcher(flags);
  Try<SecretResolver*> secretResolver = SecretResolver::create();
  EXPECT_SOME(secretResolver);

  Try<MesosContainerizer*> containerizer =
    MesosContainerizer::create(flags, false, &fetcher, secretResolver.get());
  EXPECT_SOME(containerizer);

  Owned<MasterDetector> detector = master.get()->createDetector();
  Try<Owned<cluster::Slave>> slave =
    StartSlave(detector.get(), containerizer.get());
  ASSERT_SOME(slave);

  MockScheduler sched;
  MesosSchedulerDriver driver(
      &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);

  EXPECT_CALL(sched, registered(&driver, _, _));

  Future<std::vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(offers);
  EXPECT_FALSE(offers->empty());

  const string commandString = strings::format(
      "env; test \"$%s\" = \"%s\"",
      SECRET_ENV_NAME,
      SECRET_VALUE).get();

  CommandInfo command;
  command.set_value(commandString);

  // Request a secret.
  // TODO(kapil): Update createEnvironment() to support secrets.
  mesos::Environment::Variable *env =
    command.mutable_environment()->add_variables();
  env->set_name(SECRET_ENV_NAME);
  env->set_type(mesos::Environment::Variable::SECRET);

  mesos::Secret* secret = env->mutable_secret();
  secret->set_type(Secret::VALUE);
  secret->mutable_value()->set_data(SECRET_VALUE);

  TaskInfo task = createTask(
      offers.get()[0].slave_id(),
      Resources::parse("cpus:0.1;mem:32").get(),
      command);

  // NOTE: Successful tasks will output two status updates.
  Future<TaskStatus> statusRunning;
  Future<TaskStatus> statusFinished;
  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusRunning))
    .WillOnce(FutureArg<1>(&statusFinished));

  driver.launchTasks(offers.get()[0].id(), {task});

  AWAIT_READY(statusRunning);
  EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
  AWAIT_READY(statusFinished);
  EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());

  driver.stop();
  driver.join();
}

unsigned applyNodes(const char *grpip, ApplyMode mode, unsigned ver, bool isdali, bool quiet)
{
    SocketEndpointArray eps;
    if (isdali&&(stricmp(grpip,"all")==0)) {
        Owned<IRemoteConnection> conn = querySDS().connect("/Environment/Software", myProcessSession(), RTM_LOCK_READ, SDS_CONNECT_TIMEOUT);
        if (!conn) 
            return 0;
        IPropertyTree* root = conn->queryRoot();
        Owned<IPropertyTreeIterator> clusters= root->getElements("ThorCluster");
        unsigned ret = 0;
        if (clusters->first()) {
            do {
                IPropertyTree &cluster = clusters->query();
                ret += applyNodes(cluster.queryProp("@name"),mode,ver,true,quiet);
            } while (clusters->next());
        }
        return ret;
    }
    SocketEndpointArray result;
    StringAttrArray resultstr;
    if (!isdali||!getCluster(grpip,eps)) {
        SocketEndpoint ep(grpip);
        if (ep.isNull()) {
            ERRLOG("%s is not a group name or ip",grpip);
            return 0;
        }
        if (ep.port==0)
            ep.port = getDaliServixPort();
        eps.append(ep);
    }
    PointerIArrayOf<ISocket> sockets;
    unsigned to=10*1000;
    unsigned n=eps.ordinality();    // use approx log scale (timeout is long but only for failure situation)
    while (n>1) {
        n/=2;
        to+=10*1000;
    }
    if (!quiet&&(n>1))
        PROGLOG("Scanning %s...",grpip);
    multiConnect(eps,sockets,to);
    CriticalSection sect;
    class casyncfor: public CAsyncFor
    {
        SocketEndpointArray &eps;
        PointerIArrayOf<ISocket> &sockets;
        ApplyMode mode;
        unsigned ver;
        SocketEndpointArray &result;
        StringAttrArray &resultstr;
        CriticalSection &sect;
    public:
        casyncfor(ApplyMode _mode, unsigned _ver,SocketEndpointArray &_eps,PointerIArrayOf<ISocket> &_sockets,SocketEndpointArray &_result, StringAttrArray &_resultstr,CriticalSection &_sect) 
            : eps(_eps), sockets(_sockets), result(_result), resultstr(_resultstr), sect(_sect)
        { 
            mode = _mode;
            ver = _ver;
        }
        void Do(unsigned i)
        {
            ISocket *sock = sockets.item(i);
            StringBuffer epstr;
            SocketEndpoint ep = eps.item(i);
            ep.getUrlStr(epstr);
//          PROGLOG("T.1 %s %x",epstr.str(),(unsigned)sock);
            StringBuffer verstr;
            unsigned rver=0;
            if (sock) {
                rver = getRemoteVersion(sock, verstr);
                switch (mode) {
                case AMcheck:
                    if (rver!=0)
                        return;
                case AMver: {
                        CriticalBlock block(sect);
                        result.append(ep);
                        StringBuffer ln;
                        ln.append(rver).append(",\"").append(verstr).append('"');
                        resultstr.append(* new StringAttrItem(ln.str()));
                    }
                    return;
                case AMstopver:
                case AMcheckver: 
                case AMcheckvermajor: {
                    // compares versions up to the '-'
                        const char *rv = verstr.str();
                        const char *v = remoteServerVersionString();
                        if (mode!=AMcheckvermajor) {
                            while (*v&&(*v!='-')&&(*v==*rv)) {
                                v++;
                                rv++;
                            }
                        }
                        if ((*rv==*v)&&(rver==ver))
                            return;
                        while (*rv&&(*rv!='-'))
                            rv++;
                        verstr.setLength(rv-verstr.str());
                        if ((mode==AMcheckver)||(mode==AMcheckvermajor))
                            break;

//.........这里部分代码省略.........

void sendRequest()
{
    if (!loggingManager)
    {
        printf("No logging manager.\n");
        return;
    }

    StringBuffer action, option, status;
    testData->getProp("action", action);
    testData->getProp("option", option);
    if (action.length() && strieq(action.str(), "getTransactionSeed"))
    {
        StringBuffer transactionSeed;
        loggingManager->getTransactionSeed(transactionSeed, status);
        if (transactionSeed.length())
            printf("Got transactionSeed: <%s>\n", transactionSeed.str());
    }
    else if (action.length() && strieq(action.str(), "UpdateLog"))
    {
        IPropertyTree* logContentTree = testData->queryPropTree("LogContent");
        if (!logContentTree)
        {
            printf("can't read log content.\n");
            return;
        }
        StringBuffer logContentXML;
        toXML(logContentTree, logContentXML);
        printf("log content: <%s>.\n", logContentXML.str());

        Owned<IEspContext> espContext =  createEspContext();
        const char* userName = logContentTree->queryProp("ESPContext/UserName");
        const char* sourceIP = logContentTree->queryProp("ESPContext/SourceIP");
        short servPort = logContentTree->getPropInt("ESPContext/Port");
        espContext->setUserID(userName);
        espContext->setServAddress(sourceIP, servPort);

        const char* backEndResp = logContentTree->queryProp("BackEndResponse");
        IPropertyTree* userContextTree = logContentTree->queryPropTree("MyUserContext");
        IPropertyTree* userRequestTree = logContentTree->queryPropTree("MyUserRequest");
        IPropertyTree* userRespTree = logContentTree->queryPropTree("MyUserResponseEx");
        IPropertyTree* logDatasetsTree = logContentTree->queryPropTree("LogDatasets");
        StringBuffer userContextXML, userRequestXML, userRespXML, logDatasetsXML;
        toXML(logDatasetsTree, logDatasetsXML);
        toXML(userRespTree, userRespXML);

        toXML(userContextTree, userContextXML);
        toXML(userRequestTree, userRequestXML);
        printf("userContextXML: <%s>.\n", userContextXML.str());
        printf("userRequestXML: <%s>.\n", userRequestXML.str());
        printf("userRespXML: <%s>.\n", userRespXML.str());
        printf("backEndResp: <%s>.\n", backEndResp);

        //Sleep(5000); //Waiting for loggingManager to start
        loggingManager->updateLog(option.str(), *espContext, userContextTree, userRequestTree, backEndResp, userRespXML.str(), logDatasetsXML.str(), status);
    }
    else if (action.length() && strieq(action.str(), "UpdateLog1"))
    {
        IPropertyTree* logContentTree = testData->queryPropTree("LogContent");
        if (!logContentTree)
        {
            printf("can't read log content.\n");
            return;
        }
        StringBuffer logContentXML;
        toXML(logContentTree, logContentXML);
        printf("log content: <%s>.\n", logContentXML.str());
        //Sleep(5000); //Waiting for loggingManager to start
        loggingManager->updateLog(option.str(), logContentXML.str(), status);
    }
    else
        printf("Invalid action.\n");
}

TEST_P(MemoryIsolatorTest, ROOT_MemUsage)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  slave::Flags flags = CreateSlaveFlags();
  flags.isolation = GetParam();

  Fetcher fetcher(flags);

  Try<MesosContainerizer*> _containerizer =
    MesosContainerizer::create(flags, true, &fetcher);

  ASSERT_SOME(_containerizer);

  Owned<MesosContainerizer> containerizer(_containerizer.get());

  Owned<MasterDetector> detector = master.get()->createDetector();

  Try<Owned<cluster::Slave>> slave = StartSlave(
      detector.get(),
      containerizer.get());

  ASSERT_SOME(slave);

  MockScheduler sched;
  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      DEFAULT_CREDENTIAL);

  EXPECT_CALL(sched, registered(&driver, _, _));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  driver.start();

  AWAIT_READY(offers);
  ASSERT_FALSE(offers->empty());

  TaskInfo task = createTask(offers.get()[0], "sleep 120");

  Future<TaskStatus> statusRunning;
  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&statusRunning));

  driver.launchTasks(offers.get()[0].id(), {task});

  AWAIT_READY(statusRunning);
  EXPECT_EQ(TASK_RUNNING, statusRunning->state());

  Future<hashset<ContainerID>> containers = containerizer->containers();
  AWAIT_READY(containers);
  ASSERT_EQ(1u, containers->size());

  ContainerID containerId = *(containers->begin());

  Future<ResourceStatistics> usage = containerizer->usage(containerId);
  AWAIT_READY(usage);

  // TODO(jieyu): Consider using a program that predictably increases
  // RSS so that we can set more meaningful expectation here.
  EXPECT_LT(0u, usage->mem_rss_bytes());

  driver.stop();
  driver.join();
}

    void init()
    {
        helper = (IHThorKeyDiffArg *)queryHelper();

        originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), helper->queryOriginalName()));
        newIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), helper->queryUpdatedName()));
        if (originalIndexFile->numParts() != newIndexFile->numParts())
            throw MakeActivityException(this, TE_KeyDiffIndexSizeMismatch, "Index %s and %s differ in width", helper->queryOriginalName(), helper->queryUpdatedName());
        if (originalIndexFile->querySuperFile() || newIndexFile->querySuperFile())
            throw MakeActivityException(this, 0, "Diffing super files not supported");  

        width = originalIndexFile->numParts();

        originalDesc.setown(originalIndexFile->getFileDescriptor());
        newIndexDesc.setown(newIndexFile->getFileDescriptor());
        Owned<IPartDescriptor> tlkDesc = originalDesc->getPart(originalDesc->numParts()-1);
        const char *kind = tlkDesc->queryProperties().queryProp("@kind");
        local = NULL == kind || 0 != stricmp("topLevelKey", kind);

        if (!local)
            width--; // 1 part == No n distributed / Monolithic key
        if (width > container.queryJob().querySlaves())
            throw MakeActivityException(this, 0, "Unsupported: keydiff(%s, %s) - Cannot diff a key that's wider(%d) than the target cluster size(%d)", originalIndexFile->queryLogicalName(), newIndexFile->queryLogicalName(), width, container.queryJob().querySlaves());

        queryThorFileManager().noteFileRead(container.queryJob(), originalIndexFile);
        queryThorFileManager().noteFileRead(container.queryJob(), newIndexFile);

        IArrayOf<IGroup> groups;
        fillClusterArray(container.queryJob(), helper->queryOutputName(), clusters, groups);
        patchDesc.setown(queryThorFileManager().create(container.queryJob(), helper->queryOutputName(), clusters, groups, 0 != (KDPoverwrite & helper->getFlags()), 0, !local, width));
    }

// This test ensures that when explicit acknowledgements are enabled,
// acknowledgements for master-generated updates are dropped by the
// driver. We test this by creating an invalid task that uses no
// resources.
TEST_F(MesosSchedulerDriverTest, ExplicitAcknowledgementsMasterGeneratedUpdate)
{
  Try<Owned<cluster::Master>> master = StartMaster();
  ASSERT_SOME(master);

  Owned<MasterDetector> detector = master.get()->createDetector();
  Try<Owned<cluster::Slave>> slave = StartSlave(detector.get());
  ASSERT_SOME(slave);

  MockScheduler sched;
  MesosSchedulerDriver driver(
      &sched,
      DEFAULT_FRAMEWORK_INFO,
      master.get()->pid,
      false,
      DEFAULT_CREDENTIAL);

  EXPECT_CALL(sched, registered(&driver, _, _));

  Future<vector<Offer>> offers;
  EXPECT_CALL(sched, resourceOffers(&driver, _))
    .WillOnce(FutureArg<1>(&offers))
    .WillRepeatedly(Return()); // Ignore subsequent offers.

  // Ensure no status update acknowledgements are sent to the master.
  EXPECT_NO_FUTURE_CALLS(
      mesos::scheduler::Call(),
      mesos::scheduler::Call::ACKNOWLEDGE,
      _ ,
      master.get()->pid);

  driver.start();

  AWAIT_READY(offers);
  EXPECT_NE(0u, offers->size());

  // Launch a task using no resources.
  TaskInfo task;
  task.set_name("");
  task.mutable_task_id()->set_value("1");
  task.mutable_slave_id()->MergeFrom(offers.get()[0].slave_id());
  task.mutable_executor()->MergeFrom(DEFAULT_EXECUTOR_INFO);

  vector<TaskInfo> tasks;
  tasks.push_back(task);

  Future<TaskStatus> status;
  EXPECT_CALL(sched, statusUpdate(&driver, _))
    .WillOnce(FutureArg<1>(&status));

  driver.launchTasks(offers.get()[0].id(), tasks);

  AWAIT_READY(status);
  ASSERT_EQ(TASK_ERROR, status->state());
  ASSERT_EQ(TaskStatus::SOURCE_MASTER, status->source());
  ASSERT_EQ(TaskStatus::REASON_TASK_INVALID, status->reason());

  // Now send the acknowledgement.
  driver.acknowledgeStatusUpdate(status.get());

  // Settle the clock to ensure driver processes the acknowledgement,
  // which should get dropped due to having come from the master.
  Clock::pause();
  Clock::settle();

  driver.stop();
  driver.join();
}

    void done()
    {
        StringBuffer scopedName;
        queryThorFileManager().addScope(container.queryJob(), helper->queryOutputName(), scopedName);
        Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock();
        Owned<IWUResult> r = wu->updateResultBySequence(helper->getSequence());
        r->setResultStatus(ResultStatusCalculated);
        r->setResultLogicalName(scopedName.str());
        r.clear();
        wu.clear();

        IPropertyTree &patchProps = patchDesc->queryProperties();
        setExpiryTime(patchProps, helper->getExpiryDays());
        IPropertyTree &originalProps = originalDesc->queryProperties();;
        if (originalProps.queryProp("ECL"))
            patchProps.setProp("ECL", originalProps.queryProp("ECL"));
        if (originalProps.getPropBool("@local"))
            patchProps.setPropBool("@local", true);
        container.queryTempHandler()->registerFile(helper->queryOutputName(), container.queryOwner().queryGraphId(), 0, false, WUFileStandard, &clusters);
        Owned<IDistributedFile> patchFile;
        // set part sizes etc
        queryThorFileManager().publish(container.queryJob(), helper->queryOutputName(), false, *patchDesc, &patchFile, 0, false);
        try { // set file size
            if (patchFile) {
                __int64 fs = patchFile->getFileSize(true,false);
                if (fs!=-1)
                    patchFile->queryAttributes().setPropInt64("@size",fs);
            }
        }
        catch (IException *e) {
            EXCLOG(e,"keydiff setting file size");
            e->Release();
        }
        // Add a new 'Patch' description to the secondary key.
        DistributedFilePropertyLock lock(newIndexFile);
        IPropertyTree &fileProps = lock.queryAttributes();
        StringBuffer path("Patch[@name=\"");
        path.append(scopedName.str()).append("\"]");
        IPropertyTree *patch = fileProps.queryPropTree(path.str());
        if (!patch) patch = fileProps.addPropTree("Patch", createPTree());
        patch->setProp("@name", scopedName.str());
        unsigned checkSum;
        if (patchFile->getFileCheckSum(checkSum))
            patch->setPropInt64("@checkSum", checkSum);

        IPropertyTree *index = patch->setPropTree("Index", createPTree());
        index->setProp("@name", originalIndexFile->queryLogicalName());
        if (originalIndexFile->getFileCheckSum(checkSum))
            index->setPropInt64("@checkSum", checkSum);
    }

void CDiskWriteSlaveActivityBase::open()
{
    if (dlfn.isExternal() && !firstNode())
    {
        input.setown(createDataLinkSmartBuffer(this, inputs.item(0), PROCESS_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), grouped, RCUNBOUND, NULL, false, &container.queryJob().queryIDiskUsage()));
        startInput(input);
        if (!rfsQueryParallel)
        {
            ActPrintLog("Blocked, waiting for previous part to complete write");
            CMessageBuffer msg;
            if (!receiveMsg(msg, container.queryJob().queryMyRank()-1, mpTag))
                return;
            rowcount_t prevRows;
            msg.read(prevRows);
            msg.read(tempExternalName); // reuse temp filename, last node will rename
            ActPrintLog("Previous write row count = %"RCPF"d", prevRows);
        }
    }
    else
    {
        input.set(inputs.item(0));
        startInput(input);
    }
    processed = THORDATALINK_STARTED;

    bool extend = 0 != (diskHelperBase->getFlags() & TDWextend);
    if (extend)
        ActPrintLog("Extending file %s", fName.get());
    size32_t exclsz = 0;
    calcFileCrc = true;

    bool external = dlfn.isExternal();
    bool query = dlfn.isQuery();
    if (query && compress)
        UNIMPLEMENTED;

    bool direct = query || (external && !firstNode());
    bool rename = !external || (!query && lastNode());
    Owned<IFileIO> iFileIO = createMultipleWrite(this, *partDesc, exclsz, compress, extend||(external&&!query), ecomp, this, direct, rename, &abortSoon, (external&&!query) ? &tempExternalName : NULL);

    if (compress)
    {
        ActPrintLog("Performing row compression on output file: %s", fName.get());
        calcFileCrc = false;
    }
    Owned<IFileIOStream> stream;
    if (wantRaw())
    {
        outraw.setown(createBufferedIOStream(iFileIO));
        stream.set(outraw);
    }
    else
    {
        stream.setown(createIOStream(iFileIO));
        out.setown(createRowWriter(stream,::queryRowSerializer(input),::queryRowAllocator(input),grouped,calcFileCrc,false)); // flushed by close
    }
    CDfsLogicalFileName dlfn;
    dlfn.set(logicalFilename);
    if (extend || (external && !query))
        stream->seek(0,IFSend);
    ActPrintLog("Created output stream for %s", fName.get());
}

 foreachvalue (const Owned<Store>& store, stores) {
   recovers.push_back(store->recover());
 }

 void serializeSlaveData(MemoryBuffer &dst, unsigned slave)
 {
     if (slave < width) // if false - due to mismatch width fitting - fill in with a blank entry
     {
         dst.append(true);
         Owned<IPartDescriptor> originalPartDesc = originalDesc->getPart(slave);
         originalPartDesc->serialize(dst);
         Owned<IPartDescriptor> newIndexPartDesc = newIndexDesc->getPart(slave);
         newIndexPartDesc->serialize(dst);
         patchDesc->queryPart(slave)->serialize(dst);
         
         if (0 == slave)
         {
             if (!local)
             {
                 dst.append(true);
                 Owned<IPartDescriptor> originalTlkPartDesc = originalDesc->getPart(originalDesc->numParts()-1);
                 originalTlkPartDesc->serialize(dst);                                        
                 Owned<IPartDescriptor> newIndexTlkPartDesc = newIndexDesc->getPart(newIndexDesc->numParts()-1);
                 newIndexTlkPartDesc->serialize(dst);
                 patchDesc->queryPart(patchDesc->numParts()-1)->serialize(dst);
             }
             else
                 dst.append(false);
         }
     }
     else
         dst.append(false); // no part
 }

void RemoteDataSourceServer::doCmdNumRows(MemoryBuffer & in, MemoryBuffer & out)
{
    Owned<IFvDataSource> ds = readDataSource(in);
    __int64 numRows = ds ? ds->numRows(true) : 0;
    out.append(numRows);
}

void CWriteMasterBase::init()
{
    published = false;
    recordsProcessed = 0;
    OwnedRoxieString fname(diskHelperBase->getFileName());
    dlfn.set(fname);
    if (diskHelperBase->getFlags() & TDWextend)
    {
        assertex(0 == (diskHelperBase->getFlags() & (TDXtemporary|TDXjobtemp)));
        Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), fname, false, true);
        if (file.get())
        {
            fileDesc.setown(file->getFileDescriptor());
            queryThorFileManager().noteFileRead(container.queryJob(), file, true);
        }
    }
    if (dlfn.isExternal())
        mpTag = container.queryJob().allocateMPTag(); // used 
    if (NULL == fileDesc.get())
    {
        bool overwriteok = 0!=(TDWoverwrite & diskHelperBase->getFlags());
        
        unsigned idx=0;
        while (true)
        {
            OwnedRoxieString cluster(diskHelperBase->getCluster(idx));
            if(!cluster)
                break;
            clusters.append(cluster);
            idx++;
        }
        IArrayOf<IGroup> groups;
        fillClusterArray(container.queryJob(), fname, clusters, groups);
        fileDesc.setown(queryThorFileManager().create(container.queryJob(), fname, clusters, groups, overwriteok, diskHelperBase->getFlags()));
        if (1 == groups.ordinality())
            targetOffset = getGroupOffset(groups.item(0), container.queryJob().querySlaveGroup());
        IPropertyTree &props = fileDesc->queryProperties();
        if (diskHelperBase->getFlags() & (TDWowned|TDXjobtemp|TDXtemporary))
            props.setPropBool("@owned", true);
        if (diskHelperBase->getFlags() & TDWresult)
            props.setPropBool("@result", true);
        const char *rececl= diskHelperBase->queryRecordECL();
        if (rececl&&*rececl)
            props.setProp("ECL", rececl);
        bool blockCompressed=false;
        void *ekey;
        size32_t ekeylen;
        diskHelperBase->getEncryptKey(ekeylen,ekey);
        if (ekeylen)
        {
            memset(ekey,0,ekeylen);
            free(ekey);
            props.setPropBool("@encrypted", true);      
            blockCompressed = true;
        }
        else if (0 != (diskHelperBase->getFlags() & TDWnewcompress) || 0 != (diskHelperBase->getFlags() & TDXcompress))
            blockCompressed = true;
        if (blockCompressed)
            props.setPropBool("@blockCompressed", true);
        if (TAKdiskwrite == container.getKind() && (0 != (diskHelperBase->getFlags() & TDXtemporary)) && container.queryOwner().queryOwner() && (!container.queryOwner().isGlobal())) // I am in a child query
        { // do early, because this will be local act. and will not come back to master until end of owning graph.
            publish();
        }
    }
}

static bool physicalPartCopy(IFile *from,const char *tofile, Owned<IException> &exc, StringBuffer *tmpname)
{
    StringBuffer tmpnamestr;
    if (!tmpname)
        tmpname = &tmpnamestr;
    tmpname->append(tofile).append("__");
    size32_t l = tmpname->length();
    genUUID(*tmpname,true); // true for windows
    StringAttr uuid(tmpname->str()+l);
    tmpname->append(".tmp");
    RemoteFilename tmpfn;
    tmpfn.setRemotePath(tmpname->str());
    //unsigned lastpc;
#ifdef LOG_PART_COPY
    PROGLOG("start physicalPartCopy(%s,%s)",from->queryFilename(),tmpname->str());
#endif
    try {
        recursiveCreateDirectoryForFile(tmpname->str());
        while(!asyncCopyFileSection(
                uuid,
                from,
                tmpfn,
                (offset_t)-1, // creates file
                0,
                (offset_t)-1, // all file
                NULL,
                PHYSICAL_COPY_POLL_TIME)) {
            // Abort check TBD
        }
    }
    catch (IException *e) {
        EXCLOG(e,"SingleFileCopy: File copy error");
        if (exc)
            exc.setown(e);
        else
            e->Release();
    }
    Owned<IFile> f = createIFile(tmpfn);
    if (!exc.get()&&(tmpnamestr.length()!=0)) {
        try {
#ifdef LOG_PART_COPY
            PROGLOG("physicalPartCopy rename(%s,%s)",tmpname->str(),pathTail(tofile));
#endif
            f->rename(pathTail(tofile));
        }
        catch (IException *e) {
            EXCLOG(e,"SingleFileCopy: File rename error");
            if (exc)
                exc.setown(e);
            else
                e->Release();
        }
    }
    if (exc.get()) {
        try {
            f->remove();
        }
        catch (IException *e) {
            // ignore
            e->Release();
        }
    }
#ifdef LOG_PART_COPY
    PROGLOG("done physicalPartCopy %s",(exc.get()==NULL)?"OK":"Failed");
#endif
    return exc.get()==NULL;
}

void testMultiCluster()
{
    Owned<IGroup> grp1 = createIGroup("192.168.51.1-5");
    Owned<IGroup> grp2 = createIGroup("192.168.16.1-5");
    Owned<IGroup> grp3 = createIGroup("192.168.53.1-5");
    queryNamedGroupStore().add("testgrp1",grp1);
    queryNamedGroupStore().add("testgrp2",grp2);
    queryNamedGroupStore().add("testgrp3",grp3);

    Owned<IFileDescriptor> fdesc = createFileDescriptor();
    fdesc->setDefaultDir("/c$/thordata/test");
    fdesc->setPartMask("testfile1._$P$_of_$N$");
    fdesc->setNumParts(5);
    ClusterPartDiskMapSpec mapping;
    fdesc->addCluster(grp1,mapping);
    fdesc->addCluster(grp2,mapping);
    fdesc->addCluster(grp3,mapping);
    queryDistributedFileDirectory().removeEntry("test::testfile1",UNKNOWN_USER);
    Owned<IDistributedFile> file = queryDistributedFileDirectory().createNew(fdesc);
    queryDistributedFileDirectory().removeEntry("test::testfile1",UNKNOWN_USER);
    file->attach("test::testfile1",UNKNOWN_USER);
    StringBuffer name;
    unsigned i;
    for (i=0;i<file->numClusters();i++)
        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
    file.clear();
    file.setown(queryDistributedFileDirectory().lookup("test::testfile1",UNKNOWN_USER));
    for (i=0;i<file->numClusters();i++)
        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
    file.clear();
    file.setown(queryDistributedFileDirectory().lookup("test::[email protected]",UNKNOWN_USER));
    for (i=0;i<file->numClusters();i++)
        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
    file.clear();
    queryDistributedFileDirectory().removePhysical("test::[email protected]",UNKNOWN_USER);
    file.setown(queryDistributedFileDirectory().lookup("test::testfile1",UNKNOWN_USER));
    for (i=0;i<file->numClusters();i++)
        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
}

    void processMessage(CMessageBuffer &mb)
    {
        ICoven &coven=queryCoven();
        MemoryBuffer params;
        params.swapWith(mb);
        int fn;
        params.read(fn);
        switch (fn) {
        case MDR_GET_VALUE: {
                StringAttr id;
                StringBuffer buf;
                params.read(id);
                if (0 == stricmp(id,"threads")) {
                    mb.append(getThreadList(buf).str());
                }
                else if (0 == stricmp(id, "mpqueue")) {
                    mb.append(getReceiveQueueDetails(buf).str());
                }
                else if (0 == stricmp(id, "locks")) { // Legacy - newer diag clients should use querySDS().getLocks() directly
                    Owned<ILockInfoCollection> lockInfoCollection = querySDS().getLocks();
                    mb.append(lockInfoCollection->toString(buf).str());
                }
                else if (0 == stricmp(id, "sdsstats")) { // Legacy - newer diag clients should use querySDS().getUsageStats() directly
                    mb.append(querySDS().getUsageStats(buf).str());
                }
                else if (0 == stricmp(id, "connections")) { // Legacy - newer diag clients should use querySDS().getConnections() directly
                    mb.append(querySDS().getConnections(buf).str());
                }
                else if (0 == stricmp(id, "sdssubscribers")) { // Legacy - newer diag clients should use querySDS().getSubscribers() directly
                    mb.append(querySDS().getSubscribers(buf).str());
                }
                else if (0 == stricmp(id, "clients")) {
                    mb.append(querySessionManager().getClientProcessList(buf).str());
                }
                else if (0 == stricmp(id, "subscriptions")) {
                    mb.append(getSubscriptionList(buf).str());
                }
                else if (0 == stricmp(id, "mpverify")) {
                    queryWorldCommunicator().verifyAll(buf);
                    mb.append(buf.str());
                }
                else if (0 == stricmp(id, "extconsistency")) {
                    mb.append(querySDS().getExternalReport(buf).str());
                }
                else if (0 == stricmp(id, "build")) {
                    mb.append("$Id: dadiags.cpp 62376 2011-02-04 21:59:58Z sort $");
                }
                else if (0 == stricmp(id, "sdsfetch")) {
                    StringAttr branchpath;
                    params.read(branchpath);
                    Linked<IPropertyTree> sroot = querySDSServer().lockStoreRead();
                    try { sroot->queryPropTree(branchpath)->serialize(mb); }
                    catch (...) { querySDSServer().unlockStoreRead(); throw; }
                    querySDSServer().unlockStoreRead();
                }
                else if (0 == stricmp(id, "perf")) {
                    getSystemTraceInfo(buf,PerfMonStandard);
                    mb.append(buf.str());
                }
                else if (0 == stricmp(id, "sdssize")) {
                    StringAttr branchpath;
                    params.read(branchpath);
                    Linked<IPropertyTree> sroot = querySDSServer().lockStoreRead();
                    StringBuffer sbuf;
                    try { 
                        toXML(sroot->queryPropTree(branchpath),sbuf); 
                        DBGLOG("sdssize '%s' = %d",branchpath.get(),sbuf.length());
                    }
                    catch (...) { 
                        querySDSServer().unlockStoreRead(); 
                        throw; 
                    }
                    querySDSServer().unlockStoreRead();
                    mb.append(sbuf.length());
                }
                else if (0 == stricmp(id, "disconnect")) {
                    StringAttr client;
                    params.read(client);
                    SocketEndpoint ep(client);
                    PROGLOG("Dalidiag request to close client connection: %s", client.get());
                    Owned<INode> node = createINode(ep);
                    queryCoven().disconnect(node);
                }
                else if (0 == stricmp(id, "unlock")) {
                    __int64 connectionId;
                    bool disconnect;
                    params.read(connectionId);
                    params.read(disconnect);
                    PROGLOG("Dalidiag request to unlock connection id: %" I64F "x", connectionId);
                    StringBuffer connectionInfo;
                    bool success = querySDSServer().unlock(connectionId, disconnect, connectionInfo);
                    mb.append(success);
                    if (success)
                        mb.append(connectionInfo);
                }
                else if (0 == stricmp(id, "save")) {
                    PROGLOG("Dalidiag requests SDS save");
                    querySDSServer().saveRequest();
                }
                else if (0 == stricmp(id, "settracetransactions")) {
//.........这里部分代码省略.........