C++ DataSet类代码示例

/** Calculate Pearson product-moment correlation between DataSets.
  * \D1 DataSet to caclculate correlation for.
  * \D2 DataSet to caclulate correlation to.
  * \return Pearson product-moment correlation coefficient.
  */
double DS_Math::CorrCoeff( DataSet& D1, DataSet& D2 ) {
  // Check if D1 and D2 are valid types
  if ( !GoodCalcType(D1) ) return 0;
  if ( !GoodCalcType(D2) ) return 0;
  // Check that D1 and D2 have same # data points.
  int Nelements = D1.Size();
  if (Nelements != D2.Size()) {
    mprinterr("Error: Corr: # elements in dataset %s (%i) not equal to\n",
              D1.Legend().c_str(), Nelements);
    mprinterr("Error:       # elements in dataset %s (%i)\n",
              D2.Legend().c_str(), D2.Size());
    return 0;
  }
  // Calculate averages
  double avg1 = Avg(D1);
  double avg2 = Avg(D2);
  // Calculate average deviations. 
  double sumdiff1_2 = 0.0;
  double sumdiff2_2 = 0.0;
  double corr_coeff = 0.0;
  //mprinterr("DATASETS %s and %s\n", c_str(), D2.c_str());
  for (int i = 0; i < Nelements; i++) {
    double diff1 = D1.Dval(i) - avg1;
    double diff2 = D2.Dval(i) - avg2;
    sumdiff1_2 += (diff1 * diff1);
    sumdiff2_2 += (diff2 * diff2);
    corr_coeff += (diff1 * diff2);
  }
  if (sumdiff1_2 == 0.0 || sumdiff2_2 == 0.0) {
    mprintf("Warning: Corr: %s to %s, Normalization is 0\n",
            D1.Legend().c_str(),  D2.Legend().c_str());
    return 0;
  }
  // Correlation coefficient
  corr_coeff /= ( sqrt( sumdiff1_2 ) * sqrt( sumdiff2_2 ) );
  //mprintf("    CORRELATION COEFFICIENT %6s to %6s IS %10.4f\n",
  //        D1_->c_str(), D2_->c_str(), corr_coeff );
  return corr_coeff;
}

size_t nnet_predict(NNet& nnet, DataSet& data) {

  const size_t batchSize = 256;
  size_t nError = 0;

  nnet.setDropout(false);
  Batches batches(batchSize, data.size());
  for (auto itr = batches.begin(); itr != batches.end(); ++itr) {
    auto d = data[itr];
    mat x = ~mat(d.x);
    mat prob = nnet.feedForward(x);
    nError += zeroOneError(prob, d.y);
  }
  nnet.setDropout(true);

  return nError;
}

void AlgorithmRunner::addFavorite(const QString &algName, const DataSet &data) {
  if (!PluginLister::pluginExists(QStringToTlpString(algName)))
    return;

  TulipSettings::instance().addFavoriteAlgorithm(algName);

  for (auto i : _favorites) {
    if (i->name() == algName)
      return;
  }

  _ui->favoritesBox->widget()->setMinimumHeight(0);
  AlgorithmRunnerItem *item = new AlgorithmRunnerItem(algName);
  item->setGraph(_graph);

  if (!data.empty()) {
    item->setData(data);
  }

  item->setFavorite(true);
  int itemPos = 0;

  for (auto i : _ui->favoritesBox->widget()->findChildren<AlgorithmRunnerItem *>()) {
    if (i->name() > item->name()) {
      break;
    }

    ++itemPos;
  }

  static_cast<QBoxLayout *>(_ui->favoritesBox->widget()->layout())->insertWidget(itemPos, item);

  _favorites += item;

  item->installEventFilter(this);

  item->setAcceptDrops(true);

  connect(item, SIGNAL(favorized(bool)), this, SLOT(favorized(bool)));

  for (auto i : findChildren<AlgorithmRunnerItem *>()) {
    if (i != item && i->name() == algName)
      i->setFavorite(true);
  }
}

void Toolbox::initWeightsRandomFromMeanAndStd(DataSet &X)
{
   //Initialise weights. We use the seed (or clock) to initiliaze random number
   //generator
   if (seed==0){
	  srand( (unsigned)time( NULL ) );
   }
   else {
	  srand(seed);
   }
	int nbRawFeatures = X.getNumberofRawFeatures();

	//Initialise weights
	dVector w(pFeatureGenerator->getNumberOfFeatures());
	double widthRangeWeight = fabs(maxRangeWeights - minRangeWeights);
	double randValue;

	// mean and std_dev
	dVector mean(nbRawFeatures);
	dVector stdDev(nbRawFeatures);
	calculateGlobalMeanAndStd(X,mean,stdDev);

	//Initialize weights with global mean and standard deviation
	// Only initialize the values especific to the HMM-like HCRF
	featureVector* vecFeatures = getAllFeatures(X);
	feature* pFeature = vecFeatures->getPtr();
	for(int j = 0; j < vecFeatures->size(); j++, pFeature++)
	{
		switch(pFeature->nodeIndex)
		{
		case SQUARE_RAW_FEATURE_ID:
			randValue = (((double)rand())/(double)RAND_MAX)*2.0*stdDev[(int)pFeature->value]-stdDev[(int)pFeature->value];
			w.setValue(pFeature->globalId,mean[(int)pFeature->value] * mean[(int)pFeature->value]+randValue);
			break;
		case ONE_FEATURE_ID:
		case RAW_FEATURE_ID:
			randValue = (((double)rand())/(double)RAND_MAX)*2.0*stdDev[(int)pFeature->value]-stdDev[(int)pFeature->value];
			w.setValue(pFeature->globalId,mean[(int)pFeature->value]+randValue);
			break;
		default:
			w.setValue(pFeature->globalId,(((double)rand())/(double)RAND_MAX)*widthRangeWeight+minRangeWeights);
		}
	}
	pModel->setWeights(w);
}

void FrozenDataSet::checkValidFrozenDataSet(const DataSet& dataset) {
  // Check that the dataset we have can be converted
  PointLayout layout = dataset.layout();

  QStringList vlength;
  foreach (const QString& name, layout.descriptorNames()) {
    if (layout.descriptorLocation(name).lengthType() == VariableLength) vlength << name;
  }

  if (!vlength.empty()) {
    throw GaiaException("The following descriptors are variable-length: ", vlength,
                        "\nCan only freeze a dataset which is entirely fixed-length at the moment...");
  }

  if (!layout.descriptorNames(StringType).empty() || !layout.descriptorNames(EnumType).empty()) {
    throw GaiaException("Can only freeze datasets which contain only real descriptors (ie: no strings)");
  }

}

bool History::addDatasetStandard(const DataSet& _dataset)
{
	/// Wait for first ADSC label
	if (datasets_standard_[0].data().empty() == true)
	{
		if (_dataset.label() != datasets_standard_[0].label())
		{
			return false;
		}
	}

	/// Look for dataset
	size_t i = 0;
	for (; i < datasets_standard_.max_size(); i++)
	{
		if (datasets_standard_[i].label() == _dataset.label())
		{
			break;
		}
	}

	/// Not found ?
	if (i == datasets_standard_.max_size()) return false;

	/// Check for new data or timestamp
	if (datasets_standard_[i].checksum() != _dataset.checksum())
	{
		/// Update Data
		datasets_standard_[i].setData(_dataset.data());

		/// Update Timestamp
		if (_dataset.timestamp().isTimestamp() == true)
		{
			datasets_standard_[i].setTimestamp(_dataset.timestamp());
		}

		/// Update Checksum
		datasets_standard_[i].setChecksum(_dataset.checksum());

		return true;
	}

	return false;
}

ReqPtr Rmi::invokeRemoteMethodAsync(DataSet<TextSerializer>& ds)
{
    lock_.lock();
    if (!serverConn_)
    {
        lock_.unlock();
        // TODO - add error message
        return ReqPtr(); // TODO should we return something more intuitive?
    }

    reqId_++;
    if(! ds.add<int>("REQUEST", "REQUESTID", reqId_))
    {
        std::ostringstream tmp;
        tmp << "unable to add REQUEST ID " << reqId_ 
            << "to section REQUEST" << std::endl;
        LOG_ERROR(tmp.str());
        return ReqPtr();
    }
    ReqPtr newRequest(new Request(reqId_)); // will store state of request until reply
                                            // due to asynchronous operation of
                                            // sending
    requests_.insert(std::pair<int32_t, ReqPtr>(reqId_,newRequest ));
    lock_.unlock();

    std::string request = ds.serialize();

    //TODO - should this be a part of the serializer itself or probably as a
    // o/p packetizer

    int32_t reqLength=request.size();
    char length[4+1];
    memset(length, 0, sizeof(length));
    sprintf(length, "%04d", reqLength);
    request.insert(0, length);

    serverConn_->send(request.c_str(), request.size());
    return newRequest; // the client is responsible for calling 
                       // getReply on the new request when he deems proper
}

void
RandomTree<Sample, Label, SplitFunction, SplitEvaluator, LeafNodeStatistics, AppContext>::UpdateLeafStatistics(DataSet<Sample, Label>& dataset)
{
	for (int s = 0; s < dataset.size(); s++)
    {
    	// get the current labelled sample
    	LabelledSample<Sample, Label>* labelled_sample = dataset[s];

    	// route it to the corresponding leaf node
    	int node_id = 0;
		while(!this->m_nodes[node_id]->m_is_leaf)
		{
			if (this->m_nodes[node_id]->Split(labelled_sample) == 0)
				node_id = (int)m_treetable(node_id, 1);
			else
				node_id = (int)m_treetable(node_id, 2);
		}

		// update the leaf node statistics
		this->m_nodes[node_id]->UpdateLeafnodeStatistics(labelled_sample);
    }
}

TEST(DataSetCoreTest, EditExternalResources)
{
    DataSet dataset;

    ExternalResource resource("metatype", "id");
    resource.Name("file1");
    dataset.ExternalResources().Add(resource);

    resource.Name("file2").ResourceId("id2");
    dataset.ExternalResources().Add(resource);
    EXPECT_EQ(2, dataset.ExternalResources().Size());

    // edit
    dataset.ExternalResources()[0].Name("some new name");
    EXPECT_EQ(string("some new name"), dataset.ExternalResources()[0].Name());
    EXPECT_EQ(string("file2"),         dataset.ExternalResources()[1].Name());
}

void GAB::MiningNeg(int n,DataSet& neg){
  const Options& opt = Options::GetInstance();
  int pool_size = opt.numThreads;
  vector<Mat> region_pool(pool_size);
  int st = neg.imgs.size();
  int all = 0;
  int need = n - st;
  double rate;

  while(st<n){
    #pragma omp parallel for
    for(int i = 0;i<pool_size;i++){
      region_pool[i] = neg.NextImage(i);
    }

    #pragma omp parallel for
    for (int i = 0; i < pool_size; i++) {
      float score = 0;
      if(NPDClassify(region_pool[i].clone(),score)){
        #pragma omp critical 
        {
          neg.imgs.push_back(region_pool[i].clone());
          neg.Fx[st]=score;
          if(opt.generate_hd){
            char di[256];
            sprintf(di,"../data/hd/%d.jpg",st);
            imwrite(di,region_pool[i].clone());
          }
          st++;
        }
      }
      all++;
    }
  }
  neg.size = n;
  rate = ((double)(need))/(double)all;
  printf("mining success rate %lf\n",rate);
}

DSptr Rmi::invokeRemoteMethod(DataSet<TextSerializer> & ds)
{
    lock_.lock();
    if (!serverConn_)
    {
        lock_.unlock();
        // TODO add error 
        return DSptr();
    }

    reqId_++;
    if(! ds.add<int>("REQUEST", "REQUESTID", reqId_))
    {
        std::ostringstream tmp;
        tmp << "unable to add REQUEST ID " << reqId_ 
            << "to section REQUEST" << std::endl;
        LOG_ERROR(tmp.str());
        return DSptr();
    }

    ReqPtr newRequest(new Request(reqId_)); // will store state of request until reply
                                            // due to asynchronous operation of
                                            // sending
    requests_.insert(std::pair<int32_t, ReqPtr>(reqId_,newRequest ));
    lock_.unlock();

    std::string request = ds.serialize();
    //TODO - should this be a part of the serializer itself or probably as a
    // o/p packetizer

    int32_t reqLength=request.size();
    char length[4+1];
    memset(length, 0, sizeof(length));
    sprintf(length, "%04d", reqLength);
    request.insert(0, length);
    serverConn_->send(request.c_str(), request.size());
    return newRequest->getReply(); // will wait until we receive reply or we timeout
}

void DataSetTest_GenericData::ColsFromDataSetWithZeroRows()
{
	GenericData gd;
	GenericFileReader reader;
	reader.SetFilename("../data/small_cel_file_with_dataset_of_zero_rows");
	reader.ReadHeader(gd);

	// Open a DataSet that has 0 rows.
	DataSet* ds = gd.DataSet(0,3);
	CPPUNIT_ASSERT(ds);
	CPPUNIT_ASSERT(ds->Cols() == 2);
	CPPUNIT_ASSERT(ds->Rows() == 0);

	ds->Delete();

	// Open another DataSet that has 0 rows.
	ds = gd.DataSet(0,4);
	CPPUNIT_ASSERT(ds);
	CPPUNIT_ASSERT(ds->Cols() == 2);
	CPPUNIT_ASSERT(ds->Rows() == 0);

	ds->Delete();
}

// Copy calculated y data to destination collection specified
void DataSpace::copy(Collection* destinationCollection)
{
    // Check for valid source and destination collections before we start...
    if (!Collection::objectValid(sourceCollection_, "source collection in DataSpace::copy()")) return;
    if (!Collection::objectValid(destinationCollection, "destination collection in DataSpace::copy()")) return;

    // Clear any existing datasets in the destination collection
    destinationCollection->clearDataSets();

    // Create destination datasets using those in the sourceCollection_ to get the z values, and size the x/y arrays
    for (int n=displayDataSetStart_; n<=displayDataSetEnd_; ++n)
    {
        DataSet* originalDataSet = sourceCollection_->dataSet(n);
        DataSet* newDataSet = destinationCollection->addDataSet();
        newDataSet->initialiseData(nPoints_);
        destinationCollection->setDataSetZ(newDataSet, originalDataSet->z());
        newDataSet->setName("Fit to: "+ originalDataSet->name());
    }

    // Copy data from DataSpaceRanges/DataSpaceData into the new datasets
    for (DataSpaceRange* fitRange = ranges_.first(); fitRange != NULL; fitRange = fitRange->next) fitRange->addCalculatedValues(destinationCollection);
}

static void addStaticDataset(QVector<float>& vals, const QString& name, const DataSet::Type type, const QString& datFileName, Mesh* mesh) {
    int nelem = mesh->elements().size();
    int nnodes = mesh->nodes().size();

    NodeOutput* o = new NodeOutput;

    o->init(nnodes, nelem, false);
    o->time = 0.0;
    o->values = vals;

    if (type == DataSet::Bed) {
        memset(o->active.data(), 1, nelem); // All cells active
    } else {
        activateElements(o, mesh);
    }

    DataSet* ds = new DataSet(datFileName);
    ds->setType(type);
    ds->setName(name, false);
    ds->setIsTimeVarying(false);
    ds->addOutput(o);  // takes ownership of the Output
    ds->updateZRange();
    mesh->addDataSet(ds);
}

//
// Draw track line with data labels
//
void ZoomScrollTrack2::trackLineLabel(XYChart *c, int mouseX)
{
    // Clear the current dynamic layer and get the DrawArea object to draw on it.
    DrawArea *d = c->initDynamicLayer();

    // The plot area object
    PlotArea *plotArea = c->getPlotArea();

    // Get the data x-value that is nearest to the mouse, and find its pixel coordinate.
    double xValue = c->getNearestXValue(mouseX);
    int xCoor = c->getXCoor(xValue);

    // Draw a vertical track line at the x-position
    d->vline(plotArea->getTopY(), plotArea->getBottomY(), xCoor,
        d->dashLineColor(0x000000, 0x0101));

    // Draw a label on the x-axis to show the track line position.
    ostringstream xlabel;
    xlabel << "<*font,bgColor=000000*> " << c->xAxis()->getFormattedLabel(xValue, "mmm dd, yyyy")
        << " <*/font*>";
    TTFText *t = d->text(xlabel.str().c_str(), "arialbd.ttf", 8);

    // Restrict the x-pixel position of the label to make sure it stays inside the chart image.
    int xLabelPos = max(0, min(xCoor - t->getWidth() / 2, c->getWidth() - t->getWidth()));
    t->draw(xLabelPos, plotArea->getBottomY() + 6, 0xffffff);
    t->destroy();

    // Iterate through all layers to draw the data labels
    for (int i = 0; i < c->getLayerCount(); ++i) {
        Layer *layer = c->getLayerByZ(i);

        // The data array index of the x-value
        int xIndex = layer->getXIndexOf(xValue);

        // Iterate through all the data sets in the layer
        for (int j = 0; j < layer->getDataSetCount(); ++j)
        {
            DataSet *dataSet = layer->getDataSetByZ(j);
            const char *dataSetName = dataSet->getDataName();

            // Get the color, name and position of the data label
            int color = dataSet->getDataColor();
            int yCoor = c->getYCoor(dataSet->getPosition(xIndex), dataSet->getUseYAxis());

            // Draw a track dot with a label next to it for visible data points in the plot area
            if ((yCoor >= plotArea->getTopY()) && (yCoor <= plotArea->getBottomY()) && (color !=
                (int)Chart::Transparent) && dataSetName && *dataSetName)
            {
                d->circle(xCoor, yCoor, 4, 4, color, color);

                ostringstream label;
                label << "<*font,bgColor=" << hex << color << "*> "
                    << c->formatValue(dataSet->getValue(xIndex), "{value|P4}") << " <*font*>";
                t = d->text(label.str().c_str(), "arialbd.ttf", 8);

                // Draw the label on the right side of the dot if the mouse is on the left side the
                // chart, and vice versa. This ensures the label will not go outside the chart image.
                if (xCoor <= (plotArea->getLeftX() + plotArea->getRightX()) / 2)
                    t->draw(xCoor + 5, yCoor, 0xffffff, Chart::Left);
                else
                    t->draw(xCoor - 5, yCoor, 0xffffff, Chart::Right);

                t->destroy();
            }
        }
    }
}