C++ Columns类代码示例

Retrieve_Template::Retrieve_Template(QString table,QString column, QString where)
{
	Columns list;
	list.clear();
	list.append(column);
	intitialize(table,list,where);
}

void   text_tree::prepare (int current_indent, int indent, Columns& columns)
{
	num_siblings = 1; // including ourselves

	for ( Children::iterator i=children.begin(); i!=children.end(); ++i )
	{
		if ( (*i)->shown )
		{
			(*i)->prepare(current_indent+indent, indent, columns);
			num_siblings += (*i)->num_siblings;
		}
	}

	if ( columns.size() < strings.size() )
	{
		columns.resize(strings.size());
	}

	Strings::iterator j = strings.begin();
	Columns::iterator c = columns.begin();

	// only count as column if theres more then 1 on the line!
	if ( strings.size() > 1 )
	{
		for ( ; j!=strings.end(); ++j, ++c )
		{
			int string_size = (int)(*j).size();
			string_size += (j==strings.begin()) ? current_indent : 0;				

			*c = std::max(string_size, *c);
		}
	}
}

void GDAFile::add(Common::SeekableReadStream *gda) {
	try {
		_gff4s.push_back(new GFF4File(gda, kG2DAID));

		const GFF4Struct &top = _gff4s.back()->getTopLevel();

		_rows.push_back(&top.getList(kGFF4G2DARowList));

		_rowStarts.push_back(_rowCount);
		_rowCount += _rows.back()->size();

		Columns columns = &top.getList(kGFF4G2DAColumnList);
		if (columns->size() != _columns->size())
			throw Common::Exception("Column counts don't match (%u vs. %u)",
			                        (uint)columns->size(), (uint)_columns->size());

		for (size_t i = 0; i < columns->size(); i++) {
			const uint32 hash1 = (uint32) (* columns)[i]->getUint(kGFF4G2DAColumnHash);
			const uint32 hash2 = (uint32) (*_columns)[i]->getUint(kGFF4G2DAColumnHash);

			const uint32 type1 = identifyType( columns, _rows.back(), i);
			const uint32 type2 = identifyType(_columns, _rows[0]    , i);

			if ((hash1 != hash2) || (type1 != type2))
				throw Common::Exception("Columns don't match (%u: %u+%u vs. %u+%u)", (uint) i,
				                        hash1, type1, hash2, type2);
		}

	} catch (Common::Exception &e) {
		clear();

		e.add("Failed adding GDA file");
		throw;
	}
}

void MergeTreePartition::serializeText(const MergeTreeData & storage, WriteBuffer & out, const FormatSettings & format_settings) const
{
    size_t key_size = storage.partition_key_sample.columns();

    if (key_size == 0)
    {
        writeCString("tuple()", out);
    }
    else if (key_size == 1)
    {
        const DataTypePtr & type = storage.partition_key_sample.getByPosition(0).type;
        auto column = type->createColumn();
        column->insert(value[0]);
        type->serializeAsText(*column, 0, out, format_settings);
    }
    else
    {
        DataTypes types;
        Columns columns;
        for (size_t i = 0; i < key_size; ++i)
        {
            const auto & type = storage.partition_key_sample.getByPosition(i).type;
            types.push_back(type);
            auto column = type->createColumn();
            column->insert(value[i]);
            columns.push_back(std::move(column));
        }

        DataTypeTuple tuple_type(types);
        auto tuple_column = ColumnTuple::create(columns);
        tuple_type.serializeText(*tuple_column, 0, out, format_settings);
    }
}

void MergeJoin::InitializeFlatLayout(IRelationalOperator * lChild, 
				     IRelationalOperator * rChild,
				     const Columns & joinColumns)
{
  m_child[LEFT] = lChild;
  m_child[RIGHT] = rChild;

  /* create (left|right)+(proj|join) schemas. */
  for (int i = 0; i < N_BRANCHES; i++)
    {
      int idx = i * 2;
      Schema * joinSchema = new Schema();

      m_consumed[i] = true;
      m_inBuffer[i] = BufferManager::getInstance()->allocate();
      
      /* create projection & join schemas */
      m_tuple[idx|PROJ].schema(m_child[i]->schema());
      m_tuple[idx|JOIN].schema(m_child[i]->schema());

      for (int j = 0; j < joinColumns.count(); j++)
	{ 
	  const Column * column = joinColumns.at(j);
	  const Schema * schema = m_tuple[idx|PROJ].schema();
	  
	  if (schema->contains(column->m_qualified_name))
	    {
	      const Attribute * a = (*schema)[column->m_qualified_name];
	      m_joinCols[idx|PROJ].push_back(a);
	      m_joinCols[idx|JOIN].push_back(a);
	    }  
	}

      m_tuple[idx|PROJ].m_data = new byte[m_tuple[idx|PROJ].schema()->rsize()];
      m_tuple[idx|JOIN].m_data = new byte[m_tuple[idx|JOIN].schema()->rsize()];

      m_tsr[i] = new TupleStreamReader(*m_inBuffer[i]);
    }
  
  /* create merged schema */
  for (int i = 0; i < N_BRANCHES; i++)
    {
      int idx = i * 2;
      for (int j = 0; j < m_tuple[idx].schema()->nitems(); j++)
	{
	  m_schema.add(m_tuple[idx].schema()->at(j));
	}
    }
  
  m_data = new byte[m_schema.rsize()];
  m_buffer = BufferManager::getInstance()->allocate();
  m_tsw = new TupleStreamWriter(*m_buffer, m_schema.rsize());
}

FunctionArrayIntersect::UnpackedArrays FunctionArrayIntersect::prepareArrays(const Columns & columns) const
{
    UnpackedArrays arrays;

    size_t columns_number = columns.size();
    arrays.is_const.assign(columns_number, false);
    arrays.null_maps.resize(columns_number);
    arrays.offsets.resize(columns_number);
    arrays.nested_columns.resize(columns_number);

    for (auto i : ext::range(0, columns_number))
    {
        auto argument_column = columns[i].get();
        if (auto argument_column_const = typeid_cast<const ColumnConst *>(argument_column))
        {
            arrays.is_const[i] = true;
            argument_column = argument_column_const->getDataColumnPtr().get();
        }

        if (auto argument_column_array = typeid_cast<const ColumnArray *>(argument_column))
        {
            arrays.offsets[i] = &argument_column_array->getOffsets();
            arrays.nested_columns[i] = &argument_column_array->getData();
            if (auto column_nullable = typeid_cast<const ColumnNullable *>(arrays.nested_columns[i]))
            {
                arrays.null_maps[i] = &column_nullable->getNullMapData();
                arrays.nested_columns[i] = &column_nullable->getNestedColumn();
            }
        }
        else
            throw Exception{"Arguments for function " + getName() + " must be arrays.", ErrorCodes::LOGICAL_ERROR};
    }

    return arrays;
}

uint32 GDAFile::identifyType(const Columns &columns, const Row &rows, size_t column) const {
	if (!columns || (column >= columns->size()) || !(*columns)[column])
		return -1;

	if ((*columns)[column]->hasField(kGFF4G2DAColumnType))
		return (uint32) (*columns)[column]->getUint(kGFF4G2DAColumnType, -1);

	if (!rows || rows->empty() || !(*rows)[0])
		return -1;

	GFF4Struct::FieldType fieldType = (*rows)[0]->getFieldType(kGFF4G2DAColumn1 + column);

	switch (fieldType) {
		case GFF4Struct::kFieldTypeString:
			return 0;

		case GFF4Struct::kFieldTypeUint:
		case GFF4Struct::kFieldTypeSint:
			return 1;

		case GFF4Struct::kFieldTypeDouble:
			return 2;

		default:
			break;
	}

	return -1;
}

void Retrieve_Template::intitialize(QString table,Columns columns,QString where){
        this->query=new QSqlQuery (theSarf->db);
	QString cols;
	for(int i=0;i<columns.count();i++)
	{
		if (i!=0)
			cols.append(" ,");
		cols.append(columns[i]);
	}
	QString stmt( "SELECT %2 FROM %1 %3");
	stmt=stmt.arg(table).arg(cols).arg(where==""?"":QString("WHERE ").append(where) );
	err= (!execute_query(stmt,*(this->query)));
	if (!err)
		this->columns=columns;
}

void ExternalQueryBuilder::composeKeyTuple(const Columns & key_columns, const size_t row, WriteBuffer & out) const
{
    writeString("(", out);

    const auto keys_size = key_columns.size();
    auto first = true;
    for (const auto i : ext::range(0, keys_size))
    {
        if (!first)
            writeString(", ", out);

        first = false;
        (*dict_struct.key)[i].type->serializeAsTextQuoted(*key_columns[i], row, out, format_settings);
    }

    writeString(")", out);
}

void ExternalQueryBuilder::composeKeyCondition(const Columns & key_columns, const size_t row, WriteBuffer & out) const
{
    writeString("(", out);

    const auto keys_size = key_columns.size();
    auto first = true;
    for (const auto i : ext::range(0, keys_size))
    {
        if (!first)
            writeString(" AND ", out);

        first = false;

        const auto & key_description = (*dict_struct.key)[i];

        /// key_i=value_i
        writeString(key_description.name, out);
        writeString("=", out);
        key_description.type->serializeAsTextQuoted(*key_columns[i], row, out, format_settings);
    }

    writeString(")", out);
}

void detectData(const string& dataFile, Asset*& calls, Asset*& puts, const string& writeTo,
		long lastDate, double lastByte){ //function used to detect data if data contains the cp_flag
										 //function returns number of bytes extracted.

	ifstream inFile(dataFile.c_str()); // object for handling file input
	string row;
	stringstream srow;
	getline(inFile,row);
	srow.str(row); //replace old row with new row;
	//used to find out what column I am in, and thus what deque to extract the data to.
	Columns columnHeaders; //instantiate an object of Columns class.

	//find out if we are using a european or american style csv file. Following conditions
	char delimiter = getDelim(row, dataFile);


	columnHeaders.setColNbr(srow, delimiter); //set column numbers
	columnHeaders.checkValid(dataFile); //check if data is valid according to column numbers
	int cpColNbr = columnHeaders.getCpColNbr();
	string callOrPutRow; //used to determine what type of option is contained in row
	int i = 0; //used for collOrPutRow

	if ((calls == NULL)){ // we initialize here the proper options objects to contain our data. This is done only at the beginning of a file.


		vector<Asset**>dummyAssets; dummyAssets.push_back(&calls); dummyAssets.push_back(&puts);//just a dummy assets used for initiateAsset.
		columnHeaders.initiateAsset(dummyAssets, dataFile); //find what the Assets object will point to depending on the rows identity.
		//uses pointer to pointer, so that the copy made automatically is not a copy of our pointer to initialize, but a copy to
		//the pointer to those addresses.

		calls->setExFromByte((double)inFile.tellg());  //skip first line for rest of code as I am starting at beginning of file.
													   //!!!!information has to be set in both objects.
		puts->setExFromByte((double)inFile.tellg());
		calls->setWriteTo(writeTo); //setting folder to write to, using data passed as argument to detectData.
		puts->setWriteTo(writeTo);
	}


	//we first detect what type of options object we are dealing with
	//and save all the data to two single deques of row objects in order to sort columns in parallel.

	bool optionsOrNot = calls->getIsOption();

	////////////////////////////////////////////////////////////////////
	//if Options data was entered, find out what kind of options it was.
	////////////////////////////////////////////////////////////////////
	if (optionsOrNot == true) {

		//Skip to wanted position, after first line if new file, and as of last accessed byte
		//if return file.
		inFile.seekg(calls->getExFromByte(), ios_base::beg);
		double before = time(0);
		while ((inFile.eof() != 1)){ //while I am in the file

			getline(inFile, row); //get next line.
			srow.clear(); //clear eofbit set from last line.
			srow.str(row); //replace old row with new row.

			for(i = 0; i < cpColNbr; ++i)
				getline(srow, callOrPutRow, delimiter);

			srow.seekg(0); //reset row for srow

			if(callOrPutRow.find("C") != string::npos){ //we have a call
				calls->extractRow(srow, delimiter); //row is extracted and put in temporary values
				//line is now completely extracted. Now dealing with file size and multiple files part
				if(calls->fileSizeManagement(lastDate, lastByte, inFile) == true){ //file read status set only for calls.
					puts->setExToDate(calls->getExToDate());
					puts->setExFromByte(calls->getExFromByte());
					break; //break only if member function returned to break, otherwise, do nothing.
				}
				calls->writeCurrValues();
			}
			else{
				puts->extractRow(srow, delimiter); //row is extracted and put in temporary values
				//line is now completely extracted. Now dealing with file size and multiple files part
				if(puts->fileSizeManagement(lastDate, lastByte, inFile) == true){ //file read status set only for calls.
					calls->setExToDate(puts->getExToDate());
					calls->setExFromByte(puts->getExFromByte());
					break; //break only if member function returned to break, otherwise, do nothing.
				}
				puts->writeCurrValues();
			}

			//Done with dealing with filesize and multiple files part.



		}
		cout << "It took " << time(0) - before << " seconds to extract " << endl;

		before = time(0);

		calls->addExRows();
		puts->addExRows();

		before = time(0);
		//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		//Program will quickly sort the data. This is currently done whether data needs sorting or not.
		//So that all the times where data is already sorted, there is no need to even go through the deque of objects.
//.........这里部分代码省略.........

void PrintTo(const Columns& bar, ::std::ostream* os) {
  PrintTo(bar.to_string(), os);
}

ColumnPtr recursiveLowCardinalityConversion(const ColumnPtr & column, const DataTypePtr & from_type, const DataTypePtr & to_type)
{
    if (!column)
        return column;

    if (from_type->equals(*to_type))
        return column;

    if (const auto * column_const = typeid_cast<const ColumnConst *>(column.get()))
        return ColumnConst::create(recursiveLowCardinalityConversion(column_const->getDataColumnPtr(), from_type, to_type),
                                   column_const->size());

    if (const auto * low_cardinality_type = typeid_cast<const DataTypeLowCardinality *>(from_type.get()))
    {
        if (to_type->equals(*low_cardinality_type->getDictionaryType()))
            return column->convertToFullColumnIfLowCardinality();
    }

    if (const auto * low_cardinality_type = typeid_cast<const DataTypeLowCardinality *>(to_type.get()))
    {
        if (from_type->equals(*low_cardinality_type->getDictionaryType()))
        {
            auto col = low_cardinality_type->createColumn();
            static_cast<ColumnLowCardinality &>(*col).insertRangeFromFullColumn(*column, 0, column->size());
            return std::move(col);
        }
    }

    if (const auto * from_array_type = typeid_cast<const DataTypeArray *>(from_type.get()))
    {
        if (const auto * to_array_type = typeid_cast<const DataTypeArray *>(to_type.get()))
        {
            const auto * column_array = typeid_cast<const ColumnArray *>(column.get());
            if (!column_array)
                throw Exception("Unexpected column " + column->getName() + " for type " + from_type->getName(),
                                ErrorCodes::ILLEGAL_COLUMN);

            auto & nested_from = from_array_type->getNestedType();
            auto & nested_to = to_array_type->getNestedType();

            return ColumnArray::create(
                    recursiveLowCardinalityConversion(column_array->getDataPtr(), nested_from, nested_to),
                    column_array->getOffsetsPtr());
        }
    }

    if (const auto * from_tuple_type = typeid_cast<const DataTypeTuple *>(from_type.get()))
    {
        if (const auto * to_tuple_type = typeid_cast<const DataTypeTuple *>(to_type.get()))
        {
            const auto * column_tuple = typeid_cast<const ColumnTuple *>(column.get());
            if (!column_tuple)
                throw Exception("Unexpected column " + column->getName() + " for type " + from_type->getName(),
                                ErrorCodes::ILLEGAL_COLUMN);

            Columns columns = column_tuple->getColumns();
            auto & from_elements = from_tuple_type->getElements();
            auto & to_elements = to_tuple_type->getElements();
            for (size_t i = 0; i < columns.size(); ++i)
            {
                auto & element = columns[i];
                element = recursiveLowCardinalityConversion(element, from_elements.at(i), to_elements.at(i));
            }
            return ColumnTuple::create(columns);
        }
    }

    throw Exception("Cannot convert: " + from_type->getName() + " to " + to_type->getName(), ErrorCodes::TYPE_MISMATCH);
}

BlockInputStreamPtr LibraryDictionarySource::loadKeys(const Columns & key_columns, const std::vector<std::size_t> & requested_rows)
{
    LOG_TRACE(log, "loadKeys " << toString() << " size = " << requested_rows.size());

    auto holder = std::make_unique<ClickHouseLibrary::Row[]>(key_columns.size());
    std::vector<std::unique_ptr<ClickHouseLibrary::Field[]>> column_data_holders;
    for (size_t i = 0; i < key_columns.size(); ++i)
    {
        auto cell_holder = std::make_unique<ClickHouseLibrary::Field[]>(requested_rows.size());
        for (size_t j = 0; j < requested_rows.size(); ++j)
        {
            auto data_ref = key_columns[i]->getDataAt(requested_rows[j]);
            cell_holder[j] = ClickHouseLibrary::Field{.data = static_cast<const void *>(data_ref.data), .size = data_ref.size};
        }
        holder[i]
            = ClickHouseLibrary::Row{.data = static_cast<ClickHouseLibrary::Field *>(cell_holder.get()), .size = requested_rows.size()};

        column_data_holders.push_back(std::move(cell_holder));
    }

    ClickHouseLibrary::Table request_cols{.data = static_cast<ClickHouseLibrary::Row *>(holder.get()), .size = key_columns.size()};

    void * data_ptr = nullptr;
    /// Get function pointer before dataNew call because library->get may throw.
    auto func_loadKeys = library->get<void * (*)(decltype(data_ptr), decltype(&settings->strings), decltype(&request_cols))>(
        "ClickHouseDictionary_v3_loadKeys");
    data_ptr = library->get<decltype(data_ptr) (*)(decltype(lib_data))>("ClickHouseDictionary_v3_dataNew")(lib_data);
    auto data = func_loadKeys(data_ptr, &settings->strings, &request_cols);
    auto block = dataToBlock(description.sample_block, data);
    SCOPE_EXIT(library->get<void (*)(decltype(lib_data), decltype(data_ptr))>("ClickHouseDictionary_v3_dataDelete")(lib_data, data_ptr));
    return std::make_shared<OneBlockInputStream>(block);
}

bool LibraryDictionarySource::isModified() const
{
    if (auto func_isModified
        = library->tryGet<bool (*)(decltype(lib_data), decltype(&settings->strings))>("ClickHouseDictionary_v3_isModified"))
        return func_isModified(lib_data, &settings->strings);
    return true;
}

bool LibraryDictionarySource::supportsSelectiveLoad() const
{
    if (auto func_supportsSelectiveLoad
        = library->tryGet<bool (*)(decltype(lib_data), decltype(&settings->strings))>("ClickHouseDictionary_v3_supportsSelectiveLoad"))
        return func_supportsSelectiveLoad(lib_data, &settings->strings);
    return true;
}

DictionarySourcePtr LibraryDictionarySource::clone() const
{
    return std::make_unique<LibraryDictionarySource>(*this);
}

std::string LibraryDictionarySource::toString() const
{
    return path;
}

void registerDictionarySourceLibrary(DictionarySourceFactory & factory)
{
    auto createTableSource = [=](const DictionaryStructure & dict_struct,
                                 const Poco::Util::AbstractConfiguration & config,
                                 const std::string & config_prefix,
                                 Block & sample_block,
                                 const Context &) -> DictionarySourcePtr
    {
        return std::make_unique<LibraryDictionarySource>(dict_struct, config, config_prefix + ".library", sample_block);
    };
    factory.registerSource("library", createTableSource);
}

}