C++ DocumentInfo类代码示例

bool OnDiskHandler::fileMoved(const string &fileName, const string &previousFileName)
{
	bool handledEvent = false;

#ifdef DEBUG
	cout << "OnDiskHandler::fileMoved: " << fileName << endl;
#endif
	pthread_mutex_lock(&m_mutex);
	unsigned int oldDocId = m_index.hasDocument(string("file://") + previousFileName);
	if (oldDocId > 0)
	{
		DocumentInfo docInfo;

		if (m_index.getDocumentInfo(oldDocId, docInfo) == true)
		{
			// Change the location
			docInfo.setLocation(string("file://") + fileName);

			handledEvent = replaceFile(oldDocId, docInfo);
		}
	}
	pthread_mutex_unlock(&m_mutex);

	return handledEvent;
}

//
// Updates a document's properties.
//
void IndexTree::updateDocumentInfo(unsigned int docId, const DocumentInfo &docInfo)
{
	if (docId == 0)
	{
		return;
	}

	// Go through the list of indexed documents
	TreeModel::Children children = m_refStore->children();
	for (TreeModel::Children::iterator iter = children.begin(); iter != children.end(); ++iter)
	{
		TreeModel::Row row = *iter;

		if (docId == row[m_indexColumns.m_id])
		{
			row[m_indexColumns.m_text] = to_utf8(docInfo.getTitle());
			row[m_indexColumns.m_type] = to_utf8(docInfo.getType());
			row[m_indexColumns.m_language] = to_utf8(docInfo.getLanguage());
			row[m_indexColumns.m_timestamp] = to_utf8(docInfo.getTimestamp());
#ifdef DEBUG
			cout << "IndexTree::updateDocumentInfo: language now " << docInfo.getLanguage() << endl;
#endif
			break;
		}
	}
}

bool ThreadsManager::index_document(const DocumentInfo &docInfo)
{
	string location(docInfo.getLocation());

	if (location.empty() == true)
	{
		// Nothing to do
		return false;
	}

	// If the document is a mail message, we can't index it again
	Url urlObj(location);
	if (urlObj.getProtocol() == "mailbox")
	{
		return false;
	}

	// Is the document being indexed/updated ?
	if (write_lock_lists() == true)
	{
		bool beingProcessed = true;

		if (m_beingIndexed.find(location) == m_beingIndexed.end())
		{
			m_beingIndexed.insert(location);
			beingProcessed = false;
		}

		unlock_lists();

		if (beingProcessed == true)
		{
			// FIXME: we may have to set labels on this document
			return false;
		}
	}

	// Is it an update ?
	IndexInterface *pIndex = PinotSettings::getInstance().getIndex(m_defaultIndexLocation);
	if (pIndex == NULL)
	{
		return false;
	}

	unsigned int docId = pIndex->hasDocument(docInfo.getLocation());
	if (docId > 0)
	{
		// Yes, it is
		start_thread(new IndexingThread(docInfo, docId, m_defaultIndexLocation));
	}
	else
	{
		// This is a new document
		start_thread(new IndexingThread(docInfo, docId, m_defaultIndexLocation));
	}
	delete pIndex;

	return true;
}

/// Returns a document's properties.
bool XapianIndex::getDocumentInfo(unsigned int docId, DocumentInfo &docInfo) const
{
	bool foundDocument = false;

	if (docId == 0)
	{
		return false;
	}

	XapianDatabase *pDatabase = XapianDatabaseFactory::getDatabase(m_databaseName, false);
	if (pDatabase == NULL)
	{
		cerr << "Bad index " << m_databaseName << endl;
		return false;
	}

	try
	{
		Xapian::Database *pIndex = pDatabase->readLock();
		if (pIndex != NULL)
		{
			Xapian::Document doc = pIndex->get_document(docId);

			// Get the current document data
			string record = doc.get_data();
			if (record.empty() == false)
			{
				string language = Languages::toLocale(StringManip::extractField(record, "language=", ""));

				docInfo = DocumentInfo(StringManip::extractField(record, "caption=", "\n"),
					StringManip::extractField(record, "url=", "\n"),
					StringManip::extractField(record, "type=", "\n"),
					language);
				docInfo.setTimestamp(StringManip::extractField(record, "timestamp=", "\n"));
#ifdef DEBUG
				cout << "XapianIndex::getDocumentInfo: language is "
					<< docInfo.getLanguage() << endl;
#endif
				foundDocument = true;
			}
		}
	}
	catch (const Xapian::Error &error)
	{
		cerr << "Couldn't get document properties: " << error.get_msg() << endl;
	}
	catch (...)
	{
		cerr << "Couldn't get document properties, unknown exception occured" << endl;
	}
	pDatabase->unlock();

	return foundDocument;
}

bool MetaDataBackup::setAttribute(const DocumentInfo &docInfo,
	const string &name, const string &value, bool noXAttr)
{
	string url(docInfo.getLocation());
	string urlWithIPath(docInfo.getLocation(true));
#ifdef HAVE_ATTR_XATTR_H
	Url urlObj(url);

	// If the file is local and isn't a nested document, use an extended attribute
	if ((noXAttr == false) &&
		(urlObj.isLocal() == true) &&
		(docInfo.getInternalPath().empty() == true))
	{
		string fileName(url.substr(urlObj.getProtocol().length() + 3));
		string attrName("pinot." + name);

		// Set an attribute, and add an entry in the table
		if (setxattr(fileName.c_str(), attrName.c_str(),
			value.c_str(), (size_t)value.length(), 0) != 0)
		{
#ifdef DEBUG
			cout << "MetaDataBackup::setAttribute: setxattr failed with " << strerror(errno) << endl;
#endif
		}
	}
#endif
	bool update = false, success = false;

	// Is there already such an item for this URL ?
	SQLResults *results = executeStatement("SELECT Url FROM MetaDataBackup \
		WHERE Url='%q' AND Name='%q';",
		Url::escapeUrl(urlWithIPath).c_str(), name.c_str());
	if (results != NULL)
	{
		SQLRow *row = results->nextRow();
		if (row != NULL)
		{
			// Yes, there is
			update = true;

			delete row;
		}

		delete results;
	}

	if (update == false)
	{
		results = executeStatement("INSERT INTO MetaDataBackup \
			VALUES('%q', '%q', '%q');",
			Url::escapeUrl(urlWithIPath).c_str(), name.c_str(), value.c_str());
	}

bool XapianIndex::prepareDocument(const DocumentInfo &info, Xapian::Document &doc,
	Xapian::termcount &termPos) const
{
	string title(info.getTitle());
	string location(info.getLocation());
	Url urlObj(location);

	// Add a magic term :-)
	doc.add_term(MAGIC_TERM);

	// Index the title with and without prefix S
	if (title.empty() == false)
	{
		Document titleDoc;
		titleDoc.setData(title.c_str(), title.length());
		Tokenizer titleTokens(&titleDoc);
		addTermsToDocument(titleTokens, doc, "S", termPos, STORE_UNSTEM);
		titleTokens.rewind();
		addTermsToDocument(titleTokens, doc, "", termPos, m_stemMode);
	}

	// Index the full URL with prefix U
	doc.add_term(limitTermLength(string("U") + location, true));
	// ...the host name and included domains with prefix H
	string hostName(StringManip::toLowerCase(urlObj.getHost()));
	if (hostName.empty() == false)
	{
		doc.add_term(limitTermLength(string("H") + hostName, true));
		string::size_type dotPos = hostName.find('.');
		while (dotPos != string::npos)
		{
			doc.add_term(limitTermLength(string("H") + hostName.substr(dotPos + 1), true));

			// Next
			dotPos = hostName.find('.', dotPos + 1);
		}
	}
	// ...and the file name with prefix P
	string fileName(urlObj.getFile());
	if (fileName.empty() == false)
	{
		doc.add_term(limitTermLength(string("P") + StringManip::toLowerCase(fileName), true));
	}
	// Finally, add the language code with prefix L
	doc.add_term(string("L") + Languages::toCode(m_stemLanguage));

	setDocumentData(doc, info, m_stemLanguage);

	return true;
}

/// Updates a document's properties.
bool DBusXapianIndex::updateDocumentInfo(unsigned int docId, const DocumentInfo &docInfo)
{
	bool updated = false;

	DBusGConnection *pBus = getBusConnection();
	if (pBus == NULL)
	{
		return false;
	}

	DBusGProxy *pBusProxy = getBusProxy(pBus);
	if (pBusProxy == NULL)
	{
		cerr << "DBusXapianIndex::updateDocumentInfo: couldn't get bus proxy" << endl;
		return false;
	}

	GError *pError = NULL;
	const char *pTitle = docInfo.getTitle().c_str();
	const char *pLocation = docInfo.getLocation().c_str();
	const char *pType = docInfo.getType().c_str();
	string language(Languages::toEnglish(docInfo.getLanguage()));
	const char *pLanguage = language.c_str();

	if (dbus_g_proxy_call(pBusProxy, "SetDocumentInfo", &pError,
		G_TYPE_UINT, docId,
		G_TYPE_STRING, pTitle,
		G_TYPE_STRING, pLocation,
		G_TYPE_STRING, pType,
		G_TYPE_STRING, pLanguage,
		G_TYPE_INVALID,
		G_TYPE_UINT, &docId,
		G_TYPE_INVALID) == TRUE)
	{
		updated = true;
	}
	else
	{
		if (pError != NULL)
		{
			cerr << "DBusXapianIndex::updateDocumentInfo: " << pError->message << endl;
			g_error_free(pError);
		}
	}

	g_object_unref(pBusProxy);
	// FIXME: don't we have to call dbus_g_connection_unref(pBus); ?

	return updated;
}

void DirectoryScannerThread::foundFile(const DocumentInfo &docInfo)
{
	if ((docInfo.getLocation().empty() == true) ||
		(m_done == true))
	{
		return;
	}

	stringstream labelStream;

	// This identifies the source
	labelStream << "X-SOURCE" << m_sourceId;
#ifdef DEBUG
	cout << "DirectoryScannerThread::foundFile: source label for " << docInfo.getLocation() << " is " << labelStream.str() << endl;
#endif
	m_signalFileFound(docInfo, labelStream.str(), false);
}

void XapianIndex::setDocumentData(const DocumentInfo &info, Xapian::Document &doc,
	const string &language) const
{
	string title(info.getTitle());
	string timestamp(info.getTimestamp());
	char timeStr[64];
	time_t timeT = TimeConverter::fromTimestamp(timestamp);

	// Set the document data omindex-style
	string record = "url=";
	record += info.getLocation();
	// The sample will be generated at query time
	record += "\nsample=";
	record += "\ncaption=";
	if (badField(title) == true)
	{
		// Modify the title if necessary
		string::size_type pos = title.find("=");
		while (pos != string::npos)
		{
			title[pos] = ' ';
			pos = title.find("=", pos + 1);
		}
#ifdef DEBUG
		cout << "XapianIndex::setDocumentData: modified title" << endl;
#endif
	}
	record += title;
	record += "\ntype=";
	record += info.getType();
	// Append a timestamp, in a format compatible with Omega
	record += "\nmodtime=";
	snprintf(timeStr, 64, "%ld", timeT);
	record += timeStr;
	// ...and the language
	record += "\nlanguage=";
	record += StringManip::toLowerCase(language);
#ifdef DEBUG
	cout << "XapianIndex::setDocumentData: document data is " << record << endl;
#endif
	doc.set_data(record);

	// Add this value to allow sorting by date
	doc.add_value(0, StringManip::integerToBinaryString((uint32_t)timeT));
}

void XapianIndex::setDocumentData(Xapian::Document &doc, const DocumentInfo &info,
	const string &language) const
{
	string title(info.getTitle());
	string timestamp(info.getTimestamp());
	char timeStr[64];

	// Set the document data omindex-style
	string record = "url=";
	record += info.getLocation();
	// The sample will be generated at query time
	record += "\nsample=";
	record += "\ncaption=";
	if (badField(title) == true)
	{
		// Modify the title if necessary
		string::size_type pos = title.find("=");
		while (pos != string::npos)
		{
			title[pos] = ' ';
			pos = title.find("=", pos + 1);
		}
#ifdef DEBUG
		cout << "XapianIndex::setDocumentData: modified title" << endl;
#endif
	}
	record += title;
	record += "\ntype=";
	record += info.getType();
	// Append a timestamp
	record += "\ntimestamp=";
	record += timestamp;
	// ...and the language
	record += "\nlanguage=";
	record += language;
#ifdef DEBUG
	cout << "XapianIndex::setDocumentData: document data is " << record << endl;
#endif
	doc.set_data(record);

	// Add this value to allow sorting by date
	snprintf(timeStr, 64, "%d", TimeConverter::fromTimestamp(timestamp));
	doc.add_value(0, timeStr);
}

/// Updates a document's properties.
bool XapianIndex::updateDocumentInfo(unsigned int docId, const DocumentInfo &docInfo)
{
	bool updated = false;

	if (docId == 0)
	{
		return false;
	}

	XapianDatabase *pDatabase = XapianDatabaseFactory::getDatabase(m_databaseName, false);
	if (pDatabase == NULL)
	{
		cerr << "Bad index " << m_databaseName << endl;
		return false;
	}

	try
	{
		Xapian::WritableDatabase *pIndex = pDatabase->writeLock();
		if (pIndex != NULL)
		{
			Xapian::Document doc = pIndex->get_document(docId);

#ifdef DEBUG
			cout << "XapianIndex::updateDocumentInfo: language is " << docInfo.getLanguage() << endl;
#endif
			// Update the document data with the current language
			setDocumentData(doc, docInfo, docInfo.getLanguage());
			pIndex->replace_document(docId, doc);
			updated = true;
		}
	}
	catch (const Xapian::Error &error)
	{
		cerr << "Couldn't update document properties: " << error.get_msg() << endl;
	}
	catch (...)
	{
		cerr << "Couldn't update document properties, unknown exception occured" << endl;
	}
	pDatabase->unlock();

	return updated;
}

bool OnDiskHandler::replaceFile(unsigned int docId, DocumentInfo &docInfo)
{
	FilterWrapper wrapFilter(&m_index);

	// Unindex the destination file
	wrapFilter.unindexDocument(docInfo.getLocation());

	// Update the document info
	return m_index.updateDocumentInfo(docId, docInfo);
}

bool DocumentInfo::operator<(const DocumentInfo& other) const
{
	string thisUrl(getField("url"));
	string otherUrl(other.getField("url"));

	if (thisUrl < otherUrl)
	{
		return true;
	}
	else if (thisUrl == otherUrl)
	{
		if (getField("ipath") < other.getField("ipath"))
		{
			return true;
		}
	}

	return false;
}

ustring ThreadsManager::index_document(const DocumentInfo &docInfo)
{
	string location(docInfo.getLocation());

	if (location.empty() == true)
	{
		// Nothing to do
		return "";
	}

	// If the document is a mail message, we can't index it again
	Url urlObj(location);
	if (urlObj.getProtocol() == "mailbox")
	{
		return _("Can't index mail here");
	}

	// Is the document being indexed/updated ?
	if (write_lock_lists() == true)
	{
		bool beingProcessed = true;

		if (m_beingIndexed.find(location) == m_beingIndexed.end())
		{
			m_beingIndexed.insert(location);
			beingProcessed = false;
		}

		unlock_lists();

		if (beingProcessed == true)
		{
			// FIXME: we may have to set labels on this document
			ustring status(location);
			status += " ";
			status += _("is already being indexed");
			return status;
		}
	}

	// Is the document blacklisted ?
	if (PinotSettings::getInstance().isBlackListed(location) == true)
	{
		ustring status(location);
		status += " ";
		status += _("is blacklisted");
		return status;
	}

	start_thread(new IndexingThread(docInfo, m_defaultIndexLocation));

	return "";
}

void XapianIndex::setDocumentData(const DocumentInfo &info, Xapian::Document &doc,
                                  const string &language) const
{
    time_t timeT = TimeConverter::fromTimestamp(info.getTimestamp());

    // Add this value to allow sorting by date
    doc.add_value(0, StringManip::integerToBinaryString((uint32_t)timeT));

    DocumentInfo docCopy(info);
    docCopy.setLanguage(language);
    doc.set_data(XapianDatabase::propsToRecord(&docCopy));
}