C# Indexing.Index类代码示例

        public IndexViewModel(Index index, IDatabase database, IRavenScreen parent)
        {
            _index = index;
            Database = database;

            ParentRavenScreen = parent;
            DisplayName = "Edit Index";
            CompositionInitializer.SatisfyImports(this);
        }

		public SuggestionQueryIndexExtension(Index indexInstance, WorkContext workContext, string key, 
			StringDistance distanceType, bool isRunInMemory, string field, float accuracy)
		{
			_indexInstance = indexInstance;
			this.workContext = workContext;
			this.field = field;

			if (isRunInMemory)
			{
				directory = new RAMDirectory();
			}
			else
			{
				directory = FSDirectory.Open(new DirectoryInfo(key));
			}

			spellChecker = new SpellChecker.Net.Search.Spell.SpellChecker(directory, null);
			spellChecker.SetAccuracy(accuracy);
			spellChecker.setStringDistance(distanceType);
			_operationText = "Suggestions for " + field + " " + distanceType + " (" + accuracy + ")";
		}

			public IndexQueryOperation(Index parent, IndexQuery indexQuery, Func<IndexQueryResult, bool> shouldIncludeInResults, FieldsToFetch fieldsToFetch, OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers)
			{
				this.parent = parent;
				this.indexQuery = indexQuery;
				this.shouldIncludeInResults = shouldIncludeInResults;
				this.fieldsToFetch = fieldsToFetch;
				this.indexQueryTriggers = indexQueryTriggers;

				if (fieldsToFetch.IsDistinctQuery)
					alreadyReturned = new HashSet<RavenJObject>(new RavenJTokenEqualityComparer());
				
			}

        private void ApplyPrecomputedBatchForNewIndex(Index index, AbstractViewGenerator generator, int pageSize, CancellationTokenSource cts)
        {
            PrecomputedIndexingBatch result = null;

            var docsToIndex = new List<JsonDocument>();
            TransactionalStorage.Batch(actions =>
            {
                var query = GetQueryForAllMatchingDocumentsForIndex(generator);

                using (DocumentCacher.SkipSetDocumentsInDocumentCache())
                using (var linked = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, WorkContext.CancellationToken))
                using (var op = new QueryActions.DatabaseQueryOperation(Database, Constants.DocumentsByEntityNameIndex, new IndexQuery
                {
                    Query = query,
                    PageSize = pageSize
                }, actions, linked)
                {
                    ShouldSkipDuplicateChecking = true
                })
                {
                    op.Init();

                    //if we are working on a test index, apply the optimization anyway, as the index is capped by small number of results
                    if (index.IsTestIndex == false && op.Header.TotalResults > pageSize)
                    {
                        // we don't apply this optimization if the total number of results 
                        // to index is more than the max numbers to index in a single batch. 
                        // The idea here is that we need to keep the amount
                        // of memory we use to a manageable level even when introducing a new index to a BIG 
                        // database
                        try
                        {
                            cts.Cancel();
                            // we have to run just a little bit of the query to properly setup the disposal
                            op.Execute(o => { });
                        }
                        catch (OperationCanceledException)
                        {
                        }
                        return;
                    }

                    if (Log.IsDebugEnabled)
                    {
                        Log.Debug("For new index {0}, using precomputed indexing batch optimization for {1} docs", 
                            index, op.Header.TotalResults);
                    }
                        
                    var totalLoadedDocumentSize = 0;
                    const int totalSizeToCheck = 16 * 1024 * 1024; //16MB
                    var localLoadedDocumentSize = 0;
                    op.Execute(document =>
                    {
                        var metadata = document.Value<RavenJObject>(Constants.Metadata);
                        var key = metadata.Value<string>("@id");
                        var etag = Etag.Parse(metadata.Value<string>("@etag"));
                        var lastModified = DateTime.Parse(metadata.Value<string>(Constants.LastModified));
                        document.Remove(Constants.Metadata);
                        var serializedSizeOnDisk = metadata.Value<int>(Constants.SerializedSizeOnDisk);
                        metadata.Remove(Constants.SerializedSizeOnDisk);

                        var doc = new JsonDocument
                        {
                            DataAsJson = document,
                            Etag = etag,
                            Key = key,
                            SerializedSizeOnDisk = serializedSizeOnDisk,
                            LastModified = lastModified,
                            SkipDeleteFromIndex = true,
                            Metadata = metadata
                        };

                        docsToIndex.Add(doc);
                        totalLoadedDocumentSize += serializedSizeOnDisk;
                        localLoadedDocumentSize += serializedSizeOnDisk;

                        if (totalLoadedDocumentSize > Database.Configuration.MaxPrecomputedBatchTotalDocumentSizeInBytes)
                        {
                            var error = string.Format(
                                @"Aborting applying precomputed batch for index id: {0}, name: {1}
                                    because we have {2}mb of documents that were fetched
                                    and the configured max data to fetch is {3}mb",
                                index.indexId, index.PublicName, totalLoadedDocumentSize, 
                                Database.Configuration.MaxPrecomputedBatchTotalDocumentSizeInBytes/1024/1024);

                            //we are aborting operation, so don't keep the references
                            docsToIndex.Clear(); 
                            throw new TotalDataSizeExceededException(error);
                        }


                        if (localLoadedDocumentSize <= totalSizeToCheck)
                            return;

                        localLoadedDocumentSize = 0;

                        if (Database.Configuration.MemoryLimitForProcessingInMb > MemoryStatistics.AvailableMemoryInMb)
                        {
                            var error = string.Format(
                                @"Aborting applying precomputed batch for index id: {0}, name: {1}
//.........这里部分代码省略.........

        private void TryApplyPrecomputedBatchForNewIndex(Index index, IndexDefinition definition)
        {
            var generator = IndexDefinitionStorage.GetViewGenerator(definition.IndexId);
            if (generator.ForEntityNames.Count == 0)
            {
                // we don't optimize if we don't have what to optimize _on, we know this is going to return all docs.
                // no need to try to optimize that, then
				index.IsMapIndexingInProgress = false;
				return;
            }

            try
            {
                Task.Factory.StartNew(() => ApplyPrecomputedBatchForNewIndex(index, generator),
                    TaskCreationOptions.LongRunning)
                    .ContinueWith(t =>
                    {
                        if (t.IsFaulted)
                        {
                            Log.Warn("Could not apply precomputed batch for index " + index, t.Exception);
                        }
                        index.IsMapIndexingInProgress = false;
                        WorkContext.ShouldNotifyAboutWork(() => "Precomputed indexing batch for " + index.PublicName + " is completed");
                        WorkContext.NotifyAboutWork();

                    });
            }
            catch (Exception)
            {
                index.IsMapIndexingInProgress = false;
                throw;
            }
        }

		private void LoadExistingSuggestionsExtentions(string indexName, Index indexImplementation)
		{
			var suggestionsForIndex = Path.Combine(configuration.IndexStoragePath, "Raven-Suggestions", indexName);
			if (!Directory.Exists(suggestionsForIndex))
				return;

		    try
		    {
		        foreach (var directory in Directory.GetDirectories(suggestionsForIndex))
		        {
		            IndexSearcher searcher;
		            using (indexImplementation.GetSearcher(out searcher))
		            {
		                var key = Path.GetFileName(directory);
		                var decodedKey = MonoHttpUtility.UrlDecode(key);
		                var lastIndexOfDash = decodedKey.LastIndexOf('-');
						var accuracy = float.Parse(decodedKey.Substring(lastIndexOfDash + 1), CultureInfo.InvariantCulture);
		                var lastIndexOfDistance = decodedKey.LastIndexOf('-', lastIndexOfDash - 1);
		                StringDistanceTypes distanceType;
		                Enum.TryParse(decodedKey.Substring(lastIndexOfDistance + 1, lastIndexOfDash - lastIndexOfDistance - 1),
		                              true, out distanceType);
		                var field = decodedKey.Substring(0, lastIndexOfDistance);
		                var extension = new SuggestionQueryIndexExtension(
							indexImplementation,
		                    documentDatabase.WorkContext,
							Path.Combine(configuration.IndexStoragePath, "Raven-Suggestions", indexName, key),
		                    SuggestionQueryRunner.GetStringDistance(distanceType),
							searcher.IndexReader.Directory() is RAMDirectory,
		                    field,
		                    accuracy);
		                indexImplementation.SetExtension(key, extension);
		            }
		        }
		    }
		    catch (Exception e)
		    {
		        log.WarnException("Could not open suggestions for index " + indexName + ", resetting the index", e);
		        try
		        {
		            IOExtensions.DeleteDirectory(suggestionsForIndex);
		        }
		        catch (Exception)
		        {
		            // ignore the failure
		        }
		        throw;
		    }
		}

		private void CheckMapIndexState(IDictionary<string, string> commitData, IndexDefinition indexDefinition, Index index)
		{
			string value;
			Etag lastEtag = null;
			if (commitData != null && commitData.TryGetValue("LastEtag", out value))
				Etag.TryParse(value, out lastEtag); // etag will be null if parsing will fail

			var lastStoredEtag = GetLastEtagForIndex(index) ?? Etag.Empty;
			lastEtag = lastEtag ?? Etag.Empty;

			if (EtagUtil.IsGreaterThanOrEqual(lastEtag, lastStoredEtag))
				return;

			var now = SystemTime.UtcNow;
			ResetLastIndexedEtag(indexDefinition, lastEtag, now);
		}

 protected abstract bool ShouldSkipIndex(Index index);

 protected override bool ShouldSkipIndex(Index index)
 {
     return index.IsTestIndex ||
            index.IsMapIndexingInProgress; // precomputed? slow? it is already running, nothing to do with it for now;
 }

        private static void FlushIndex(Index value, bool onlyAddIndexError = false)
        {
            var sp = Stopwatch.StartNew();
            
            try
            {
                value.Flush(value.GetLastEtagFromStats());
            }
            catch (Exception e)
            {
                value.IncrementWriteErrors(e);
                log.WarnException(string.Format("Failed to flush {0} index: {1} (id: {2})",
                    GetIndexType(value.IsMapReduce), value.PublicName, value.IndexId), e);

                if (onlyAddIndexError)
                {
                    value.AddIndexFailedFlushError(e);
                    return;
                }

                throw;
            }

            if (log.IsDebugEnabled)
            {
                log.Debug("Flushed {0} index: {1} (id: {2}), took {3}ms",
                    GetIndexType(value.IsMapReduce), value.PublicName, value.IndexId, sp.ElapsedMilliseconds);
            }
        }

		protected override bool ShouldSkipIndex(Index index)
		{
			return index.IsTestIndex;
		}

		private JsonDocument[] GetJsonDocuments(MoreLikeThisQuery parameters, IndexSearcher searcher, Index index, string indexName, IEnumerable<ScoreDoc> hits, int baseDocId)
		{
			if (string.IsNullOrEmpty(parameters.DocumentId) == false)
			{
				var documentIds = hits
					.Where(hit => hit.Doc != baseDocId)
					.Select(hit => searcher.Doc(hit.Doc).Get(Constants.DocumentIdFieldName))
					.Where(x => x != null)
					.Distinct();

				return documentIds
					.Select(docId => database.Documents.Get(docId, null))
					.Where(it => it != null)
					.ToArray();
			}

			var fields = searcher.Doc(baseDocId).GetFields().Cast<AbstractField>().Select(x => x.Name).Distinct().ToArray();
			var etag = database.Indexes.GetIndexEtag(indexName, null);
			return hits
				.Where(hit => hit.Doc != baseDocId)
				.Select(hit => new JsonDocument
				{
					DataAsJson = Index.CreateDocumentFromFields(searcher.Doc(hit.Doc),
					                                            new FieldsToFetch(fields, false, index.IsMapReduce ? Constants.ReduceKeyFieldName : Constants.DocumentIdFieldName)),
					Etag = etag
				})
				.ToArray();
		}

			public IndexQueryOperation(
				Index parent,
				IndexQuery indexQuery,
				Func<IndexQueryResult, bool> shouldIncludeInResults,
				FieldsToFetch fieldsToFetch)
			{
				this.parent = parent;
				this.indexQuery = indexQuery;
				this.shouldIncludeInResults = shouldIncludeInResults;
				this.fieldsToFetch = fieldsToFetch;

				if (fieldsToFetch.IsDistinctQuery)
					alreadyReturned = new HashSet<JObject>(new JTokenEqualityComparer());
				
			}

		private void PerformSearch(IHttpContext context, string indexName, Index index, MoreLikeThisQueryParameters parameters)
		{
			IndexSearcher searcher;
			using (Database.IndexStorage.GetCurrentIndexSearcher(indexName, out searcher))
			{
				var documentQuery = new BooleanQuery();

				if (!string.IsNullOrEmpty(parameters.DocumentId))
				{
					documentQuery.Add(new TermQuery(new Term(Constants.DocumentIdFieldName, parameters.DocumentId)),
					                  Lucene.Net.Search.BooleanClause.Occur.MUST);
				}

				foreach (string key in parameters.MapGroupFields.Keys)
				{
					documentQuery.Add(new TermQuery(new Term(key, parameters.MapGroupFields[key])),
					                  Lucene.Net.Search.BooleanClause.Occur.MUST);
				}

				var td = searcher.Search(documentQuery, 1);

				// get the current Lucene docid for the given RavenDB doc ID
				if (td.ScoreDocs.Length == 0)
				{
					context.SetStatusToNotFound();
					context.WriteJson(new {Error = "Document " + parameters.DocumentId + " could not be found"});
					return;
				}

				var ir = searcher.GetIndexReader();
				var mlt = new RavenMoreLikeThis(ir);

				AssignParameters(mlt, parameters);

				if (!string.IsNullOrWhiteSpace(parameters.StopWordsDocumentId))
				{
					var stopWordsDoc = Database.Get(parameters.StopWordsDocumentId, null);
					if (stopWordsDoc == null)
					{
						context.SetStatusToNotFound();
						context.WriteJson(
							new
							{
								Error = "Stop words document " + parameters.StopWordsDocumentId + " could not be found"
							});
						return;
					}
					var stopWords = stopWordsDoc.DataAsJson.JsonDeserialization<StopWordsSetup>().StopWords;
					mlt.SetStopWords(new Hashtable(stopWords.ToDictionary(x => x.ToLower())));
				}

				var fieldNames = parameters.Fields ?? GetFieldNames(ir);
				mlt.SetFieldNames(fieldNames);

				var toDispose = new List<Action>();
				PerFieldAnalyzerWrapper perFieldAnalyzerWrapper = null;
				try
				{
					perFieldAnalyzerWrapper = index.CreateAnalyzer(new LowerCaseKeywordAnalyzer(), toDispose, true);
					mlt.SetAnalyzer(perFieldAnalyzerWrapper);

					var mltQuery = mlt.Like(td.ScoreDocs[0].doc);
					var tsdc = TopScoreDocCollector.create(context.GetPageSize(Database.Configuration.MaxPageSize), true);
					searcher.Search(mltQuery, tsdc);
					var hits = tsdc.TopDocs().ScoreDocs;
					var jsonDocuments = GetJsonDocuments(parameters, searcher, indexName, hits, td.ScoreDocs[0].doc);

					var result = new MultiLoadResult();

					var includedEtags = new List<byte>(jsonDocuments.SelectMany(x => x.Etag.Value.ToByteArray()));
					includedEtags.AddRange(Database.GetIndexEtag(indexName, null).ToByteArray());
					var loadedIds = new HashSet<string>(jsonDocuments.Select(x => x.Key));
					var addIncludesCommand = new AddIncludesCommand(Database, GetRequestTransaction(context), (etag, includedDoc) =>
					{
						includedEtags.AddRange(etag.ToByteArray());
						result.Includes.Add(includedDoc);
					}, context.Request.QueryString.GetValues("include") ?? new string[0], loadedIds);

					foreach (var jsonDocumet in jsonDocuments)
					{
						result.Results.Add(jsonDocumet.ToJson());
						addIncludesCommand.Execute(jsonDocumet.DataAsJson);
					}

					Guid computedEtag;
					using (var md5 = MD5.Create())
					{
						var computeHash = md5.ComputeHash(includedEtags.ToArray());
						computedEtag = new Guid(computeHash);
					}

					if (context.MatchEtag(computedEtag))
					{
						context.SetStatusToNotModified();
						return;
					}

					context.Response.AddHeader("ETag", computedEtag.ToString());
					context.WriteJson(result);
				}
//.........这里部分代码省略.........

		private void CheckMapIndexState(IDictionary<string, string> commitData, IndexDefinition indexDefinition, Index index)
		{
			string value;
			Etag lastEtag = null;
			if (commitData != null && commitData.TryGetValue("LastEtag", out value))
				Etag.TryParse(value, out lastEtag); // etag will be null if parsing will fail

			var lastStoredEtag = GetLastEtagForIndex(index) ?? Etag.Empty;
			lastEtag = lastEtag ?? Etag.Empty;

			if (EtagUtil.IsGreaterThanOrEqual(lastEtag, lastStoredEtag))
				return;

			log.Info(string.Format("Resetting index '{0} ({1})'. Last stored etag: {2}. Last commit etag: {3}.", indexDefinition.Name, index.indexId, lastStoredEtag, lastEtag));

			var now = SystemTime.UtcNow;
			ResetLastIndexedEtag(indexDefinition, lastEtag, now);
		}