C# SmugglerOptions.ExcludeExpired方法代码示例

		private async Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
		{
			var totalCount = 0;
			var lastReport = SystemTime.UtcNow;
			var reportInterval = TimeSpan.FromSeconds(2);
			var errorcount = 0;
			ShowProgress("Exporting Documents");
			
			while (true)
			{
				bool hasDocs = false;

                try {
                    var maxRecords = options.Limit - totalCount;
                    if (maxRecords > 0)
			        {
			            using (var documents = await GetDocuments(lastEtag, maxRecords))
			            {
			                var watch = Stopwatch.StartNew();

			                while (await documents.MoveNextAsync())
			                {
			                    hasDocs = true;
			                    var document = documents.Current;
			                    lastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag"));

			                    if (!options.MatchFilters(document))
			                        continue;

			                    if (options.ShouldExcludeExpired && options.ExcludeExpired(document))
			                        continue;
			                    document.WriteTo(jsonWriter);
			                    totalCount++;

			                    if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
			                    {
			                        ShowProgress("Exported {0} documents", totalCount);
			                        lastReport = SystemTime.UtcNow;
			                    }

			                    if (watch.ElapsedMilliseconds > 100)
			                        errorcount++;
			                    watch.Start();
			                }
			            }
			        
			            if (hasDocs)
			                continue;

			            // The server can filter all the results. In this case, we need to try to go over with the next batch.
			            // Note that if the ETag' server restarts number is not the same, this won't guard against an infinite loop.
                        // (This code provides support for legacy RavenDB version: 1.0)
			            var databaseStatistics = await GetStats();
			            var lastEtagComparable = new ComparableByteArray(lastEtag);
			            if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
			            {
                            lastEtag = EtagUtil.Increment(lastEtag, maxRecords);
			                ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);

			                continue;
			            }
                    }
			    }
                catch (Exception e)
                {
                    ShowProgress("Got Exception during smuggler export. Exception: {0}. ", e.Message);
                    ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
                    throw new SmugglerExportException(e.Message, e)
                    {
                        LastEtag = lastEtag,
                    };
                }

			    ShowProgress("Done with reading documents, total: {0}, lastEtag: {1}", totalCount, lastEtag);
				return lastEtag;
			}
		}

		private static async Task<Etag> ExportDocuments(DocumentStore exportStore, DocumentStore importStore, SmugglerOptions options, ServerSupportedFeatures exportStoreSupportedFeatures, int exportBatchSize, int importBatchSize)
		{
			var now = SystemTime.UtcNow;

			string lastEtag = options.StartDocsEtag;
			var totalCount = 0;
			var lastReport = SystemTime.UtcNow;
			var reportInterval = TimeSpan.FromSeconds(2);
			ShowProgress("Exporting Documents");

			var bulkInsertOperation = importStore.BulkInsert(null, new BulkInsertOptions
			                                                       {
																	   BatchSize = importBatchSize,
				                                                       OverwriteExisting = true,
			                                                       });
			bulkInsertOperation.Report += text => ShowProgress(text);

			try
			{
				while (true)
				{
					if (exportStoreSupportedFeatures.IsDocsStreamingSupported)
					{
						ShowProgress("Streaming documents from " + lastEtag);
						using (var documentsEnumerator = await exportStore.AsyncDatabaseCommands.StreamDocsAsync(lastEtag))
						{
							while (await documentsEnumerator.MoveNextAsync())
							{
								var document = documentsEnumerator.Current;

								if (!options.MatchFilters(document))
									continue;
								if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
									continue;

								var metadata = document.Value<RavenJObject>("@metadata");
								var id = metadata.Value<string>("@id");
								var etag = Etag.Parse(metadata.Value<string>("@etag"));
								document.Remove("@metadata");
								bulkInsertOperation.Store(document, metadata, id);
								totalCount++;

								if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
								{
									ShowProgress("Exported {0} documents", totalCount);
									lastReport = SystemTime.UtcNow;
								}

								lastEtag = etag;
							}
						}
					}
					else
					{
						int retries = RetriesCount;
						var originalRequestTimeout = exportStore.JsonRequestFactory.RequestTimeout;
						var timeout = options.Timeout.Seconds;
						if (timeout < 30)
							timeout = 30;
						try
						{
							var operationMetadata = new OperationMetadata(exportStore.Url, exportStore.Credentials, exportStore.ApiKey);

							while (true)
							{
								try
								{
									ShowProgress("Get documents from " + lastEtag);
									var documents = await ((AsyncServerClient)exportStore.AsyncDatabaseCommands).GetDocumentsInternalAsync(null, lastEtag, exportBatchSize, operationMetadata);
									foreach (RavenJObject document in documents)
									{
										var metadata = document.Value<RavenJObject>("@metadata");
										var id = metadata.Value<string>("@id");
										var etag = Etag.Parse(metadata.Value<string>("@etag"));
										document.Remove("@metadata");
										metadata.Remove("@id");
										metadata.Remove("@etag");

										if (!options.MatchFilters(document))
											continue;
										if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
											continue;

										bulkInsertOperation.Store(document, metadata, id);
										totalCount++;

										if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
										{
											ShowProgress("Exported {0} documents", totalCount);
											lastReport = SystemTime.UtcNow;
										}
										lastEtag = etag;
									}
									break;
								}
								catch (Exception e)
								{
									if (retries-- == 0)
										throw;
									exportStore.JsonRequestFactory.RequestTimeout = TimeSpan.FromSeconds(timeout *= 2);
//.........这里部分代码省略.........

		private async Task<int> ImportDocuments(JsonTextReader jsonReader, SmugglerOptions options)
		{
		    var now = SystemTime.UtcNow;
			var count = 0;

			if (jsonReader.Read() == false)
				return count;
			if (jsonReader.TokenType != JsonToken.PropertyName)
				throw new InvalidDataException("PropertyName was expected");
			if (Equals("Docs", jsonReader.Value) == false)
				throw new InvalidDataException("Docs property was expected");
			if (jsonReader.Read() == false)
				return count;
			if (jsonReader.TokenType != JsonToken.StartArray)
				throw new InvalidDataException("StartArray was expected");

			while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
			{
				var document = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
			    var size = GetRoughSize(document);
				if (size > 1024 * 1024)
				{
					Console.WriteLine("Large document warning: {0:#,#.##;;0} kb - {1}",
									  (double)size / 1024,
									  document["@metadata"].Value<string>("@id"));
				}
				if ((options.OperateOnTypes & ItemType.Documents) != ItemType.Documents)
					continue;
				if (options.MatchFilters(document) == false)
					continue;

                if (options.ShouldExcludeExpired && options.ExcludeExpired(document, now))
                    continue;

				if (!string.IsNullOrEmpty(options.TransformScript))
					document = await TransformDocument(document, options.TransformScript);

				if (document == null)
					continue;

				await PutDocument(document);

				count++;

				if (count % options.BatchSize == 0)
				{
					ShowProgress("Read {0} documents", count);
				}
			}

			await PutDocument(null); // force flush

			return count;
		}

		private async Task<Etag> ExportDocuments(SmugglerOptions options, JsonTextWriter jsonWriter, Etag lastEtag)
		{
			var totalCount = 0;
			var lastReport = SystemTime.UtcNow;
			var reportInterval = TimeSpan.FromSeconds(2);
			var errorcount = 0;
			ShowProgress("Exporting Documents");

			while (true)
			{
				using (var documents = await GetDocuments(lastEtag))
				{
					var watch = Stopwatch.StartNew();					

					while (await documents.MoveNextAsync())
					{
					
						var document = documents.Current;

						if (!options.MatchFilters(document))
							continue;

						if (options.ShouldExcludeExpired && options.ExcludeExpired(document))
							continue;
						document.WriteTo(jsonWriter);
						totalCount++;
						
						if (totalCount%1000 == 0 || SystemTime.UtcNow - lastReport > reportInterval)
						{
							ShowProgress("Exported {0} documents", totalCount);
							lastReport = SystemTime.UtcNow;
						}

						lastEtag = Etag.Parse(document.Value<RavenJObject>("@metadata").Value<string>("@etag"));
						if (watch.ElapsedMilliseconds > 100)
							errorcount++;
						watch.Start();
					}
				}

				var databaseStatistics = await GetStats();
				var lastEtagComparable = new ComparableByteArray(lastEtag);
				if (lastEtagComparable.CompareTo(databaseStatistics.LastDocEtag) < 0)
				{
					lastEtag = EtagUtil.Increment(lastEtag, SmugglerOptions.BatchSize);
					ShowProgress("Got no results but didn't get to the last doc etag, trying from: {0}", lastEtag);

					continue;
				}

				ShowProgress("Done with reading documents, total: {0}", totalCount);
				return lastEtag;
			}
		}