C# StorageEnvironment.FlushLogToDataFile方法代码示例

		public static void Execute(StorageEnvironmentOptions srcOptions, StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions compactOptions, Action<CompactionProgress> progressReport = null)
		{
			if (srcOptions.IncrementalBackupEnabled)
				throw new InvalidOperationException(CannotCompactBecauseOfIncrementalBackup);

			long minimalCompactedDataFileSize;

			srcOptions.ManualFlushing = true; // prevent from flushing during compaction - we shouldn't touch any source files
			compactOptions.ManualFlushing = true; // let us flush manually during data copy

			using(var existingEnv = new StorageEnvironment(srcOptions))
			using (var compactedEnv = new StorageEnvironment(compactOptions))
			{
				CopyTrees(existingEnv, compactedEnv, progressReport);

				compactedEnv.FlushLogToDataFile(allowToFlushOverwrittenPages: true);

				compactedEnv.Journal.Applicator.SyncDataFile(compactedEnv.OldestTransaction);
				compactedEnv.Journal.Applicator.DeleteCurrentAlreadyFlushedJournal();

				minimalCompactedDataFileSize = compactedEnv.NextPageNumber*AbstractPager.PageSize;
			}

			using (var compactedDataFile = new FileStream(Path.Combine(compactOptions.BasePath, Constants.DatabaseFilename), FileMode.Open, FileAccess.ReadWrite))
			{
				compactedDataFile.SetLength(minimalCompactedDataFileSize);
			}
		}

        public void AllScratchPagesShouldBeReleased()
        {
            var options = StorageEnvironmentOptions.CreateMemoryOnly();
            options.ManualFlushing = true;
            using (var env = new StorageEnvironment(options))
            {
                using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    env.CreateTree(txw, "test");

                    txw.Commit();
                }

                using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    var tree = txw.Environment.State.GetTree(txw, "test");

                    tree.Add("key/1", new MemoryStream(new byte[100]));
                    tree.Add("key/1", new MemoryStream(new byte[200]));
                    txw.Commit();
                }

                env.FlushLogToDataFile(); // non read nor write transactions, so it should flush and release everything from scratch

                Assert.Equal(0, env.ScratchBufferPool.GetNumberOfAllocations(0));
            }
        }

        public void AllScratchPagesShouldBeReleased()
        {
            var options = StorageEnvironmentOptions.CreateMemoryOnly();
            options.ManualFlushing = true;
            using (var env = new StorageEnvironment(options))
            {
                using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    env.CreateTree(txw, "test");

                    txw.Commit();
                }

                using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    var tree = txw.Environment.CreateTree(txw, "test");

                    tree.Add("key/1", new MemoryStream(new byte[100]));
                    tree.Add("key/1", new MemoryStream(new byte[200]));
                    txw.Commit();
                }

                env.FlushLogToDataFile(); // non read nor write transactions, so it should flush and release everything from scratch

                // we keep track of the pages in scratch for one additional transaction, to avoid race
                // condition with FlushLogToDataFile concurrently with new read transactions
                Assert.Equal(2, env.ScratchBufferPool.GetNumberOfAllocations(0));
            }
        }

        public void ShouldProperlyRecover()
        {
            var sequentialLargeIds = ReadData("non-leaf-page-seq-id-large-values-2.txt");

            var enumerator = sequentialLargeIds.GetEnumerator();

            if (Directory.Exists("tests"))
                Directory.Delete("tests", true);

            var options = StorageEnvironmentOptions.ForPath("tests");
            options.ManualFlushing = true;

            using (var env = new StorageEnvironment(options))
            {
                for (var transactions = 0; transactions < 100; transactions++)
                {
                    using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
                    {
                        for (var i = 0; i < 100; i++)
                        {
                            enumerator.MoveNext();

                            tx.Root.Add			(enumerator.Current.Key.ToString("0000000000000000"), new MemoryStream(enumerator.Current.Value));
                        }

                        tx.Commit();
                    }

                    if (transactions == 50)
                        env.FlushLogToDataFile();
                }

                ValidateRecords(env, new List<string> { "Root" }, sequentialLargeIds.Select(x => x.Key.ToString("0000000000000000")).ToList());
            }

            options = StorageEnvironmentOptions.ForPath("tests");
            options.ManualFlushing = true;

            using (var env = new StorageEnvironment(options))
            {
                ValidateRecords(env, new List<string> { "Root" }, sequentialLargeIds.Select(x => x.Key.ToString("0000000000000000")).ToList());
            }
        }

        public void MultipleTxPagesCanPointToOnePageNumberWhichShouldNotBeCausingIssuesDuringFlushing()
        {
            var options = StorageEnvironmentOptions.CreateMemoryOnly();
            options.ManualFlushing = true;
            using (var env = new StorageEnvironment(options))
            {
                var trees = CreateTrees(env, 2, "tree");
                var tree1 = trees[0];
                var tree2 = trees[1];

                using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    var t1 = tx.State.GetTree(tx, tree1);

                    t1.MultiAdd("key", "value/1");
                    t1.MultiAdd("key", "value/2");

                    tx.Commit();
                }

                using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    var t1 = tx.State.GetTree(tx, tree1);
                    var t2 = tx.State.GetTree(tx, tree2);

                    var buffer = new byte[1000];

                    t1.MultiDelete("key", "value/1");
                    t1.MultiDelete("key", "value/2");

                    t2.Add("key/1", new MemoryStream(buffer));
                    t2.Add("key/2", new MemoryStream(buffer));
                    t2.Add("key/3", new MemoryStream(buffer));
                    t2.Add("key/4", new MemoryStream(buffer));
                    t2.Add("key/5", new MemoryStream(buffer));

                    tx.Commit();
                }

                env.FlushLogToDataFile();
            }
        }

        public void OldestActiveTransactionShouldBeCalculatedProperly()
        {
            var directory = "Test";

            if (Directory.Exists(directory))
                Directory.Delete(directory, true);

            var options = StorageEnvironmentOptions.ForPath(directory);

            options.ManualFlushing = true;
            using (var env = new StorageEnvironment(options))
            {
                var trees = CreateTrees(env, 1, "tree");
                var transactions = new List<Transaction>();

                for (int a = 0; a < 100; a++)
                {
                    var random = new Random(1337);
                    var buffer = new byte[random.Next(100, 1000)];
                    random.NextBytes(buffer);

                    using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
                    {
                        for (int i = 0; i < 100; i++)
                        {
                            foreach (var tree in trees)
                            {
                                tx.Environment.State.GetTree(tx,tree).Add(tx, string.Format("key/{0}/{1}", a, i), new MemoryStream(buffer));
                            }

                        }

                        tx.Commit();
                        env.FlushLogToDataFile(tx);
                        var txr = env.NewTransaction(TransactionFlags.Read);

                        transactions.Add(txr);
                    }
                }

                Assert.Equal(transactions.OrderBy(x => x.Id).First().Id, env.OldestTransaction);

                foreach (var tx in transactions)
                {
                    foreach (var tree in trees)
                    {
                        using (var iterator = tx.Environment.State.GetTree(tx,tree).Iterate(tx))
                        {
                            if (!iterator.Seek(Slice.BeforeAllKeys))
                                continue;

                            do
                            {
                                Assert.Contains("key/", iterator.CurrentKey.ToString());
                            }
                            while (iterator.MoveNext());
                        }
                    }
                }
            }
        }

        private void Restore(StorageEnvironment env, string singleBackupFile)
        {
            using (env.Journal.Applicator.TakeFlushingLock())
            {
                using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    using (env.Options.AllowManualFlushing())
                    {
                        env.FlushLogToDataFile(txw);
                    }

                    using (var package = ZipFile.Open(singleBackupFile, ZipArchiveMode.Read))
                    {
                        if (package.Entries.Count == 0)
                            return;

                        var toDispose = new List<IDisposable>();

						var tempDir = Directory.CreateDirectory(Path.GetTempPath() + Guid.NewGuid()).FullName;

                        try
                        {
                            TransactionHeader* lastTxHeader = null;
                            var pagesToWrite = new Dictionary<long, Func<Page>>();

                            long journalNumber = -1;
                            foreach (var entry in package.Entries)
                            {
                                switch (Path.GetExtension(entry.Name))
                                {
                                    case ".journal":

										var jounalFileName = Path.Combine(tempDir, entry.Name);
                                        using (var output = new FileStream(jounalFileName, FileMode.Create))
                                        using (var input = entry.Open())
                                        {
                                            output.Position = output.Length;
                                            input.CopyTo(output);
                                        }

                                        var pager = new Win32MemoryMapPager(jounalFileName);
                                        toDispose.Add(pager);

                                        if (long.TryParse(Path.GetFileNameWithoutExtension(entry.Name), out journalNumber) == false)
                                        {
                                            throw new InvalidOperationException("Cannot parse journal file number");
                                        }

										var recoveryPager = new Win32MemoryMapPager(Path.Combine(tempDir, StorageEnvironmentOptions.JournalRecoveryName(journalNumber)));
                                        toDispose.Add(recoveryPager);

                                        var reader = new JournalReader(pager, recoveryPager, 0, lastTxHeader);

                                        while (reader.ReadOneTransaction(env.Options))
                                        {
                                            lastTxHeader = reader.LastTransactionHeader;
                                        }

                                        foreach (var translation in reader.TransactionPageTranslation)
                                        {
                                            var pageInJournal = translation.Value.JournalPos;
                                            pagesToWrite[translation.Key] = () => recoveryPager.Read(pageInJournal);
                                        }

                                        break;
                                    default:
                                        throw new InvalidOperationException("Unknown file, cannot restore: " + entry);
                                }
                            }

                            var sortedPages = pagesToWrite.OrderBy(x => x.Key)
                                .Select(x => x.Value())
                                .ToList();

                            var last = sortedPages.Last();

                            env.Options.DataPager.EnsureContinuous(txw, last.PageNumber,
                                last.IsOverflow
                                    ? env.Options.DataPager.GetNumberOfOverflowPages(
                                        last.OverflowSize)
                                    : 1);

                            foreach (var page in sortedPages)
                            {
                                env.Options.DataPager.Write(page);
                            }

                            env.Options.DataPager.Sync();

                            txw.State.Root = Tree.Open(txw, env._sliceComparer, &lastTxHeader->Root);
                            txw.State.FreeSpaceRoot = Tree.Open(txw, env._sliceComparer, &lastTxHeader->FreeSpace);

                            txw.State.FreeSpaceRoot.Name = Constants.FreeSpaceTreeName;
                            txw.State.Root.Name = Constants.RootTreeName;

                            txw.State.NextPageNumber = lastTxHeader->LastPageNumber + 1;

                            env.Journal.Clear(txw);

                            txw.Commit();
//.........这里部分代码省略.........

        public void ScratchPagesShouldNotBeReleasedUntilNotUsed()
        {
            var directory = "Test2";

            if (Directory.Exists(directory))
                Directory.Delete(directory, true);

            var options = StorageEnvironmentOptions.ForPath(directory);

            options.ManualFlushing = true;
            using (var env = new StorageEnvironment(options))
            {
                CreateTrees(env, 2, "tree");
                for (int a = 0; a < 3; a++)
                {
                    using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
                    {
                        tx.Environment.State.GetTree(tx, "tree0").Add(string.Format("key/{0}/{1}/1", new string('0', 1000), a), new MemoryStream());
                        tx.Environment.State.GetTree(tx, "tree0").Add(string.Format("key/{0}/{1}/2", new string('0', 1000), a), new MemoryStream());

                        tx.Commit();
                    }
                }

                using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    tx.Environment.State.GetTree(tx, "tree1").Add("yek/1", new MemoryStream());

                    tx.Commit();
                }

                using (var txr = env.NewTransaction(TransactionFlags.Read))
                {
                    using (var iterator = txr.Environment.State.GetTree(txr, "tree0").Iterate())
                    {
                        Assert.True(iterator.Seek(Slice.BeforeAllKeys)); // all pages are from scratch (one from position 11)

                        var currentKey = iterator.CurrentKey.ToString();

                        env.FlushLogToDataFile(); // frees pages from scratch (including the one at position 11)

                        using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
                        {
                            var tree = txw.Environment.State.GetTree(txw, "tree1");
                            tree.Add(string.Format("yek/{0}/0/0", new string('0', 1000)), new MemoryStream()); // allocates new page from scratch (position 11)

                            txw.Commit();
                        }

                        Assert.Equal(currentKey, iterator.CurrentKey.ToString());

                        using (var txw = env.NewTransaction(TransactionFlags.ReadWrite))
                        {
                            var tree = txw.Environment.State.GetTree(txw, "tree1");
                            tree.Add("fake", new MemoryStream());

                            txw.Commit();
                        }

                        Assert.Equal(currentKey, iterator.CurrentKey.ToString());

                        var count = 0;

                        do
                        {
                            currentKey = iterator.CurrentKey.ToString();
                            count++;

                            Assert.Contains("key/", currentKey);
                        }
                        while (iterator.MoveNext());

                        Assert.Equal(6, count);
                    }
                }
            }
        }

		private static void CopyTrees(StorageEnvironment existingEnv, StorageEnvironment compactedEnv, Action<CompactionProgress> progressReport = null)
		{
			using (var rootIterator = existingEnv.State.Root.Iterate())
			{
				if (rootIterator.Seek(Slice.BeforeAllKeys) == false)
					return;

				var totalTreesCount = existingEnv.State.Root.State.EntriesCount;
				var copiedTrees = 0L;

				do
				{
					var treeName = rootIterator.CurrentKey.ToString();

					using (var txr = existingEnv.NewTransaction(TransactionFlags.Read))
					{
						var existingTree = existingEnv.State.GetTree(txr, treeName);

						Report(treeName, copiedTrees, totalTreesCount, 0, existingTree.State.EntriesCount, progressReport);

						using (var existingTreeIterator = existingTree.Iterate())
						{
							if (existingTreeIterator.Seek(Slice.BeforeAllKeys) == false)
								continue;

							using (var txw = compactedEnv.NewTransaction(TransactionFlags.ReadWrite))
							{
								compactedEnv.CreateTree(txw, treeName);
								txw.Commit();
							}

							var copiedEntries = 0L;

							do
							{
								var transactionSize = 0L;

								using (var txw = compactedEnv.NewTransaction(TransactionFlags.ReadWrite))
								{
									var newTree = txw.ReadTree(treeName);

									do
									{
										var key = existingTreeIterator.CurrentKey;

										if (existingTreeIterator.Current->Flags == NodeFlags.MultiValuePageRef)
										{
											using (var multiTreeIterator = existingTree.MultiRead(key))
											{
												if (multiTreeIterator.Seek(Slice.BeforeAllKeys) == false)
													continue;

												do
												{
													var multiValue = multiTreeIterator.CurrentKey;
													newTree.MultiAdd(key, multiValue);
													transactionSize += multiValue.Size;
												} while (multiTreeIterator.MoveNext());
											}
										}
										else
										{
											using (var value = existingTree.Read(key).Reader.AsStream())
											{
												newTree.Add(key, value);
												transactionSize += value.Length;
											}
										}

										copiedEntries++;

									} while (transactionSize < compactedEnv.Options.MaxLogFileSize/2 && existingTreeIterator.MoveNext());

									txw.Commit();
								}

								if (copiedEntries == existingTree.State.EntriesCount)
									copiedTrees++;

								Report(treeName, copiedTrees, totalTreesCount, copiedEntries, existingTree.State.EntriesCount, progressReport);

								compactedEnv.FlushLogToDataFile();

							} while (existingTreeIterator.MoveNext());
						}
					}
				} while (rootIterator.MoveNext());
			}
		}

        public void Record_debug_journal_and_replay_it_with_manual_flushing()
        {
            using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
            {
                env.DebugJournal = new DebugJournal(debugJouralName, env, true);
                using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
                {
                    env.CreateTree(tx, "test-tree");
                    tx.Commit();
                }

                using (var writeBatch = new WriteBatch())
                {
                    var valueBuffer = new MemoryStream(Encoding.UTF8.GetBytes("{ \"title\": \"foo\",\"name\":\"bar\"}"));
                    writeBatch.Add("foo", valueBuffer, "test-tree");
                    env.Writer.Write(writeBatch);
                }

                using (env.Options.AllowManualFlushing())
                {
                    env.FlushLogToDataFile();
                }

                using (var tx = env.NewTransaction(TransactionFlags.ReadWrite))
                using (env.Options.AllowManualFlushing())
                {
                    env.FlushLogToDataFile(tx);
                    tx.Commit();
                }
            }

            using (var env = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
            {
                env.DebugJournal = DebugJournal.FromFile(debugJouralName, env);
                env.DebugJournal.Replay();

                using (var snapshot = env.CreateSnapshot())
                {
                    Assert.Equal("{ \"title\": \"foo\",\"name\":\"bar\"}", snapshot.Read("test-tree", "foo").Reader.ToStringValue());
                }
            }

        }

		public void StorageEnvironment_should_be_able_to_accept_transactionsToShip_with_new_trees()
		{
			var transactionsToShip = new ConcurrentBag<TransactionToShip>();

			using (var shippingSourceEnv = new StorageEnvironment(StorageEnvironmentOptions.CreateMemoryOnly()))
			{
				shippingSourceEnv.Journal.OnTransactionCommit += tx =>
				{
					tx.CreatePagesSnapshot();
					transactionsToShip.Add(tx);
				};

				using (var tx = shippingSourceEnv.NewTransaction(TransactionFlags.ReadWrite))
				{
					shippingSourceEnv.CreateTree(tx, "TestTree");
					tx.Commit();
				}
			}

			var storageEnvironmentOptions = StorageEnvironmentOptions.CreateMemoryOnly();
			storageEnvironmentOptions.ManualFlushing = true;
			using (var shippingDestinationEnv = new StorageEnvironment(storageEnvironmentOptions))
			{
				foreach (var tx in transactionsToShip)
					shippingDestinationEnv.Journal.Shipper.ApplyShippedLog(tx.PagesSnapshot);

				shippingDestinationEnv.FlushLogToDataFile();

				using (var snapshot = shippingDestinationEnv.CreateSnapshot())
				{
					snapshot.Read("TestTree", "Foo");
				}
			}
		}