C# Chunks.TFChunkDb类代码示例

        public void try_read_does_not_cache_anything_and_returns_record_once_it_is_written_later()
        {
            var writerchk = new InMemoryCheckpoint(0);
            var db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                       new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                       10000,
                                                       0,
                                                       writerchk,
                                                       new InMemoryCheckpoint()));
            db.OpenVerifyAndClean();

            var writer = new TFChunkWriter(db);
            writer.Open();

            var reader = new TFChunkSequentialReader(db, writerchk, 0);

            LogRecord record;
            Assert.IsFalse(reader.TryReadNext(out record));

            var rec = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "ES", -1, "ET", new byte[] { 7 }, null);
            long tmp;
            Assert.IsTrue(writer.Write(rec, out tmp));
            writer.Flush();
            writer.Close();

            Assert.IsTrue(reader.TryReadNext(out record));
            Assert.AreEqual(rec, record);

            reader.Close();
            db.Close();
        }

        private void ReOpenDb()
        {
            Db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                   new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                   10000,
                                                   0,
                                                   WriterCheckpoint,
                                                   ChaserCheckpoint,
                                                   new InMemoryCheckpoint(-1),
                                                   new InMemoryCheckpoint(-1)));

            Db.Open();

            var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
            var lowHasher = new XXHashUnsafe();
            var highHasher = new Murmur3AUnsafe();
            TableIndex = new TableIndex(Path.Combine(PathName, "index"), lowHasher, highHasher,
                                        () => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
                                        () => new TFReaderLease(readers),
                                        PTableVersions.Index64Bit,
                                        MaxEntriesInMemTable);
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      0,
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: MetastreamMaxCount,
                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);
            ReadIndex.Init(ChaserCheckpoint.Read());
        }

        private void ReOpenDb()
        {
            Db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                   new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                   10000,
                                                   0,
                                                   WriterCheckpoint,
                                                   ChaserCheckpoint,
                                                   new InMemoryCheckpoint(-1),
                                                   new InMemoryCheckpoint(-1)));

            Db.Open();

            var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
            TableIndex = new TableIndex(Path.Combine(PathName, "index"),
                                        () => new HashListMemTable(MaxEntriesInMemTable * 2),
                                        () => new TFReaderLease(readers),
                                        MaxEntriesInMemTable);
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      new ByLengthHasher(),
                                      0,
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: MetastreamMaxCount);
            ReadIndex.Init(ChaserCheckpoint.Read());
        }

 public void SetUp()
 {
     _writerCheckpoint = new InMemoryCheckpoint();
     _db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                             new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                             1024,
                                             0,
                                             _writerCheckpoint,
                                             new InMemoryCheckpoint(),
                                             new InMemoryCheckpoint(-1),
                                             new InMemoryCheckpoint(-1)));
     _db.Open();
     _writer = new TFChunkWriter(_db);
     _writer.Open();
     _record = new PrepareLogRecord(logPosition: 0,
                                    eventId: _eventId,
                                    correlationId: _correlationId,
                                    transactionPosition: 0xDEAD,
                                    transactionOffset: 0xBEEF,
                                    eventStreamId: "WorldEnding",
                                    expectedVersion: 1234,
                                    timeStamp: new DateTime(2012, 12, 21),
                                    flags: PrepareFlags.SingleWrite,
                                    eventType: "type",
                                    data: new byte[] { 1, 2, 3, 4, 5 },
                                    metadata: new byte[] {7, 17});
     long newPos;
     _writer.Write(_record, out newPos);
     _writer.Flush();
 }

        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            _db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                    new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                    4096,
                                                    0,
                                                    new InMemoryCheckpoint(),
                                                    new InMemoryCheckpoint(),
                                                    new InMemoryCheckpoint(-1),
                                                    new InMemoryCheckpoint(-1)));
            _db.Open();
            
            var chunk = _db.Manager.GetChunk(0);

            _records = new LogRecord[RecordsCount];
            _results = new RecordWriteResult[RecordsCount];

            for (int i = 0; i < _records.Length; ++i)
            {
                _records[i] = LogRecord.SingleWrite(i == 0 ? 0 : _results[i - 1].NewPosition,
                                                    Guid.NewGuid(), Guid.NewGuid(), "es1", ExpectedVersion.Any, "et1",
                                                    new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
                _results[i] = chunk.TryAppend(_records[i]);
            }

            chunk.Flush();
            _db.Config.WriterCheckpoint.Write(_results[RecordsCount - 1].NewPosition);
            _db.Config.WriterCheckpoint.Flush();
        }

 public Projections(
     TFChunkDb db, QueuedHandler mainQueue, InMemoryBus mainBus, TimerService timerService,
     HttpService httpService, int projectionWorkerThreadCount)
 {
     _projectionWorkerThreadCount = projectionWorkerThreadCount;
     SetupMessaging(db, mainQueue, mainBus, timerService, httpService);
 }

        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            _db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                    new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                    16 * 1024,
                                                    0,
                                                    new InMemoryCheckpoint(),
                                                    new ICheckpoint[0]));
            _db.OpenVerifyAndClean();

            var chunk = _db.Manager.GetChunk(0);

            _rec1 = LogRecord.SingleWrite(0, Guid.NewGuid(), Guid.NewGuid(), "es1", ExpectedVersion.Any, "et1",
                                          new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
            _res1 = chunk.TryAppend(_rec1);

            _rec2 = LogRecord.SingleWrite(_res1.NewPosition,
                                          Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
                                          new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
            _res2 = chunk.TryAppend(_rec2);

            _rec3 = LogRecord.SingleWrite(_res2.NewPosition,
                                          Guid.NewGuid(), Guid.NewGuid(), "es-to-scavenge", ExpectedVersion.Any, "et1",
                                          new byte[] { 0, 1, 2 }, new byte[] { 5, 7 });
            _res3 = chunk.TryAppend(_rec3);

            chunk.Complete();

            var scavenger = new TFChunkScavenger(_db, new FakeReadIndex(x => x == "es-to-scavenge"));
            scavenger.Scavenge(alwaysKeepScavenged: true);

            _scavengedChunk = _db.Manager.GetChunk(0);
        }

        private void ReOpenDb()
        {
            Db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                   new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                   10000,
                                                   0,
                                                   WriterCheckpoint,
                                                   ChaserCheckpoint,
                                                   new InMemoryCheckpoint(-1),
                                                   new InMemoryCheckpoint(-1)));

            Db.Open();

            TableIndex = new TableIndex(Path.Combine(PathName, "index"),
                                        () => new HashListMemTable(MaxEntriesInMemTable * 2),
                                        MaxEntriesInMemTable);
            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      2,
                                      5,
                                      () => new TFChunkReader(Db, Db.Config.WriterCheckpoint),
                                      TableIndex,
                                      new ByLengthHasher(),
                                      new NoLRUCache<string, StreamCacheInfo>(),
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: MetastreamMaxCount);
            ReadIndex.Init(WriterCheckpoint.Read(), ChaserCheckpoint.Read());
        }

 public TFChunkScavenger(TFChunkDb db, IReadIndex readIndex)
 {
     _db = db;
     _readIndex = readIndex;
     Ensure.NotNull(db, "db");
     Ensure.NotNull(readIndex, "readIndex");
 }

        public MiniNode(string pathname)
        {
            var ip = GetLocalIp();

            int extTcpPort = GetAvailablePort(ip);
            int extHttpPort = GetAvailablePort(ip);

            _dbPath = Path.Combine(pathname, string.Format("mini-node-db-{0}-{1}", extTcpPort, extHttpPort));
            Directory.CreateDirectory(_dbPath);
            _tfChunkDb = new TFChunkDb(CreateOneTimeDbConfig(1024*1024, _dbPath, 1));

            TcpEndPoint = new IPEndPoint(ip, extTcpPort);
            HttpEndPoint = new IPEndPoint(ip, extHttpPort);

            var singleVNodeSettings = new SingleVNodeSettings(TcpEndPoint,
                                                              HttpEndPoint,
                                                              new[] {HttpEndPoint.ToHttpUrl()},
                                                              1,
                                                              1,
                                                              1,
                                                              TimeSpan.FromHours(1),
                                                              StatsStorage.None);

            _node = new SingleVNode(_tfChunkDb, singleVNodeSettings, dbVerifyHashes: true, memTableEntryCount: 1000);
        }

        public void try_read_returns_false_when_writer_checksum_is_equal_to_reader_checksum()
        {
            var writerchk = new InMemoryCheckpoint();
            var chaserchk = new InMemoryCheckpoint(Checkpoint.Chaser, 0);
            var db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                       new PrefixFileNamingStrategy(PathName, "prefix.tf"),
                                                       10000,
                                                       0,
                                                       writerchk,
                                                       chaserchk,
                                                       new[] {writerchk, chaserchk}));
            db.OpenVerifyAndClean();
            writerchk.Write(12);
            writerchk.Flush();
            chaserchk.Write(12);
            chaserchk.Flush();

            var chaser = new TFChunkChaser(db, writerchk, chaserchk);
            chaser.Open();

            LogRecord record;
            Assert.IsFalse(chaser.TryReadNext(out record));
            Assert.AreEqual(12, chaserchk.Read());

            chaser.Close();
            db.Dispose();
        }

        public MiniClusterNode(
            string pathname, int debugIndex, IPEndPoint internalTcp, IPEndPoint internalTcpSec, IPEndPoint internalHttp,
            IPEndPoint externalTcp, IPEndPoint externalTcpSec, IPEndPoint externalHttp, IPEndPoint[] gossipSeeds,
            ISubsystem[] subsystems = null, int? chunkSize = null, int? cachedChunkSize = null,
            bool enableTrustedAuth = false, bool skipInitializeStandardUsersCheck = true, int memTableSize = 1000,
            bool inMemDb = true, bool disableFlushToDisk = false)
        {
            RunningTime.Start();
            RunCount += 1;

            _dbPath = Path.Combine(
                pathname,
                string.Format(
                    "mini-cluster-node-db-{0}-{1}-{2}", externalTcp.Port, externalTcpSec.Port, externalHttp.Port));

            Directory.CreateDirectory(_dbPath);
            FileStreamExtensions.ConfigureFlush(disableFlushToDisk);
            Db =
                new TFChunkDb(
                    CreateDbConfig(chunkSize ?? ChunkSize, _dbPath, cachedChunkSize ?? CachedChunkSize, inMemDb));

            InternalTcpEndPoint = internalTcp;
            InternalTcpSecEndPoint = internalTcpSec;
            InternalHttpEndPoint = internalHttp;

            ExternalTcpEndPoint = externalTcp;
            ExternalTcpSecEndPoint = externalTcpSec;
            ExternalHttpEndPoint = externalHttp;

            var singleVNodeSettings = new ClusterVNodeSettings(
                Guid.NewGuid(), debugIndex, InternalTcpEndPoint, InternalTcpSecEndPoint, ExternalTcpEndPoint,
                ExternalTcpSecEndPoint, InternalHttpEndPoint, ExternalHttpEndPoint,
                new Data.GossipAdvertiseInfo(InternalTcpEndPoint, InternalTcpSecEndPoint,
                                             ExternalTcpEndPoint, ExternalTcpSecEndPoint,
                                             InternalHttpEndPoint, ExternalHttpEndPoint),
                new[] {InternalHttpEndPoint.ToHttpUrl()}, new[]{ExternalHttpEndPoint.ToHttpUrl()}, enableTrustedAuth, ssl_connections.GetCertificate(), 1, false,
                "", gossipSeeds, TFConsts.MinFlushDelayMs, 3, 2, 2, TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(2),
                false, "", false, TimeSpan.FromHours(1), StatsStorage.None, 0,
                new InternalAuthenticationProviderFactory(), disableScavengeMerging: true, adminOnPublic: true,
                statsOnPublic: true, gossipOnPublic: true, gossipInterval: TimeSpan.FromSeconds(1),
                gossipAllowedTimeDifference: TimeSpan.FromSeconds(1), gossipTimeout: TimeSpan.FromSeconds(1),
                extTcpHeartbeatTimeout: TimeSpan.FromSeconds(10), extTcpHeartbeatInterval: TimeSpan.FromSeconds(10),
                intTcpHeartbeatTimeout: TimeSpan.FromSeconds(10), intTcpHeartbeatInterval: TimeSpan.FromSeconds(10),
                verifyDbHash: false, maxMemtableEntryCount: memTableSize, developmentMode: false);

            Log.Info(
                "\n{0,-25} {1} ({2}/{3}, {4})\n" + "{5,-25} {6} ({7})\n" + "{8,-25} {9} ({10}-bit)\n"
                + "{11,-25} {12}\n" + "{13,-25} {14}\n" + "{15,-25} {16}\n" + "{17,-25} {18}\n" + "{19,-25} {20}\n\n",
                "ES VERSION:", VersionInfo.Version, VersionInfo.Branch, VersionInfo.Hashtag, VersionInfo.Timestamp,
                "OS:", OS.OsFlavor, Environment.OSVersion, "RUNTIME:", OS.GetRuntimeVersion(),
                Marshal.SizeOf(typeof (IntPtr))*8, "GC:",
                GC.MaxGeneration == 0
                    ? "NON-GENERATION (PROBABLY BOEHM)"
                    : string.Format("{0} GENERATIONS", GC.MaxGeneration + 1), "DBPATH:", _dbPath, "ExTCP ENDPOINT:",
                ExternalTcpEndPoint, "ExTCP SECURE ENDPOINT:", ExternalTcpSecEndPoint, "ExHTTP ENDPOINT:",
                ExternalHttpEndPoint);

            Node = new ClusterVNode(Db, singleVNodeSettings, infoController: new InfoController(null, ProjectionType.None), subsystems: subsystems, gossipSeedSource: new KnownEndpointGossipSeedSource(gossipSeeds));
            Node.ExternalHttpService.SetupController(new TestController(Node.MainQueue));
        }

        public override void TestFixtureSetUp()
        {
            base.TestFixtureSetUp();

            WriterCheckpoint = new InMemoryCheckpoint(0);
            ChaserCheckpoint = new InMemoryCheckpoint(0);

            Bus = new InMemoryBus("bus");
            IODispatcher = new IODispatcher(Bus, new PublishEnvelope(Bus));

            Db = new TFChunkDb(new TFChunkDbConfig(PathName,
                                                   new VersionedPatternFileNamingStrategy(PathName, "chunk-"),
                                                   10000,
                                                   0,
                                                   WriterCheckpoint,
                                                   ChaserCheckpoint,
                                                   new InMemoryCheckpoint(-1),
                                                   new InMemoryCheckpoint(-1)));

            Db.Open();
            // create db
            Writer = new TFChunkWriter(Db);
            Writer.Open();
            WriteTestScenario();
            Writer.Close();
            Writer = null;

            WriterCheckpoint.Flush();
            ChaserCheckpoint.Write(WriterCheckpoint.Read());
            ChaserCheckpoint.Flush();

            var readers = new ObjectPool<ITransactionFileReader>("Readers", 2, 5, () => new TFChunkReader(Db, Db.Config.WriterCheckpoint));
            var lowHasher = new XXHashUnsafe();
            var highHasher = new Murmur3AUnsafe();
            TableIndex = new TableIndex(GetFilePathFor("index"), lowHasher, highHasher,
                                        () => new HashListMemTable(PTableVersions.Index64Bit, MaxEntriesInMemTable * 2),
                                        () => new TFReaderLease(readers),
                                        PTableVersions.Index64Bit,
                                        MaxEntriesInMemTable);

            ReadIndex = new ReadIndex(new NoopPublisher(),
                                      readers,
                                      TableIndex,
                                      0,
                                      additionalCommitChecks: true,
                                      metastreamMaxCount: MetastreamMaxCount,
                                      hashCollisionReadLimit: Opts.HashCollisionReadLimitDefault);

            ReadIndex.Init(ChaserCheckpoint.Read());

            // scavenge must run after readIndex is built
            if (_scavenge)
            {
                if (_completeLastChunkOnScavenge)
                    Db.Manager.GetChunk(Db.Manager.ChunksCount - 1).Complete();
                _scavenger = new TFChunkScavenger(Db, IODispatcher, TableIndex, ReadIndex, Guid.NewGuid(), "fakeNodeIp");
                _scavenger.Scavenge(alwaysKeepScavenged: true, mergeChunks: _mergeChunks);
            }
        }

 public void Register(
     TFChunkDb db, QueuedHandler mainQueue, ISubscriber mainBus, TimerService timerService,
     ITimeProvider timeProvider, IHttpForwarder httpForwarder, HttpService[] httpServices, IPublisher networkSendService)
 {
     _projections = new EventStore.Projections.Core.Projections(
         db, mainQueue, mainBus, timerService, timeProvider, httpForwarder, httpServices, networkSendService,
         projectionWorkerThreadCount: _projectionWorkerThreadCount, runProjections: _runProjections);
 }

        public TFChunkReader(TFChunkDb db, ICheckpoint checkpoint)
        {
            Ensure.NotNull(db, "dbConfig");
            Ensure.NotNull(checkpoint, "writerCheckpoint");

            _db = db;
            _checkpoint = checkpoint;
        }