C# BytesRef.CopyChars方法代码示例

 public virtual void TestSize()
 {
     BytesRef @ref = new BytesRef();
     int num = AtLeast(2);
     for (int j = 0; j < num; j++)
     {
         int mod = 1 + Random().Next(39);
         for (int i = 0; i < 797; i++)
         {
             string str;
             do
             {
                 str = TestUtil.RandomRealisticUnicodeString(Random(), 1000);
             } while (str.Length == 0);
             @ref.CopyChars(str);
             int count = Hash.Size();
             int key = Hash.Add(@ref);
             if (key < 0)
             {
                 Assert.AreEqual(Hash.Size(), count);
             }
             else
             {
                 Assert.AreEqual(Hash.Size(), count + 1);
             }
             if (i % mod == 0)
             {
                 Hash.Clear();
                 Assert.AreEqual(0, Hash.Size());
                 Hash.Reinit();
             }
         }
     }
 }

        public static void Main(string[] args)
        {
            FileInfo input = new FileInfo("/home/dweiss/tmp/shuffled.dict");

            int buckets = 20;
            int shareMaxTail = 10;

            ExternalRefSorter sorter = new ExternalRefSorter(new OfflineSorter());
            FSTCompletionBuilder builder = new FSTCompletionBuilder(buckets, sorter, shareMaxTail);

            TextReader reader =
                new StreamReader(
                    new FileStream(input.FullName, FileMode.Open), Encoding.UTF8);

            BytesRef scratch = new BytesRef();
            string line;
            int count = 0;
            while ((line = reader.ReadLine()) != null)
            {
                scratch.CopyChars(line);
                builder.Add(scratch, count % buckets);
                if ((count++ % 100000) == 0)
                {
                    Console.WriteLine("Line: " + count);
                }
            }

            Console.WriteLine("Building FSTCompletion.");
            FSTCompletion completion = builder.Build();

            FileInfo fstFile = new FileInfo("completion.fst");
            Console.WriteLine("Done. Writing automaton: " + fstFile.FullName);
            completion.FST.Save(fstFile);
            sorter.Dispose();
        }

        public virtual void TestAppend()
        {
            Random random = Random();
            BytesRefArray list = new BytesRefArray(Util.Counter.NewCounter());
            IList<string> stringList = new List<string>();
            for (int j = 0; j < 2; j++)
            {
                if (j > 0 && random.NextBoolean())
                {
                    list.Clear();
                    stringList.Clear();
                }
                int entries = AtLeast(500);
                BytesRef spare = new BytesRef();
                int initSize = list.Size();
                for (int i = 0; i < entries; i++)
                {
                    string randomRealisticUnicodeString = TestUtil.RandomRealisticUnicodeString(random);
                    spare.CopyChars(randomRealisticUnicodeString);
                    Assert.AreEqual(i + initSize, list.Append(spare));
                    stringList.Add(randomRealisticUnicodeString);
                }
                for (int i = 0; i < entries; i++)
                {
                    Assert.IsNotNull(list.Get(spare, i));
                    Assert.AreEqual(stringList[i], spare.Utf8ToString(), "entry " + i + " doesn't match");
                }

                // check random
                for (int i = 0; i < entries; i++)
                {
                    int e = random.Next(entries);
                    Assert.IsNotNull(list.Get(spare, e));
                    Assert.AreEqual(stringList[e], spare.Utf8ToString(), "entry " + i + " doesn't match");
                }
                for (int i = 0; i < 2; i++)
                {
                    BytesRefIterator iterator = list.Iterator();
                    foreach (string @string in stringList)
                    {
                        Assert.AreEqual(@string, iterator.Next().Utf8ToString());
                    }
                }
            }
        }

        public virtual void TestAdd()
        {
            BytesRef @ref = new BytesRef();
            BytesRef scratch = new BytesRef();
            int num = AtLeast(2);
            for (int j = 0; j < num; j++)
            {
                HashSet<string> strings = new HashSet<string>();
                int uniqueCount = 0;
                for (int i = 0; i < 797; i++)
                {
                    string str;
                    do
                    {
                        str = TestUtil.RandomRealisticUnicodeString(Random(), 1000);
                    } while (str.Length == 0);
                    @ref.CopyChars(str);
                    int count = Hash.Size();
                    int key = Hash.Add(@ref);

                    if (key >= 0)
                    {
                        Assert.IsTrue(strings.Add(str));
                        Assert.AreEqual(uniqueCount, key);
                        Assert.AreEqual(Hash.Size(), count + 1);
                        uniqueCount++;
                    }
                    else
                    {
                        Assert.IsFalse(strings.Add(str));
                        Assert.IsTrue((-key) - 1 < count);
                        Assert.AreEqual(str, Hash.Get((-key) - 1, scratch).Utf8ToString());
                        Assert.AreEqual(count, Hash.Size());
                    }
                }

                AssertAllIn(strings, Hash);
                Hash.Clear();
                Assert.AreEqual(0, Hash.Size());
                Hash.Reinit();
            }
        }

 public virtual void TestReadAndWrite()
 {
     Counter bytesUsed = Util.Counter.NewCounter();
     ByteBlockPool pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(bytesUsed));
     pool.NextBuffer();
     bool reuseFirst = Random().NextBoolean();
     for (int j = 0; j < 2; j++)
     {
         IList<BytesRef> list = new List<BytesRef>();
         int maxLength = AtLeast(500);
         int numValues = AtLeast(100);
         BytesRef @ref = new BytesRef();
         for (int i = 0; i < numValues; i++)
         {
             string value = TestUtil.RandomRealisticUnicodeString(Random(), maxLength);
             list.Add(new BytesRef(value));
             @ref.CopyChars(value);
             pool.Append(@ref);
         }
         // verify
         long position = 0;
         foreach (BytesRef expected in list)
         {
             @ref.Grow(expected.Length);
             @ref.Length = expected.Length;
             pool.ReadBytes(position, @ref.Bytes, @ref.Offset, @ref.Length);
             Assert.AreEqual(expected, @ref);
             position += @ref.Length;
         }
         pool.Reset(Random().NextBoolean(), reuseFirst);
         if (reuseFirst)
         {
             Assert.AreEqual(ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed.Get());
         }
         else
         {
             Assert.AreEqual(0, bytesUsed.Get());
             pool.NextBuffer(); // prepare for next iter
         }
     }
 }

        public virtual void TestSort()
        {
            Random random = Random();
            BytesRefArray list = new BytesRefArray(Util.Counter.NewCounter());
            List<string> stringList = new List<string>();

            for (int j = 0; j < 2; j++)
            {
                if (j > 0 && random.NextBoolean())
                {
                    list.Clear();
                    stringList.Clear();
                }
                int entries = AtLeast(500);
                BytesRef spare = new BytesRef();
                int initSize = list.Size();
                for (int i = 0; i < entries; i++)
                {
                    string randomRealisticUnicodeString = TestUtil.RandomRealisticUnicodeString(random);
                    spare.CopyChars(randomRealisticUnicodeString);
                    Assert.AreEqual(initSize + i, list.Append(spare));
                    stringList.Add(randomRealisticUnicodeString);
                }

                stringList.Sort();
                BytesRefIterator iter = list.Iterator(BytesRef.UTF8SortedAsUTF16Comparer);
                int a = 0;
                while ((spare = iter.Next()) != null)
                {
                    Assert.AreEqual(stringList[a], spare.Utf8ToString(), "entry " + a + " doesn't match");
                    a++;
                }
                Assert.IsNull(iter.Next());
                Assert.AreEqual(a, stringList.Count);
            }
        }

        public virtual void TestBinary()
        {
            Directory dir = NewDirectory();
            Document doc = new Document();
            BytesRef @ref = new BytesRef();
            Field field = new BinaryDocValuesField("bytes", @ref);
            doc.Add(field);

            IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, null);
            iwc.SetMergePolicy(NewLogMergePolicy());
            RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc);

            int numDocs = AtLeast(500);
            for (int i = 0; i < numDocs; i++)
            {
                @ref.CopyChars(TestUtil.RandomUnicodeString(Random()));
                iw.AddDocument(doc);
                if (Random().Next(17) == 0)
                {
                    iw.Commit();
                }
            }
            DirectoryReader ir = iw.Reader;
            iw.ForceMerge(1);
            DirectoryReader ir2 = iw.Reader;
            AtomicReader merged = GetOnlySegmentReader(ir2);
            iw.Dispose();

            BinaryDocValues multi = MultiDocValues.GetBinaryValues(ir, "bytes");
            BinaryDocValues single = merged.GetBinaryDocValues("bytes");
            BytesRef actual = new BytesRef();
            BytesRef expected = new BytesRef();
            for (int i = 0; i < numDocs; i++)
            {
                single.Get(i, expected);
                multi.Get(i, actual);
                Assert.AreEqual(expected, actual);
            }
            ir.Dispose();
            ir2.Dispose();
            dir.Dispose();
        }

 /// <summary>
 /// returns the bytes representation of the str val - TODO: should this return the indexed raw bytes not? </summary>
 public virtual bool BytesVal(int doc, BytesRef target)
 {
     string s = StrVal(doc);
     if (s == null)
     {
         target.Length = 0;
         return false;
     }
     target.CopyChars(s);
     return true;
 }

 private void AssertAllIn(ISet<string> strings, BytesRefHash hash)
 {
     BytesRef @ref = new BytesRef();
     BytesRef scratch = new BytesRef();
     int count = hash.Size();
     foreach (string @string in strings)
     {
         @ref.CopyChars(@string);
         int key = hash.Add(@ref); // add again to check duplicates
         Assert.AreEqual(@string, hash.Get((-key) - 1, scratch).Utf8ToString());
         Assert.AreEqual(count, hash.Size());
         Assert.IsTrue(key < count, "key: " + key + " count: " + count + " string: " + @string);
     }
 }

 public virtual void TestSort()
 {
     BytesRef @ref = new BytesRef();
     int num = AtLeast(2);
     for (int j = 0; j < num; j++)
     {
         SortedSet<string> strings = new SortedSet<string>();
         for (int k = 0; k < 797; k++)
         {
             string str;
             do
             {
                 str = TestUtil.RandomRealisticUnicodeString(Random(), 1000);
             } while (str.Length == 0);
             @ref.CopyChars(str);
             Hash.Add(@ref);
             strings.Add(str);
         }
         // We use the UTF-16 comparator here, because we need to be able to
         // compare to native String.CompareTo() [UTF-16]:
         int[] sort = Hash.Sort(BytesRef.UTF8SortedAsUTF16Comparer);
         Assert.IsTrue(strings.Count < sort.Length);
         int i = 0;
         BytesRef scratch = new BytesRef();
         foreach (string @string in strings)
         {
             @ref.CopyChars(@string);
             Assert.AreEqual(@ref, Hash.Get(sort[i++], scratch));
         }
         Hash.Clear();
         Assert.AreEqual(0, Hash.Size());
         Hash.Reinit();
     }
 }

 public virtual void TestGet()
 {
     BytesRef @ref = new BytesRef();
     BytesRef scratch = new BytesRef();
     int num = AtLeast(2);
     for (int j = 0; j < num; j++)
     {
         IDictionary<string, int?> strings = new Dictionary<string, int?>();
         int uniqueCount = 0;
         for (int i = 0; i < 797; i++)
         {
             string str;
             do
             {
                 str = TestUtil.RandomRealisticUnicodeString(Random(), 1000);
             } while (str.Length == 0);
             @ref.CopyChars(str);
             int count = Hash.Size();
             int key = Hash.Add(@ref);
             if (key >= 0)
             {
                 Assert.IsFalse(strings.ContainsKey(str));
                 strings[str] = Convert.ToInt32(key);
                 Assert.AreEqual(uniqueCount, key);
                 uniqueCount++;
                 Assert.AreEqual(Hash.Size(), count + 1);
             }
             else
             {
                 Assert.IsTrue((-key) - 1 < count);
                 Assert.AreEqual(Hash.Size(), count);
             }
         }
         foreach (KeyValuePair<string, int?> entry in strings)
         {
             @ref.CopyChars(entry.Key);
             Assert.AreEqual(@ref, Hash.Get((int)entry.Value, scratch));
         }
         Hash.Clear();
         Assert.AreEqual(0, Hash.Size());
         Hash.Reinit();
     }
 }

 public virtual void TestCompact()
 {
     BytesRef @ref = new BytesRef();
     int num = AtLeast(2);
     for (int j = 0; j < num; j++)
     {
         int numEntries = 0;
         const int size = 797;
         BitArray bits = new BitArray(size);
         for (int i = 0; i < size; i++)
         {
             string str;
             do
             {
                 str = TestUtil.RandomRealisticUnicodeString(Random(), 1000);
             } while (str.Length == 0);
             @ref.CopyChars(str);
             int key = Hash.Add(@ref);
             if (key < 0)
             {
                 Assert.IsTrue(bits.SafeGet((-key) - 1));
             }
             else
             {
                 Assert.IsFalse(bits.SafeGet(key));
                 bits.SafeSet(key, true);
                 numEntries++;
             }
         }
         Assert.AreEqual(Hash.Size(), bits.Cardinality());
         Assert.AreEqual(numEntries, bits.Cardinality());
         Assert.AreEqual(numEntries, Hash.Size());
         int[] compact = Hash.Compact();
         Assert.IsTrue(numEntries < compact.Length);
         for (int i = 0; i < numEntries; i++)
         {
             bits.SafeSet(compact[i], false);
         }
         Assert.AreEqual(0, bits.Cardinality());
         Hash.Clear();
         Assert.AreEqual(0, Hash.Size());
         Hash.Reinit();
     }
 }

        public virtual void TestAddByPoolOffset()
        {
            BytesRef @ref = new BytesRef();
            BytesRef scratch = new BytesRef();
            BytesRefHash offsetHash = NewHash(Pool);
            int num = AtLeast(2);
            for (int j = 0; j < num; j++)
            {
                HashSet<string> strings = new HashSet<string>();
                int uniqueCount = 0;
                for (int i = 0; i < 797; i++)
                {
                    string str;
                    do
                    {
                        str = TestUtil.RandomRealisticUnicodeString(Random(), 1000);
                    } while (str.Length == 0);
                    @ref.CopyChars(str);
                    int count = Hash.Size();
                    int key = Hash.Add(@ref);

                    if (key >= 0)
                    {
                        Assert.IsTrue(strings.Add(str));
                        Assert.AreEqual(uniqueCount, key);
                        Assert.AreEqual(Hash.Size(), count + 1);
                        int offsetKey = offsetHash.AddByPoolOffset(Hash.ByteStart(key));
                        Assert.AreEqual(uniqueCount, offsetKey);
                        Assert.AreEqual(offsetHash.Size(), count + 1);
                        uniqueCount++;
                    }
                    else
                    {
                        Assert.IsFalse(strings.Add(str));
                        Assert.IsTrue((-key) - 1 < count);
                        Assert.AreEqual(str, Hash.Get((-key) - 1, scratch).Utf8ToString());
                        Assert.AreEqual(count, Hash.Size());
                        int offsetKey = offsetHash.AddByPoolOffset(Hash.ByteStart((-key) - 1));
                        Assert.IsTrue((-offsetKey) - 1 < count);
                        Assert.AreEqual(str, Hash.Get((-offsetKey) - 1, scratch).Utf8ToString());
                        Assert.AreEqual(count, Hash.Size());
                    }
                }

                AssertAllIn(strings, Hash);
                foreach (string @string in strings)
                {
                    @ref.CopyChars(@string);
                    int key = Hash.Add(@ref);
                    BytesRef bytesRef = offsetHash.Get((-key) - 1, scratch);
                    Assert.AreEqual(@ref, bytesRef);
                }

                Hash.Clear();
                Assert.AreEqual(0, Hash.Size());
                offsetHash.Clear();
                Assert.AreEqual(0, offsetHash.Size());
                Hash.Reinit(); // init for the next round
                offsetHash.Reinit();
            }
        }