C# BytesRef.Utf8ToString方法代码示例

        /// <summary>
        /// Creates this, pulling doc values from the specified
        /// field. 
        /// </summary>
        public DefaultSortedSetDocValuesReaderState(IndexReader reader, string field = FacetsConfig.DEFAULT_INDEX_FIELD_NAME)
        {
            this.field = field;
            this.origReader = reader;

            // We need this to create thread-safe MultiSortedSetDV
            // per collector:
            topReader = SlowCompositeReaderWrapper.Wrap(reader);
            SortedSetDocValues dv = topReader.GetSortedSetDocValues(field);
            if (dv == null)
            {
                throw new System.ArgumentException("field \"" + field + "\" was not indexed with SortedSetDocValues");
            }
            if (dv.ValueCount > int.MaxValue)
            {
                throw new System.ArgumentException("can only handle valueCount < Integer.MAX_VALUE; got " + dv.ValueCount);
            }
            valueCount = (int)dv.ValueCount;

            // TODO: we can make this more efficient if eg we can be
            // "involved" when IOrdinalMap is being created?  Ie see
            // each term/ord it's assigning as it goes...
            string lastDim = null;
            int startOrd = -1;

            // TODO: this approach can work for full hierarchy?;
            // TaxoReader can't do this since ords are not in
            // "sorted order" ... but we should generalize this to
            // support arbitrary hierarchy:
            for (int ord = 0; ord < valueCount; ord++)
            {
                BytesRef term = new BytesRef();
                dv.LookupOrd(ord, term);
                string[] components = FacetsConfig.StringToPath(term.Utf8ToString());
                if (components.Length != 2)
                {
                    throw new System.ArgumentException("this class can only handle 2 level hierarchy (dim/value); got: " + Arrays.ToString(components) + " " + term.Utf8ToString());
                }
                if (!components[0].Equals(lastDim))
                {
                    if (lastDim != null)
                    {
                        prefixToOrdRange[lastDim] = new OrdRange(startOrd, ord - 1);
                    }
                    startOrd = ord;
                    lastDim = components[0];
                }
            }

            if (lastDim != null)
            {
                prefixToOrdRange[lastDim] = new OrdRange(startOrd, valueCount - 1);
            }
        }

 public virtual void TestAppend()
 {
     var bytes = new[] { (byte)'a', (byte)'b', (byte)'c', (byte)'d' };
     BytesRef b = new BytesRef(bytes, 1, 3); // bcd
     b.Append(new BytesRef("e"));
     Assert.AreEqual("bcde", b.Utf8ToString());
 }

 public virtual void TestCopyBytes()
 {
     sbyte[] bytes = new sbyte[] { (sbyte)'a', (sbyte)'b', (sbyte)'c', (sbyte)'d' };
     BytesRef b = new BytesRef(bytes, 1, 3); // bcd
     b.CopyBytes(new BytesRef("bcde"));
     Assert.AreEqual("bcde", b.Utf8ToString());
 }

        public virtual void TestAppend()
        {
            Random random = Random();
            BytesRefArray list = new BytesRefArray(Util.Counter.NewCounter());
            IList<string> stringList = new List<string>();
            for (int j = 0; j < 2; j++)
            {
                if (j > 0 && random.NextBoolean())
                {
                    list.Clear();
                    stringList.Clear();
                }
                int entries = AtLeast(500);
                BytesRef spare = new BytesRef();
                int initSize = list.Size();
                for (int i = 0; i < entries; i++)
                {
                    string randomRealisticUnicodeString = TestUtil.RandomRealisticUnicodeString(random);
                    spare.CopyChars(randomRealisticUnicodeString);
                    Assert.AreEqual(i + initSize, list.Append(spare));
                    stringList.Add(randomRealisticUnicodeString);
                }
                for (int i = 0; i < entries; i++)
                {
                    Assert.IsNotNull(list.Get(spare, i));
                    Assert.AreEqual(stringList[i], spare.Utf8ToString(), "entry " + i + " doesn't match");
                }

                // check random
                for (int i = 0; i < entries; i++)
                {
                    int e = random.Next(entries);
                    Assert.IsNotNull(list.Get(spare, e));
                    Assert.AreEqual(stringList[e], spare.Utf8ToString(), "entry " + i + " doesn't match");
                }
                for (int i = 0; i < 2; i++)
                {
                    BytesRefIterator iterator = list.Iterator();
                    foreach (string @string in stringList)
                    {
                        Assert.AreEqual(@string, iterator.Next().Utf8ToString());
                    }
                }
            }
        }

        public virtual void TestFromBytes()
        {
            sbyte[] bytes = new sbyte[] { (sbyte)'a', (sbyte)'b', (sbyte)'c', (sbyte)'d' };
            BytesRef b = new BytesRef(bytes);
            Assert.AreEqual(bytes, b.Bytes);
            Assert.AreEqual(0, b.Offset);
            Assert.AreEqual(4, b.Length);

            BytesRef b2 = new BytesRef(bytes, 1, 3);
            Assert.AreEqual("bcd", b2.Utf8ToString());

            Assert.IsFalse(b.Equals(b2));
        }

 public virtual void TestIntConversionAndOrdering()
 {
     // generate a series of encoded ints, each numerical one bigger than the one before
     BytesRef last = null, act = new BytesRef(NumericUtils.BUF_SIZE_INT);
     for (int i = -100000; i < 100000; i++)
     {
         NumericUtils.IntToPrefixCodedBytes(i, 0, act);
         if (last != null)
         {
             // test if smaller
             Assert.IsTrue(last.CompareTo(act) < 0, "actual bigger than last (BytesRef)");
             Assert.IsTrue(last.Utf8ToString().CompareToOrdinal(act.Utf8ToString()) < 0, "actual bigger than last (as String)");
         }
         // test is back and forward conversion works
         Assert.AreEqual(i, NumericUtils.PrefixCodedToInt(act), "forward and back conversion should generate same int");
         // next step
         last = act;
         act = new BytesRef(NumericUtils.BUF_SIZE_INT);
     }
 }

        private void CheckTermsOrder(IndexReader r, ISet<string> allTerms, bool isTop)
        {
            TermsEnum terms = MultiFields.GetFields(r).Terms("f").Iterator(null);

            BytesRef last = new BytesRef();

            HashSet<string> seenTerms = new HashSet<string>();

            while (true)
            {
                BytesRef term = terms.Next();
                if (term == null)
                {
                    break;
                }

                Assert.IsTrue(last.CompareTo(term) < 0);
                last.CopyBytes(term);

                string s = term.Utf8ToString();
                Assert.IsTrue(allTerms.Contains(s), "term " + TermDesc(s) + " was not added to index (count=" + allTerms.Count + ")");
                seenTerms.Add(s);
            }

            if (isTop)
            {
                Assert.IsTrue(allTerms.SetEquals(seenTerms));
            }

            // Test seeking:
            IEnumerator<string> it = seenTerms.GetEnumerator();
            while (it.MoveNext())
            {
                BytesRef tr = new BytesRef(it.Current);
                Assert.AreEqual(TermsEnum.SeekStatus.FOUND, terms.SeekCeil(tr), "seek failed for term=" + TermDesc(tr.Utf8ToString()));
            }
        }

            public override SeekStatus SeekCeil(BytesRef term)
            {
                if (DEBUG_SURROGATES)
                {
                    Console.WriteLine("TE.seek target=" + UnicodeUtil.ToHexString(term.Utf8ToString()));
                }
                SkipNext = false;
                TermInfosReader tis = OuterInstance.TermsDict;
                Term t0 = new Term(fieldInfo.Name, term);

                Debug.Assert(TermEnum != null);

                tis.SeekEnum(TermEnum, t0, false);

                Term t = TermEnum.Term();

                if (t != null && t.Field() == InternedFieldName && term.BytesEquals(t.Bytes()))
                {
                    // If we found an exact match, no need to do the
                    // surrogate dance
                    if (DEBUG_SURROGATES)
                    {
                        Console.WriteLine("  seek exact match");
                    }
                    Current = t.Bytes();
                    return SeekStatus.FOUND;
                }
                else if (t == null || t.Field() != InternedFieldName)
                {
                    // TODO: maybe we can handle this like the next()
                    // into null?  set term as prevTerm then dance?

                    if (DEBUG_SURROGATES)
                    {
                        Console.WriteLine("  seek hit EOF");
                    }

                    // We hit EOF; try end-case surrogate dance: if we
                    // find an E, try swapping in S, backwards:
                    ScratchTerm.CopyBytes(term);

                    Debug.Assert(ScratchTerm.Offset == 0);

                    for (int i = ScratchTerm.Length - 1; i >= 0; i--)
                    {
                        if (IsHighBMPChar(ScratchTerm.Bytes, i))
                        {
                            if (DEBUG_SURROGATES)
                            {
                                Console.WriteLine("    found E pos=" + i + "; try seek");
                            }

                            if (SeekToNonBMP(SeekTermEnum, ScratchTerm, i))
                            {
                                ScratchTerm.CopyBytes(SeekTermEnum.Term().Bytes());
                                OuterInstance.TermsDict.SeekEnum(TermEnum, SeekTermEnum.Term(), false);

                                NewSuffixStart = 1 + i;

                                DoPushes();

                                // Found a match
                                // TODO: faster seek?
                                Current = TermEnum.Term().Bytes();
                                return SeekStatus.NOT_FOUND;
                            }
                        }
                    }

                    if (DEBUG_SURROGATES)
                    {
                        Console.WriteLine("  seek END");
                    }

                    Current = null;
                    return SeekStatus.END;
                }
                else
                {
                    // We found a non-exact but non-null term; this one
                    // is fun -- just treat it like next, by pretending
                    // requested term was prev:
                    PrevTerm.CopyBytes(term);

                    if (DEBUG_SURROGATES)
                    {
                        Console.WriteLine("  seek hit non-exact term=" + UnicodeUtil.ToHexString(t.Text()));
                    }

                    BytesRef br = t.Bytes();
                    Debug.Assert(br.Offset == 0);

                    SetNewSuffixStart(term, br);

                    SurrogateDance();

                    Term t2 = TermEnum.Term();
                    if (t2 == null || t2.Field() != InternedFieldName)
                    {
                        // PreFlex codec interns field names; verify:
//.........这里部分代码省略.........

        public void TestFieldZzz()
        {
            try
            {
                indexReader = DirectoryReader.Open(store);

                ld = new LuceneDictionary(indexReader, "zzz");
                it = ld.EntryIterator;

                assertNotNull("First element doesn't exist.", spare = it.Next());
                assertEquals("First element isn't correct", "bar", spare.Utf8ToString());
                assertNull("More elements than expected", it.Next());
            }
            finally
            {
                if (indexReader != null) { indexReader.Dispose(); }
            }
        }

 // for debugging
 internal virtual string BrToString(BytesRef b)
 {
     if (b == null)
     {
         return "null";
     }
     else
     {
         try
         {
             return b.Utf8ToString() + " " + b;
         }
         catch (Exception)
         {
             // If BytesRef isn't actually UTF8, or it's eg a
             // prefix of UTF8 that ends mid-unicode-char, we
             // fallback to hex:
             return b.ToString();
         }
     }
 }

            // Swap in S, in place of E:
            internal virtual bool SeekToNonBMP(SegmentTermEnum te, BytesRef term, int pos)
            {
                int savLength = term.Length;

                Debug.Assert(term.Offset == 0);

                // The 3 bytes starting at downTo make up 1
                // unicode character:
                Debug.Assert(IsHighBMPChar(term.Bytes, pos));

                // NOTE: we cannot make this assert, because
                // AutomatonQuery legitimately sends us malformed UTF8
                // (eg the UTF8 bytes with just 0xee)
                // assert term.length >= pos + 3: "term.length=" + term.length + " pos+3=" + (pos+3) + " byte=" + Integer.toHexString(term.bytes[pos]) + " term=" + term.toString();

                // Save the bytes && length, since we need to
                // restore this if seek "back" finds no matching
                // terms
                if (term.Bytes.Length < 4 + pos)
                {
                    term.Grow(4 + pos);
                }

                Scratch[0] = (sbyte)term.Bytes[pos];
                Scratch[1] = (sbyte)term.Bytes[pos + 1];
                Scratch[2] = (sbyte)term.Bytes[pos + 2];

                term.Bytes[pos] = unchecked((byte)0xf0);
                term.Bytes[pos + 1] = unchecked((byte)0x90);
                term.Bytes[pos + 2] = unchecked((byte)0x80);
                term.Bytes[pos + 3] = unchecked((byte)0x80);
                term.Length = 4 + pos;

                if (DEBUG_SURROGATES)
                {
                    Console.WriteLine("      try seek term=" + UnicodeUtil.ToHexString(term.Utf8ToString()));
                }

                // Seek "back":
                OuterInstance.TermsDict.SeekEnum(te, new Term(fieldInfo.Name, term), true);

                // Test if the term we seek'd to in fact found a
                // surrogate pair at the same position as the E:
                Term t2 = te.Term();

                // Cannot be null (or move to next field) because at
                // "worst" it'd seek to the same term we are on now,
                // unless we are being called from seek
                if (t2 == null || t2.Field() != InternedFieldName)
                {
                    return false;
                }

                if (DEBUG_SURROGATES)
                {
                    Console.WriteLine("      got term=" + UnicodeUtil.ToHexString(t2.Text()));
                }

                // Now test if prefix is identical and we found
                // a non-BMP char at the same position:
                BytesRef b2 = t2.Bytes();
                Debug.Assert(b2.Offset == 0);

                bool matches;
                if (b2.Length >= term.Length && IsNonBMPChar(b2.Bytes, pos))
                {
                    matches = true;
                    for (int i = 0; i < pos; i++)
                    {
                        if (term.Bytes[i] != b2.Bytes[i])
                        {
                            matches = false;
                            break;
                        }
                    }
                }
                else
                {
                    matches = false;
                }

                // Restore term:
                term.Length = savLength;
                term.Bytes[pos] = (byte)Scratch[0];
                term.Bytes[pos + 1] = (byte)Scratch[1];
                term.Bytes[pos + 2] = (byte)Scratch[2];

                return matches;
            }

        private void VerifyEnum(ThreadState threadState, string field, BytesRef term, TermsEnum termsEnum, FieldInfo.IndexOptions maxTestOptions, FieldInfo.IndexOptions maxIndexOptions, ISet<Option> options, bool alwaysTestMax)
        // Maximum options (docs/freqs/positions/offsets) to test:
        {
            if (VERBOSE)
            {
                Console.WriteLine("  verifyEnum: options=" + options + " maxTestOptions=" + maxTestOptions);
            }

            // Make sure TermsEnum really is positioned on the
            // expected term:
            Assert.AreEqual(term, termsEnum.Term());

            // 50% of the time time pass liveDocs:
            bool useLiveDocs = options.Contains(Option.LIVE_DOCS) && Random().NextBoolean();
            Bits liveDocs;
            if (useLiveDocs)
            {
                liveDocs = GlobalLiveDocs;
                if (VERBOSE)
                {
                    Console.WriteLine("  use liveDocs");
                }
            }
            else
            {
                liveDocs = null;
                if (VERBOSE)
                {
                    Console.WriteLine("  no liveDocs");
                }
            }

            FieldInfo fieldInfo = CurrentFieldInfos.FieldInfo(field);

            // NOTE: can be empty list if we are using liveDocs:
            SeedPostings expected = GetSeedPostings(term.Utf8ToString(), Fields[field][term], useLiveDocs, maxIndexOptions);
            Assert.AreEqual(expected.DocFreq, termsEnum.DocFreq());

            bool allowFreqs = fieldInfo.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS && maxTestOptions.CompareTo(FieldInfo.IndexOptions.DOCS_AND_FREQS) >= 0;
            bool doCheckFreqs = allowFreqs && (alwaysTestMax || Random().Next(3) <= 2);

            bool allowPositions = fieldInfo.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS && maxTestOptions.CompareTo(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
            bool doCheckPositions = allowPositions && (alwaysTestMax || Random().Next(3) <= 2);

            bool allowOffsets = fieldInfo.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS && maxTestOptions.CompareTo(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
            bool doCheckOffsets = allowOffsets && (alwaysTestMax || Random().Next(3) <= 2);

            bool doCheckPayloads = options.Contains(Option.PAYLOADS) && allowPositions && fieldInfo.HasPayloads() && (alwaysTestMax || Random().Next(3) <= 2);

            DocsEnum prevDocsEnum = null;

            DocsEnum docsEnum;
            DocsAndPositionsEnum docsAndPositionsEnum;

            if (!doCheckPositions)
            {
                if (allowPositions && Random().Next(10) == 7)
                {
                    // 10% of the time, even though we will not check positions, pull a DocsAndPositions enum

                    if (options.Contains(Option.REUSE_ENUMS) && Random().Next(10) < 9)
                    {
                        prevDocsEnum = threadState.ReuseDocsAndPositionsEnum;
                    }

                    int flags = 0;
                    if (alwaysTestMax || Random().NextBoolean())
                    {
                        flags |= DocsAndPositionsEnum.FLAG_OFFSETS;
                    }
                    if (alwaysTestMax || Random().NextBoolean())
                    {
                        flags |= DocsAndPositionsEnum.FLAG_PAYLOADS;
                    }

                    if (VERBOSE)
                    {
                        Console.WriteLine("  get DocsAndPositionsEnum (but we won't check positions) flags=" + flags);
                    }

                    threadState.ReuseDocsAndPositionsEnum = termsEnum.DocsAndPositions(liveDocs, (DocsAndPositionsEnum)prevDocsEnum, flags);
                    docsEnum = threadState.ReuseDocsAndPositionsEnum;
                    docsAndPositionsEnum = threadState.ReuseDocsAndPositionsEnum;
                }
                else
                {
                    if (VERBOSE)
                    {
                        Console.WriteLine("  get DocsEnum");
                    }
                    if (options.Contains(Option.REUSE_ENUMS) && Random().Next(10) < 9)
                    {
                        prevDocsEnum = threadState.ReuseDocsEnum;
                    }
                    threadState.ReuseDocsEnum = termsEnum.Docs(liveDocs, prevDocsEnum, doCheckFreqs ? DocsEnum.FLAG_FREQS : DocsEnum.FLAG_NONE);
                    docsEnum = threadState.ReuseDocsEnum;
                    docsAndPositionsEnum = null;
                }
            }
            else
//.........这里部分代码省略.........

 public override string StrVal(int doc)
 {
     var bytes = new BytesRef();
     return BytesVal(doc, bytes) ? bytes.Utf8ToString() : null;
 }

        public virtual void TestMultipleDocValuesTypes()
        {
            Directory dir = NewDirectory();
            IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
            conf.SetMaxBufferedDocs(10); // prevent merges
            IndexWriter writer = new IndexWriter(dir, conf);

            for (int i = 0; i < 4; i++)
            {
                Document doc = new Document();
                doc.Add(new StringField("dvUpdateKey", "dv", Store.NO));
                doc.Add(new NumericDocValuesField("ndv", i));
                doc.Add(new BinaryDocValuesField("bdv", new BytesRef(Convert.ToString(i))));
                doc.Add(new SortedDocValuesField("sdv", new BytesRef(Convert.ToString(i))));
                doc.Add(new SortedSetDocValuesField("ssdv", new BytesRef(Convert.ToString(i))));
                doc.Add(new SortedSetDocValuesField("ssdv", new BytesRef(Convert.ToString(i * 2))));
                writer.AddDocument(doc);
            }
            writer.Commit();

            // update all docs' bdv field
            writer.UpdateBinaryDocValue(new Term("dvUpdateKey", "dv"), "bdv", ToBytes(17L));
            writer.Dispose();

            DirectoryReader reader = DirectoryReader.Open(dir);
            AtomicReader r = (AtomicReader)reader.Leaves[0].Reader;
            NumericDocValues ndv = r.GetNumericDocValues("ndv");
            BinaryDocValues bdv = r.GetBinaryDocValues("bdv");
            SortedDocValues sdv = r.GetSortedDocValues("sdv");
            SortedSetDocValues ssdv = r.GetSortedSetDocValues("ssdv");
            BytesRef scratch = new BytesRef();
            for (int i = 0; i < r.MaxDoc; i++)
            {
                Assert.AreEqual(i, ndv.Get(i));
                Assert.AreEqual(17, GetValue(bdv, i, scratch));
                sdv.Get(i, scratch);
                Assert.AreEqual(new BytesRef(Convert.ToString(i)), scratch);
                ssdv.Document = i;
                long ord = ssdv.NextOrd();
                ssdv.LookupOrd(ord, scratch);
                Assert.AreEqual(i, Convert.ToInt32(scratch.Utf8ToString()));
                if (i != 0)
                {
                    ord = ssdv.NextOrd();
                    ssdv.LookupOrd(ord, scratch);
                    Assert.AreEqual(i * 2, Convert.ToInt32(scratch.Utf8ToString()));
                }
                Assert.AreEqual(SortedSetDocValues.NO_MORE_ORDS, ssdv.NextOrd());
            }

            reader.Dispose();
            dir.Dispose();
        }

        public void TestCodecUsesOwnSortedBytesEachTime()
        {
            Analyzer analyzer = new MockAnalyzer(Random());

            Directory directory = NewDirectory();
            IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
            conf.SetMergePolicy(NewLogMergePolicy());
            RandomIndexWriter iwriter = new RandomIndexWriter(Random(), directory, conf);
            Document doc = new Document();
            doc.Add(new SortedDocValuesField("dv", new BytesRef("foo!")));
            iwriter.AddDocument(doc);
            doc = new Document();
            doc.Add(new SortedDocValuesField("dv", new BytesRef("bar!")));
            iwriter.AddDocument(doc);
            iwriter.Dispose();

            // Now search the index:
            IndexReader ireader = DirectoryReader.Open(directory); // read-only=true
            Debug.Assert(ireader.Leaves.Count == 1);
            BinaryDocValues dv = ((AtomicReader)ireader.Leaves[0].Reader).GetSortedDocValues("dv");
            BytesRef scratch = new BytesRef();
            dv.Get(0, scratch);
            Assert.AreEqual("foo!", scratch.Utf8ToString());

            BytesRef scratch2 = new BytesRef();
            dv.Get(1, scratch2);
            Assert.AreEqual("bar!", scratch2.Utf8ToString());
            // check scratch is still valid
            Assert.AreEqual("foo!", scratch.Utf8ToString());

            ireader.Dispose();
            directory.Dispose();
        }