C# BytesRef.Grow方法代码示例

 public virtual void TestReadAndWrite()
 {
     Counter bytesUsed = Util.Counter.NewCounter();
     ByteBlockPool pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(bytesUsed));
     pool.NextBuffer();
     bool reuseFirst = Random().NextBoolean();
     for (int j = 0; j < 2; j++)
     {
         IList<BytesRef> list = new List<BytesRef>();
         int maxLength = AtLeast(500);
         int numValues = AtLeast(100);
         BytesRef @ref = new BytesRef();
         for (int i = 0; i < numValues; i++)
         {
             string value = TestUtil.RandomRealisticUnicodeString(Random(), maxLength);
             list.Add(new BytesRef(value));
             @ref.CopyChars(value);
             pool.Append(@ref);
         }
         // verify
         long position = 0;
         foreach (BytesRef expected in list)
         {
             @ref.Grow(expected.Length);
             @ref.Length = expected.Length;
             pool.ReadBytes(position, @ref.Bytes, @ref.Offset, @ref.Length);
             Assert.AreEqual(expected, @ref);
             position += @ref.Length;
         }
         pool.Reset(Random().NextBoolean(), reuseFirst);
         if (reuseFirst)
         {
             Assert.AreEqual(ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed.Get());
         }
         else
         {
             Assert.AreEqual(0, bytesUsed.Get());
             pool.NextBuffer(); // prepare for next iter
         }
     }
 }

        // TODO: this currently requites a determinized machine,
        // but it need not -- we can speed it up by walking the
        // NFA instead.  it'd still be fail fast.
        public static BytesRef GetCommonPrefixBytesRef(Automaton a)
        {
            if (a.IsSingleton)
            {
                return new BytesRef(a.singleton);
            }
            BytesRef @ref = new BytesRef(10);
            HashSet<State> visited = new HashSet<State>();
            State s = a.Initial;
            bool done;
            do
            {
                done = true;
                visited.Add(s);
                if (!s.accept && s.NumTransitions() == 1)
                {
                    var iter = s.Transitions.GetEnumerator();
                    iter.MoveNext();
                    Transition t = iter.Current;

                    if (t.Min_Renamed == t.Max_Renamed && !visited.Contains(t.To))
                    {
                        @ref.Grow([email protected]);
                        @ref.Bytes[@ref.Length - 1] = (byte)t.Min_Renamed;
                        s = t.To;
                        done = false;
                    }
                }
            } while (!done);
            return @ref;
        }

		/// <summary>
		/// Builds an <seealso cref="SynonymMap"/> and returns it.
		/// </summary>
		public virtual SynonymMap Build()
		{
		  ByteSequenceOutputs outputs = ByteSequenceOutputs.Singleton;
		  // TODO: are we using the best sharing options?
		  var builder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE4, outputs);

		  BytesRef scratch = new BytesRef(64);
		  ByteArrayDataOutput scratchOutput = new ByteArrayDataOutput();

		  HashSet<int?> dedupSet;

		  if (dedup)
		  {
			dedupSet = new HashSet<int?>();
		  }
		  else
		  {
			dedupSet = null;
		  }

		  
            var spare = new sbyte[5];

		  Dictionary<CharsRef, MapEntry>.KeyCollection keys = workingSet.Keys;
		  CharsRef[] sortedKeys = keys.ToArray();
		  Arrays.Sort(sortedKeys, CharsRef.UTF16SortedAsUTF8Comparator);

		  IntsRef scratchIntsRef = new IntsRef();

		  //System.out.println("fmap.build");
		  for (int keyIdx = 0; keyIdx < sortedKeys.Length; keyIdx++)
		  {
			CharsRef input = sortedKeys[keyIdx];
			MapEntry output = workingSet[input];

			int numEntries = output.ords.Count;
			// output size, assume the worst case
			int estimatedSize = 5 + numEntries * 5; // numEntries + one ord for each entry

			scratch.Grow(estimatedSize);
			scratchOutput.Reset(scratch.Bytes, scratch.Offset, scratch.Bytes.Length);
			Debug.Assert(scratch.Offset == 0);

			// now write our output data:
			int count = 0;
			for (int i = 0; i < numEntries; i++)
			{
			  if (dedupSet != null)
			  {
				// box once
				int? ent = output.ords[i];
				if (dedupSet.Contains(ent))
				{
				  continue;
				}
				dedupSet.Add(ent);
			  }
			  scratchOutput.WriteVInt(output.ords[i]);
			  count++;
			}

			int pos = scratchOutput.Position;
			scratchOutput.WriteVInt(count << 1 | (output.includeOrig ? 0 : 1));
			int pos2 = scratchOutput.Position;
			int vIntLen = pos2 - pos;

			// Move the count + includeOrig to the front of the byte[]:
			Array.Copy(scratch.Bytes, pos, spare, 0, vIntLen);
			Array.Copy(scratch.Bytes, 0, scratch.Bytes, vIntLen, pos);
			Array.Copy(spare, 0, scratch.Bytes, 0, vIntLen);

			if (dedupSet != null)
			{
			  dedupSet.Clear();
			}

			scratch.Length = scratchOutput.Position - scratch.Offset;
			//System.out.println("  add input=" + input + " output=" + scratch + " offset=" + scratch.offset + " length=" + scratch.length + " count=" + count);
			builder.Add(Util.ToUTF32(input, scratchIntsRef), BytesRef.DeepCopyOf(scratch));
		  }

		  FST<BytesRef> fst = builder.Finish();
		  return new SynonymMap(fst, words, maxHorizontalContext);
		}

 /// <summary>
 /// Returns the <i>n'th</i> element of this <seealso cref="BytesRefArray"/> </summary>
 /// <param name="spare"> a spare <seealso cref="BytesRef"/> instance </param>
 /// <param name="index"> the elements index to retrieve </param>
 /// <returns> the <i>n'th</i> element of this <seealso cref="BytesRefArray"/> </returns>
 public BytesRef Get(BytesRef spare, int index)
 {
     if (LastElement > index)
     {
         int offset = Offsets[index];
         int length = index == LastElement - 1 ? CurrentOffset - offset : Offsets[index + 1] - offset;
         Debug.Assert(spare.Offset == 0);
         spare.Grow(length);
         spare.Length = length;
         Pool.ReadBytes(offset, spare.Bytes, spare.Offset, spare.Length);
         return spare;
     }
     throw new System.IndexOutOfRangeException("index " + index + " must be less than the size: " + LastElement);
 }

            // Swap in S, in place of E:
            internal virtual bool SeekToNonBMP(SegmentTermEnum te, BytesRef term, int pos)
            {
                int savLength = term.Length;

                Debug.Assert(term.Offset == 0);

                // The 3 bytes starting at downTo make up 1
                // unicode character:
                Debug.Assert(IsHighBMPChar(term.Bytes, pos));

                // NOTE: we cannot make this assert, because
                // AutomatonQuery legitimately sends us malformed UTF8
                // (eg the UTF8 bytes with just 0xee)
                // assert term.length >= pos + 3: "term.length=" + term.length + " pos+3=" + (pos+3) + " byte=" + Integer.toHexString(term.bytes[pos]) + " term=" + term.toString();

                // Save the bytes && length, since we need to
                // restore this if seek "back" finds no matching
                // terms
                if (term.Bytes.Length < 4 + pos)
                {
                    term.Grow(4 + pos);
                }

                Scratch[0] = (sbyte)term.Bytes[pos];
                Scratch[1] = (sbyte)term.Bytes[pos + 1];
                Scratch[2] = (sbyte)term.Bytes[pos + 2];

                term.Bytes[pos] = unchecked((byte)0xf0);
                term.Bytes[pos + 1] = unchecked((byte)0x90);
                term.Bytes[pos + 2] = unchecked((byte)0x80);
                term.Bytes[pos + 3] = unchecked((byte)0x80);
                term.Length = 4 + pos;

                if (DEBUG_SURROGATES)
                {
                    Console.WriteLine("      try seek term=" + UnicodeUtil.ToHexString(term.Utf8ToString()));
                }

                // Seek "back":
                OuterInstance.TermsDict.SeekEnum(te, new Term(fieldInfo.Name, term), true);

                // Test if the term we seek'd to in fact found a
                // surrogate pair at the same position as the E:
                Term t2 = te.Term();

                // Cannot be null (or move to next field) because at
                // "worst" it'd seek to the same term we are on now,
                // unless we are being called from seek
                if (t2 == null || t2.Field() != InternedFieldName)
                {
                    return false;
                }

                if (DEBUG_SURROGATES)
                {
                    Console.WriteLine("      got term=" + UnicodeUtil.ToHexString(t2.Text()));
                }

                // Now test if prefix is identical and we found
                // a non-BMP char at the same position:
                BytesRef b2 = t2.Bytes();
                Debug.Assert(b2.Offset == 0);

                bool matches;
                if (b2.Length >= term.Length && IsNonBMPChar(b2.Bytes, pos))
                {
                    matches = true;
                    for (int i = 0; i < pos; i++)
                    {
                        if (term.Bytes[i] != b2.Bytes[i])
                        {
                            matches = false;
                            break;
                        }
                    }
                }
                else
                {
                    matches = false;
                }

                // Restore term:
                term.Length = savLength;
                term.Bytes[pos] = (byte)Scratch[0];
                term.Bytes[pos + 1] = (byte)Scratch[1];
                term.Bytes[pos + 2] = (byte)Scratch[2];

                return matches;
            }

 /// <summary>
 /// Compare the fields of the terms first, and if not equals return from
 /// compare. If equal compare terms.
 /// </summary>
 /// <param name="term">
 ///          the term to compare. </param>
 /// <param name="termIndex">
 ///          the position of the term in the input to compare </param>
 /// <param name="input">
 ///          the input buffer. </param>
 /// <returns> int. </returns>
 /// <exception cref="IOException"> If there is a low-level I/O error. </exception>
 private int CompareTo(Term term, int termIndex, PagedBytesDataInput input, BytesRef reuse)
 {
     // if term field does not equal mid's field index, then compare fields
     // else if they are equal, compare term's string values...
     int c = CompareField(term, termIndex, input);
     if (c == 0)
     {
         reuse.Length = input.ReadVInt();
         reuse.Grow(reuse.Length);
         input.ReadBytes(reuse.Bytes, 0, reuse.Length);
         return Comparator.Compare(term.Bytes(), reuse);
     }
     return c;
 }

        public override Fields Get(int doc)
        {
            var fields = new SortedDictionary<string, SimpleTVTerms>();

            _input.Seek(_offsets[doc]);
            ReadLine();
            Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextTermVectorsWriter.NUMFIELDS));
            var numFields = ParseIntAt(SimpleTextTermVectorsWriter.NUMFIELDS.Length);
            if (numFields == 0)
            {
                return null; // no vectors for this doc
            }
            for (var i = 0; i < numFields; i++)
            {
                ReadLine();
                Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextTermVectorsWriter.FIELD));
                // skip fieldNumber:
                ParseIntAt(SimpleTextTermVectorsWriter.FIELD.Length);

                ReadLine();
                Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextTermVectorsWriter.FIELDNAME));
                var fieldName = ReadString(SimpleTextTermVectorsWriter.FIELDNAME.Length, _scratch);

                ReadLine();
                Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextTermVectorsWriter.FIELDPOSITIONS));
                var positions = Convert.ToBoolean(ReadString(SimpleTextTermVectorsWriter.FIELDPOSITIONS.Length, _scratch), CultureInfo.InvariantCulture);

                ReadLine();
                Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextTermVectorsWriter.FIELDOFFSETS));
                var offsets = Convert.ToBoolean(ReadString(SimpleTextTermVectorsWriter.FIELDOFFSETS.Length, _scratch), CultureInfo.InvariantCulture);

                ReadLine();
                Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextTermVectorsWriter.FIELDPAYLOADS));
                var payloads = Convert.ToBoolean(ReadString(SimpleTextTermVectorsWriter.FIELDPAYLOADS.Length, _scratch), CultureInfo.InvariantCulture);

                ReadLine();
                Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextTermVectorsWriter.FIELDTERMCOUNT));
                var termCount = ParseIntAt(SimpleTextTermVectorsWriter.FIELDTERMCOUNT.Length);

                var terms = new SimpleTVTerms(offsets, positions, payloads);
                fields.Add(fieldName, terms);

                for (var j = 0; j < termCount; j++)
                {
                    ReadLine();
                    Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextTermVectorsWriter.TERMTEXT));
                    var term = new BytesRef();
                    var termLength = _scratch.Length - SimpleTextTermVectorsWriter.TERMTEXT.Length;
                    term.Grow(termLength);
                    term.Length = termLength;
                    Array.Copy(_scratch.Bytes, _scratch.Offset + SimpleTextTermVectorsWriter.TERMTEXT.Length, term.Bytes, term.Offset, termLength);

                    var postings = new SimpleTVPostings();
                    terms.TERMS.Add(term, postings);

                    ReadLine();
                    Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextTermVectorsWriter.TERMFREQ));
                    postings.FREQ = ParseIntAt(SimpleTextTermVectorsWriter.TERMFREQ.Length);

                    if (!positions && !offsets) continue;

                    if (positions)
                    {
                        postings.POSITIONS = new int[postings.FREQ];
                        if (payloads)
                        {
                            postings.PAYLOADS = new BytesRef[postings.FREQ];
                        }
                    }

                    if (offsets)
                    {
                        postings.START_OFFSETS = new int[postings.FREQ];
                        postings.END_OFFSETS = new int[postings.FREQ];
                    }

                    for (var k = 0; k < postings.FREQ; k++)
                    {
                        if (positions)
                        {
                            ReadLine();
                            Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextTermVectorsWriter.POSITION));
                            postings.POSITIONS[k] = ParseIntAt(SimpleTextTermVectorsWriter.POSITION.Length);
                            if (payloads)
                            {
                                ReadLine();
                                Debug.Assert(StringHelper.StartsWith(_scratch, SimpleTextTermVectorsWriter.PAYLOAD));
                                if (_scratch.Length - SimpleTextTermVectorsWriter.PAYLOAD.Length == 0)
                                {
                                    postings.PAYLOADS[k] = null;
                                }
                                else
                                {
                                    var payloadBytes = new byte[_scratch.Length - SimpleTextTermVectorsWriter.PAYLOAD.Length];
                                    Array.Copy(_scratch.Bytes, _scratch.Offset + SimpleTextTermVectorsWriter.PAYLOAD.Length, payloadBytes, 0,
                                        payloadBytes.Length);
                                    postings.PAYLOADS[k] = new BytesRef(payloadBytes);
                                }
                            }
                        }
//.........这里部分代码省略.........

        /// <summary>
        /// Finds largest term accepted by this Automaton, that's
        ///  <= the provided input term.  The result is placed in
        ///  output; it's fine for output and input to point to
        ///  the same BytesRef.  The returned result is either the
        ///  provided output, or null if there is no floor term
        ///  (ie, the provided input term is before the first term
        ///  accepted by this Automaton).
        /// </summary>
        public virtual BytesRef Floor(BytesRef input, BytesRef output)
        {
            output.Offset = 0;
            //if (DEBUG) System.out.println("CA.floor input=" + input.utf8ToString());

            int state = RunAutomaton.InitialState;

            // Special case empty string:
            if (input.Length == 0)
            {
                if (RunAutomaton.IsAccept(state))
                {
                    output.Length = 0;
                    return output;
                }
                else
                {
                    return null;
                }
            }

            IList<int> stack = new List<int>();

            int idx = 0;
            while (true)
            {
                int label = ((sbyte)input.Bytes[input.Offset + idx]) & 0xff;
                int nextState = RunAutomaton.Step(state, label);
                //if (DEBUG) System.out.println("  cycle label=" + (char) label + " nextState=" + nextState);

                if (idx == input.Length - 1)
                {
                    if (nextState != -1 && RunAutomaton.IsAccept(nextState))
                    {
                        // Input string is accepted
                        if (idx >= output.Bytes.Length)
                        {
                            output.Grow(1 + idx);
                        }
                        output.Bytes[idx] = (byte)label;
                        output.Length = input.Length;
                        //if (DEBUG) System.out.println("  input is accepted; return term=" + output.utf8ToString());
                        return output;
                    }
                    else
                    {
                        nextState = -1;
                    }
                }

                if (nextState == -1)
                {
                    // Pop back to a state that has a transition
                    // <= our label:
                    while (true)
                    {
                        Transition[] transitions = SortedTransitions[state];
                        if (transitions.Length == 0)
                        {
                            Debug.Assert(RunAutomaton.IsAccept(state));
                            output.Length = idx;
                            //if (DEBUG) System.out.println("  return " + output.utf8ToString());
                            return output;
                        }
                        else if (label - 1 < transitions[0].Min_Renamed)
                        {
                            if (RunAutomaton.IsAccept(state))
                            {
                                output.Length = idx;
                                //if (DEBUG) System.out.println("  return " + output.utf8ToString());
                                return output;
                            }
                            // pop
                            if (stack.Count == 0)
                            {
                                //if (DEBUG) System.out.println("  pop ord=" + idx + " return null");
                                return null;
                            }
                            else
                            {
                                state = stack[stack.Count - 1];
                                stack.RemoveAt(stack.Count - 1);
                                idx--;
                                //if (DEBUG) System.out.println("  pop ord=" + (idx+1) + " label=" + (char) label + " first trans.min=" + (char) transitions[0].min);
                                label = input.Bytes[input.Offset + idx] & 0xff;
                            }
                        }
                        else
                        {
                            //if (DEBUG) System.out.println("  stop pop ord=" + idx + " first trans.min=" + (char) transitions[0].min);
                            break;
//.........这里部分代码省略.........

        //private static final boolean DEBUG = BlockTreeTermsWriter.DEBUG;

        private BytesRef AddTail(int state, BytesRef term, int idx, int leadLabel)
        {
            // Find biggest transition that's < label
            // TODO: use binary search here
            Transition maxTransition = null;
            foreach (Transition transition in SortedTransitions[state])
            {
                if (transition.Min_Renamed < leadLabel)
                {
                    maxTransition = transition;
                }
            }

            Debug.Assert(maxTransition != null);

            // Append floorLabel
            int floorLabel;
            if (maxTransition.Max_Renamed > leadLabel - 1)
            {
                floorLabel = leadLabel - 1;
            }
            else
            {
                floorLabel = maxTransition.Max_Renamed;
            }
            if (idx >= term.Bytes.Length)
            {
                term.Grow(1 + idx);
            }
            //if (DEBUG) System.out.println("  add floorLabel=" + (char) floorLabel + " idx=" + idx);
            term.Bytes[idx] = (byte)floorLabel;

            state = maxTransition.To.Number;
            idx++;

            // Push down to last accept state
            while (true)
            {
                Transition[] transitions = SortedTransitions[state];
                if (transitions.Length == 0)
                {
                    Debug.Assert(RunAutomaton.IsAccept(state));
                    term.Length = idx;
                    //if (DEBUG) System.out.println("  return " + term.utf8ToString());
                    return term;
                }
                else
                {
                    // We are pushing "top" -- so get last label of
                    // last transition:
                    Debug.Assert(transitions.Length != 0);
                    Transition lastTransition = transitions[transitions.Length - 1];
                    if (idx >= term.Bytes.Length)
                    {
                        term.Grow(1 + idx);
                    }
                    //if (DEBUG) System.out.println("  push maxLabel=" + (char) lastTransition.max + " idx=" + idx);
                    term.Bytes[idx] = (byte)lastTransition.Max_Renamed;
                    state = lastTransition.To.Number;
                    idx++;
                }
            }
        }

        private IEnumerable<BytesRef> GetBytesIterator(int maxDocParam)
        {
            // Use yield return instead of ucsom IEnumerable

            AppendingDeltaPackedLongBuffer.Iterator lengthsIterator = Lengths.GetIterator();
            int size = (int)Lengths.Size();
            DataInput bytesIterator = Bytes.DataInput;
            int maxDoc = maxDocParam;
            int upto = 0;

            while (upto < maxDoc)
            {
                BytesRef v = null;
                if (upto < size)
                {
                    int length = (int)lengthsIterator.Next();
                    var value = new BytesRef();
                    value.Grow(length);
                    value.Length = length;
                    bytesIterator.ReadBytes(value.Bytes, value.Offset, value.Length);

                    if (DocsWithField.Get(upto))
                    {
                        v = value;
                    }
                }

                upto++;
                yield return v;
            }
        }

        private IEnumerable<BytesRef> GetBytesIterator(int maxDocParam)
        {
            // Use yield return instead of ucsom IEnumerable

            BytesRef value = new BytesRef();
            AppendingDeltaPackedLongBuffer.Iterator lengthsIterator = (AppendingDeltaPackedLongBuffer.Iterator)Lengths.GetIterator();
            int size = (int)Lengths.Size();
            DataInput bytesIterator = Bytes.DataInput;
            int maxDoc = maxDocParam;
            int upto = 0;
            long byteOffset = 0L;

            while (upto < maxDoc)
            {
                if (upto < size)
                {
                    int length = (int)lengthsIterator.Next();
                    value.Grow(length);
                    value.Length = length;
                    //LUCENE TODO: This modification is slightly fishy, 4x port uses ByteBlockPool
                    bytesIterator.ReadBytes(/*byteOffset,*/ value.Bytes, value.Offset, value.Length);
                    byteOffset += length;
                }
                else
                {
                    // This is to handle last N documents not having
                    // this DV field in the end of the segment:
                    value.Length = 0;
                }

                upto++;
                yield return value;
            }
        }