Golang document.NewDocument函数代码示例

func TestEnablingDisablingStoringDynamicFields(t *testing.T) {

	data := map[string]interface{}{
		"name": "bleve",
	}
	doc := document.NewDocument("x")
	mapping := NewIndexMapping()
	err := mapping.mapDocument(doc, data)
	if err != nil {
		t.Fatal(err)
	}
	for _, field := range doc.Fields {
		if field.Name() == "name" && !field.Options().IsStored() {
			t.Errorf("expected field 'name' to be stored, isn't")
		}
	}

	StoreDynamic = false
	defer func() {
		StoreDynamic = true
	}()

	doc = document.NewDocument("y")
	err = mapping.mapDocument(doc, data)
	if err != nil {
		t.Fatal(err)
	}
	for _, field := range doc.Fields {
		if field.Name() == "name" && field.Options().IsStored() {
			t.Errorf("expected field 'name' to be not stored, is")
		}
	}
}

func TestIndexInsertMultiple(t *testing.T) {
	defer os.RemoveAll("test")

	store, err := boltdb.Open("test", "bleve")
	analysisQueue := NewAnalysisQueue(1)
	idx := NewUpsideDownCouch(store, analysisQueue)
	err = idx.Open()
	if err != nil {
		t.Errorf("error opening index: %v", err)
	}

	var expectedCount uint64

	doc := document.NewDocument("1")
	doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
	err = idx.Update(doc)
	if err != nil {
		t.Errorf("Error updating index: %v", err)
	}
	expectedCount++

	doc = document.NewDocument("2")
	doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
	err = idx.Update(doc)
	if err != nil {
		t.Errorf("Error updating index: %v", err)
	}
	expectedCount++

	// should have 4 rows (1 for version, 1 for schema field, and 2 for single term, and 1 for the term count,  and 2 for the back index entries)
	expectedLength := uint64(1 + 1 + 2 + 1 + 2)
	rowCount := idx.rowCount()
	if rowCount != expectedLength {
		t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
	}

	// close and reopen and and one more to testing counting works correctly
	idx.Close()
	store, err = boltdb.Open("test", "bleve")
	idx = NewUpsideDownCouch(store, analysisQueue)
	err = idx.Open()
	if err != nil {
		t.Errorf("error opening index: %v", err)
	}
	defer idx.Close()

	doc = document.NewDocument("3")
	doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
	err = idx.Update(doc)
	if err != nil {
		t.Errorf("Error updating index: %v", err)
	}
	expectedCount++

	docCount := idx.DocCount()
	if docCount != expectedCount {
		t.Errorf("expected doc count: %d, got %d", expectedCount, docCount)
	}
}

func TestEnablingDisablingStoringDynamicFields(t *testing.T) {

	// first verify that with system defaults, dynamic field is stored
	data := map[string]interface{}{
		"name": "bleve",
	}
	doc := document.NewDocument("x")
	mapping := NewIndexMapping()
	err := mapping.mapDocument(doc, data)
	if err != nil {
		t.Fatal(err)
	}
	for _, field := range doc.Fields {
		if field.Name() == "name" && !field.Options().IsStored() {
			t.Errorf("expected field 'name' to be stored, isn't")
		}
	}

	// now change system level defaults, verify dynamic field is not stored
	StoreDynamic = false
	defer func() {
		StoreDynamic = true
	}()

	mapping = NewIndexMapping()
	doc = document.NewDocument("y")
	err = mapping.mapDocument(doc, data)
	if err != nil {
		t.Fatal(err)
	}
	for _, field := range doc.Fields {
		if field.Name() == "name" && field.Options().IsStored() {
			t.Errorf("expected field 'name' to be not stored, is")
		}
	}

	// now override the system level defaults inside the index mapping
	mapping = NewIndexMapping()
	mapping.StoreDynamic = true
	doc = document.NewDocument("y")
	err = mapping.mapDocument(doc, data)
	if err != nil {
		t.Fatal(err)
	}
	for _, field := range doc.Fields {
		if field.Name() == "name" && !field.Options().IsStored() {
			t.Errorf("expected field 'name' to be stored, isn't")
		}
	}
}

func TestIndexInsertThenUpdate(t *testing.T) {
	defer os.RemoveAll("test")

	store, err := boltdb.Open("test", "bleve")
	analysisQueue := NewAnalysisQueue(1)
	idx := NewUpsideDownCouch(store, analysisQueue)
	err = idx.Open()
	if err != nil {
		t.Errorf("error opening index: %v", err)
	}
	defer idx.Close()

	doc := document.NewDocument("1")
	doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
	err = idx.Update(doc)
	if err != nil {
		t.Errorf("Error updating index: %v", err)
	}

	// this update should overwrite one term, and introduce one new one
	doc = document.NewDocument("1")
	doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test fail"), testAnalyzer))
	err = idx.Update(doc)
	if err != nil {
		t.Errorf("Error deleting entry from index: %v", err)
	}

	// should have 2 row (1 for version, 1 for schema field, and 2 for the two term, and 2 for the term counts, and 1 for the back index entry)
	expectedLength := uint64(1 + 1 + 2 + 2 + 1)
	rowCount := idx.rowCount()
	if rowCount != expectedLength {
		t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
	}

	// now do another update that should remove one of term
	doc = document.NewDocument("1")
	doc.AddField(document.NewTextField("name", []uint64{}, []byte("fail")))
	err = idx.Update(doc)
	if err != nil {
		t.Errorf("Error deleting entry from index: %v", err)
	}

	// should have 2 row (1 for version, 1 for schema field, and 1 for the remaining term, and 1 for the term count, and 1 for the back index entry)
	expectedLength = uint64(1 + 1 + 1 + 1 + 1)
	rowCount = idx.rowCount()
	if rowCount != expectedLength {
		t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
	}
}

func TestIndexDocumentFieldTerms(t *testing.T) {
	defer os.RemoveAll("test")

	store, err := boltdb.Open("test", "bleve")
	if err != nil {
		t.Error(err)
	}
	idx := NewUpsideDownCouch(store)
	err = idx.Open()
	if err != nil {
		t.Errorf("error opening index: %v", err)
	}
	defer idx.Close()

	doc := document.NewDocument("1")
	doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors))
	doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors))
	err = idx.Update(doc)
	if err != nil {
		t.Errorf("Error updating index: %v", err)
	}

	fieldTerms, err := idx.DocumentFieldTerms("1")
	if err != nil {
		t.Error(err)
	}
	expectedFieldTerms := index.FieldTerms{
		"name":  []string{"test"},
		"title": []string{"mister"},
	}
	if !reflect.DeepEqual(fieldTerms, expectedFieldTerms) {
		t.Errorf("expected field terms: %#v, got: %#v", expectedFieldTerms, fieldTerms)
	}
}

func CommonBenchmarkIndexBatch(b *testing.B, create KVStoreCreate, destroy KVStoreDestroy, analysisWorkers, batchSize int) {

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed("standard")
	if err != nil {
		b.Fatal(err)
	}

	b.ResetTimer()
	b.StopTimer()
	for i := 0; i < b.N; i++ {

		s, err := create()
		if err != nil {
			b.Fatal(err)
		}
		analysisQueue := index.NewAnalysisQueue(analysisWorkers)
		idx := NewUpsideDownCouch(s, analysisQueue)

		err = idx.Open()
		if err != nil {
			b.Fatal(err)
		}

		b.StartTimer()
		batch := index.NewBatch()
		for j := 0; j < 1000; j++ {
			if j%batchSize == 0 {
				if len(batch.IndexOps) > 0 {
					err := idx.Batch(batch)
					if err != nil {
						b.Fatal(err)
					}
				}
				batch = index.NewBatch()
			}
			indexDocument := document.NewDocument("").
				AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[j%10]), analyzer))
			indexDocument.ID = strconv.Itoa(i) + "-" + strconv.Itoa(j)
			batch.Update(indexDocument)
		}
		// close last batch
		if len(batch.IndexOps) > 0 {
			err := idx.Batch(batch)
			if err != nil {
				b.Fatal(err)
			}
		}
		b.StopTimer()
		err = idx.Close()
		if err != nil {
			b.Fatal(err)
		}
		err = destroy()
		if err != nil {
			b.Fatal(err)
		}
		analysisQueue.Close()
	}
}

func TestIndexInsertWithStore(t *testing.T) {
	defer os.RemoveAll("test")

	store, err := boltdb.Open("test", "bleve")
	if err != nil {
		t.Error(err)
	}
	analysisQueue := NewAnalysisQueue(1)
	idx := NewUpsideDownCouch(store, analysisQueue)
	err = idx.Open()
	if err != nil {
		t.Errorf("error opening index: %v", err)
	}
	defer idx.Close()

	var expectedCount uint64
	docCount := idx.DocCount()
	if docCount != expectedCount {
		t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
	}

	doc := document.NewDocument("1")
	doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
	err = idx.Update(doc)
	if err != nil {
		t.Errorf("Error updating index: %v", err)
	}
	expectedCount++

	docCount = idx.DocCount()
	if docCount != expectedCount {
		t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
	}

	// should have 6 rows (1 for version, 1 for schema field, and 1 for single term, and 1 for the stored field and 1 for the term count,  and 1 for the back index entry)
	expectedLength := uint64(1 + 1 + 1 + 1 + 1 + 1)
	rowCount := idx.rowCount()
	if rowCount != expectedLength {
		t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
	}

	indexReader := idx.Reader()
	defer indexReader.Close()

	storedDoc, err := indexReader.Document("1")
	if err != nil {
		t.Error(err)
	}

	if len(storedDoc.Fields) != 1 {
		t.Errorf("expected 1 stored field, got %d", len(storedDoc.Fields))
	}
	textField, ok := storedDoc.Fields[0].(*document.TextField)
	if !ok {
		t.Errorf("expected text field")
	}
	if string(textField.Value()) != "test" {
		t.Errorf("expected field content 'test', got '%s'", string(textField.Value()))
	}
}

func (r *firestormReader) Document(id string) (*document.Document, error) {
	docID := []byte(id)
	docNum, err := r.currDocNumForId(docID)
	if err != nil {
		return nil, err
	} else if docNum == 0 {
		return nil, nil
	}
	rv := document.NewDocument(id)
	prefix := StoredPrefixDocIDNum(docID, docNum)
	err = visitPrefix(r.r, prefix, func(key, val []byte) (bool, error) {
		safeVal := make([]byte, len(val))
		copy(safeVal, val)
		row, err := NewStoredRowKV(key, safeVal)
		if err != nil {
			return false, err
		}
		if row != nil {
			fieldName := r.f.fieldCache.FieldIndexed(row.field)
			field := r.decodeFieldType(fieldName, row.arrayPositions, row.value.GetRaw())
			if field != nil {
				rv.AddField(field)
			}
		}
		return true, nil
	})
	if err != nil {
		return nil, err
	}
	return rv, nil
}

func TestIndexTermReaderCompositeFields(t *testing.T) {
	defer func() {
		err := DestroyTest()
		if err != nil {
			t.Fatal(err)
		}
	}()

	analysisQueue := index.NewAnalysisQueue(1)
	idx, err := NewUpsideDownCouch(boltdb.Name, boltTestConfig, analysisQueue)
	if err != nil {
		t.Fatal(err)
	}
	err = idx.Open()
	if err != nil {
		t.Errorf("error opening index: %v", err)
	}
	defer func() {
		err := idx.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	doc := document.NewDocument("1")
	doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors))
	doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors))
	doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField|document.IncludeTermVectors))
	err = idx.Update(doc)
	if err != nil {
		t.Errorf("Error updating index: %v", err)
	}

	indexReader, err := idx.Reader()
	if err != nil {
		t.Error(err)
	}
	defer func() {
		err := indexReader.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	termFieldReader, err := indexReader.TermFieldReader([]byte("mister"), "_all")
	if err != nil {
		t.Error(err)
	}

	tfd, err := termFieldReader.Next()
	for tfd != nil && err == nil {
		if tfd.ID != "1" {
			t.Errorf("expected to find document id 1")
		}
		tfd, err = termFieldReader.Next()
	}
	if err != nil {
		t.Error(err)
	}
}

func TestMappingJSONWithNull(t *testing.T) {

	mapping := NewIndexMapping()

	jsonbytes := []byte(`{"name":"marty", "age": null}`)
	var jsondoc interface{}
	err := json.Unmarshal(jsonbytes, &jsondoc)
	if err != nil {
		t.Fatal(err)
	}

	doc := document.NewDocument("1")
	err = mapping.mapDocument(doc, jsondoc)
	if err != nil {
		t.Fatal(err)
	}
	found := false
	count := 0
	for _, f := range doc.Fields {
		if f.Name() == "name" {
			found = true
		}
		count++
	}
	if !found {
		t.Errorf("expected to find field named 'name'")
	}
	if count != 1 {
		t.Errorf("expected to find 1 find, found %d", count)
	}
}

func BenchmarkAnalyze(b *testing.B) {

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(standard_analyzer.Name)
	if err != nil {
		b.Fatal(err)
	}

	analysisQueue := index.NewAnalysisQueue(1)
	idx, err := NewFirestorm(null.Name, nil, analysisQueue)
	if err != nil {
		b.Fatal(err)
	}

	d := document.NewDocument("1")
	f := document.NewTextFieldWithAnalyzer("desc", nil, bleveWikiArticle1K, analyzer)
	d.AddField(f)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		rv := idx.Analyze(d)
		if len(rv.Rows) < 92 || len(rv.Rows) > 93 {
			b.Fatalf("expected 512-13 rows, got %d", len(rv.Rows))
		}
	}
}

func TestMappingPrimitives(t *testing.T) {

	tests := []struct {
		data interface{}
	}{
		{data: "marty"},
		{data: int(1)},
		{data: int8(2)},
		{data: int16(3)},
		{data: int32(4)},
		{data: int64(5)},
		{data: uint(6)},
		{data: uint8(7)},
		{data: uint16(8)},
		{data: uint32(9)},
		{data: uint64(10)},
		{data: float32(11.0)},
		{data: float64(12.0)},
		{data: false},
	}

	m := NewIndexMapping()
	for _, test := range tests {
		doc := document.NewDocument("x")
		err := m.MapDocument(doc, test.data)
		if err != nil {
			t.Fatal(err)
		}
		if len(doc.Fields) != 1 {
			t.Errorf("expected 1 field, got %d for %v", len(doc.Fields), test.data)
		}
	}
}

func TestAnonymousStructFieldWithJSONStructTagEmptString(t *testing.T) {
	type InterfaceThing interface{}
	type Thing struct {
		InterfaceThing `json:""`
	}

	x := Thing{
		InterfaceThing: map[string]interface{}{
			"key": "value",
		},
	}

	doc := document.NewDocument("1")
	m := NewIndexMapping()
	err := m.MapDocument(doc, x)
	if err != nil {
		t.Fatal(err)
	}

	if len(doc.Fields) != 1 {
		t.Fatalf("expected 1 field, got %d", len(doc.Fields))
	}
	if doc.Fields[0].Name() != "key" {
		t.Errorf("expected field named 'key', got '%s'", doc.Fields[0].Name())
	}
}

func BenchmarkBatch(b *testing.B) {

	cache := registry.NewCache()
	analyzer, err := cache.AnalyzerNamed(standard_analyzer.Name)
	if err != nil {
		b.Fatal(err)
	}

	analysisQueue := index.NewAnalysisQueue(1)
	idx, err := NewUpsideDownCouch(null.Name, nil, analysisQueue)
	if err != nil {
		b.Fatal(err)
	}
	err = idx.Open()
	if err != nil {
		b.Fatal(err)
	}

	batch := index.NewBatch()
	for i := 0; i < 100; i++ {
		d := document.NewDocument(strconv.Itoa(i))
		f := document.NewTextFieldWithAnalyzer("desc", nil, bleveWikiArticle1K, analyzer)
		d.AddField(f)
		batch.Update(d)
	}

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		err = idx.Batch(batch)
		if err != nil {
			b.Fatal(err)
		}
	}
}

func TestMappingStructWithPointerToString(t *testing.T) {

	mapping := buildMapping()

	name := "marty"

	x := struct {
		Name *string
	}{
		Name: &name,
	}

	doc := document.NewDocument("1")
	err := mapping.mapDocument(doc, x)
	if err != nil {
		t.Fatal(err)
	}
	found := false
	count := 0
	for _, f := range doc.Fields {
		if f.Name() == "Name" {
			found = true
		}
		count++
	}
	if !found {
		t.Errorf("expected to find field named 'Name'")
	}
	if count != 1 {
		t.Errorf("expected to find 1 find, found %d", count)
	}
}