Golang message.Msg类代码示例

// writeMessage writes one message to the destination mongo, or sends an error down the pipe
// TODO this can be cleaned up.  I'm not sure whether this should pipe the error, or whether the
//   caller should pipe the error
func (m *Mongodb) writeMessage(msg *message.Msg) (*message.Msg, error) {
	collection := m.mongoSession.DB(m.database).C(m.collection)

	if !msg.IsMap() {
		m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (document must be a bson document, got %T instead)", msg.Data), msg.Data)
		return msg, nil
	}

	doc := msg.Map()
	if m.bulk {
		m.bulkWriteChannel <- doc
	} else if msg.Op == message.Delete {
		err := collection.Remove(doc)
		if err != nil {
			m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error removing (%s)", err.Error()), msg.Data)
		}
	} else {
		err := collection.Insert(doc)
		if mgo.IsDup(err) {
			err = collection.Update(bson.M{"_id": doc["_id"]}, doc)
		}
		if err != nil {
			m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (%s)", err.Error()), msg.Data)
		}
	}

	return msg, nil
}

/*
 * dump each message to the file
 */
func (d *File) dumpMessage(msg *message.Msg) (*message.Msg, error) {
	var line string

	if msg.IsMap() {
		ba, err := json.Marshal(msg.Map())
		if err != nil {
			d.pipe.Err <- NewError(ERROR, d.path, fmt.Sprintf("Can't unmarshal document (%s)", err.Error()), msg.Data)
			return msg, nil
		}
		line = string(ba)
	} else {
		line = fmt.Sprintf("%v", msg.Data)
	}

	if strings.HasPrefix(d.uri, "stdout://") {
		fmt.Println(line)
	} else {
		_, err := fmt.Fprintln(d.filehandle, line)
		if err != nil {
			d.pipe.Err <- NewError(ERROR, d.path, fmt.Sprintf("Error writing to file (%s)", err.Error()), msg.Data)
			return msg, nil
		}
	}

	return msg, nil
}

func (e *Elasticsearch) applyOp(msg *message.Msg) (*message.Msg, error) {
	if msg.Op == message.Command {
		err := e.runCommand(msg)
		if err != nil {
			e.pipe.Err <- NewError(ERROR, e.path, fmt.Sprintf("elasticsearch error (%s)", err), msg.Data)
		}
		return msg, nil
	}

	// TODO there might be some inconsistency here.  elasticsearch uses the _id field for an primary index,
	//  and we're just mapping it to a string here.
	id, err := msg.IDString("_id")
	if err != nil {
		id = ""
	}

	_, _type, err := msg.SplitNamespace()
	if err != nil {
		e.pipe.Err <- NewError(ERROR, e.path, fmt.Sprintf("unable to determine type from msg.Namespace (%s)", msg.Namespace), msg)
		return msg, nil
	}
	switch msg.Op {
	case message.Delete:
		e.indexer.Delete(e.index, _type, id, false)
		err = nil
	default:
		err = e.indexer.Index(e.index, _type, id, "", "", nil, msg.Data, false)
	}
	if err != nil {
		e.pipe.Err <- NewError(ERROR, e.path, fmt.Sprintf("elasticsearch error (%s)", err), msg.Data)
	}
	return msg, nil
}

func (a *Appbase) addBulkCommand(msg *message.Msg) (*message.Msg, error) {
	id, err := msg.IDString("_id")
	if err != nil {
		id = ""
	}

	switch msg.Op {
	case message.Delete:
		bulkRequest := elastic.NewBulkDeleteRequest().Index(a.appName).Type(a.typename).Id(id)
		a.AddBulkRequestSize(bulkRequest)
		a.bulkService.Add(bulkRequest)
		break
	case message.Update:
		bulkRequest := elastic.NewBulkUpdateRequest().Index(a.appName).Type(a.typename).Id(id).Doc(msg.Data)
		a.AddBulkRequestSize(bulkRequest)
		a.bulkService.Add(bulkRequest)
		break
	default:
		bulkRequest := elastic.NewBulkIndexRequest().Index(a.appName).Type(a.typename).Id(id).Doc(msg.Data)
		a.AddBulkRequestSize(bulkRequest)
		a.bulkService.Add(bulkRequest)
		break
	}

	a.commitBulk(false)

	return msg, nil
}

func (e *Elasticsearch) runCommand(msg *message.Msg) error {
	if !msg.IsMap() {
		return nil
	}

	if _, hasKey := msg.Map()["flush"]; hasKey {
		e.indexer.Flush()
	}
	return nil
}

/*
 * dump each message to the file
 */
func (d *Twitter) dumpMessage(msg *message.Msg) (*message.Msg, error) {
	var line string

	if msg.IsMap() {
		ba, err := json.Marshal(msg.Map())
		if err != nil {
			d.pipe.Err <- NewError(ERROR, d.path, fmt.Sprintf("Can't unmarshal document (%s)", err.Error()), msg.Data)
			return msg, nil
		}
		line = string(ba)
	} else {
		line = fmt.Sprintf("%v", msg.Data)
	}

	fmt.Println(line)

	return msg, nil
}

// applyOp applies one operation to the database
func (r *Rethinkdb) applyOp(msg *message.Msg) (*message.Msg, error) {
	var (
		resp gorethink.WriteResponse
		err  error
	)

	_, msgTable, err := msg.SplitNamespace()
	if err != nil {
		r.pipe.Err <- NewError(ERROR, r.path, fmt.Sprintf("rethinkdb error (msg namespace improperly formatted, must be database.table, got %s)", msg.Namespace), msg.Data)
		return msg, nil
	}
	if !msg.IsMap() {
		r.pipe.Err <- NewError(ERROR, r.path, "rethinkdb error (document must be a json document)", msg.Data)
		return msg, nil
	}
	doc := msg.Map()

	switch msg.Op {
	case message.Delete:
		id, err := msg.IDString("id")
		if err != nil {
			r.pipe.Err <- NewError(ERROR, r.path, "rethinkdb error (cannot delete an object with a nil id)", msg.Data)
			return msg, nil
		}
		resp, err = gorethink.Table(msgTable).Get(id).Delete().RunWrite(r.client)
	case message.Insert:
		resp, err = gorethink.Table(msgTable).Insert(doc).RunWrite(r.client)
	case message.Update:
		resp, err = gorethink.Table(msgTable).Insert(doc, gorethink.InsertOpts{Conflict: "replace"}).RunWrite(r.client)
	}
	if err != nil {
		r.pipe.Err <- NewError(ERROR, r.path, "rethinkdb error (%s)", err)
		return msg, nil
	}

	err = r.handleResponse(&resp)
	if err != nil {
		r.pipe.Err <- NewError(ERROR, r.path, "rethinkdb error (%s)", err)
	}

	return msg, nil
}

func (t *Transformer) toMsg(incoming interface{}, msg *message.Msg) error {

	switch newMsg := incoming.(type) {
	case map[string]interface{}: // we're a proper message.Msg, so copy the data over
		msg.Op = message.OpTypeFromString(newMsg["op"].(string))
		msg.Timestamp = newMsg["ts"].(int64)
		msg.Namespace = newMsg["ns"].(string)

		switch data := newMsg["data"].(type) {
		case otto.Value:
			exported, err := data.Export()
			if err != nil {
				t.pipe.Err <- t.transformerError(ERROR, err, msg)
				return nil
			}
			d, err := mejson.Unmarshal(exported.(map[string]interface{}))
			if err != nil {
				t.pipe.Err <- t.transformerError(ERROR, err, msg)
				return nil
			}
			msg.Data = map[string]interface{}(d)
		case map[string]interface{}:
			d, err := mejson.Unmarshal(data)
			if err != nil {
				t.pipe.Err <- t.transformerError(ERROR, err, msg)
				return nil
			}
			msg.Data = map[string]interface{}(d)
		default:
			msg.Data = data
		}
	case bool: // skip this doc if we're a bool and we're false
		if !newMsg {
			msg.Op = message.Noop
			return nil
		}
	default: // something went wrong
		return fmt.Errorf("returned doc was not a map[string]interface{}")
	}

	return nil
}

func (e *Elasticsearch) applyOp(msg *message.Msg) (*message.Msg, error) {
	if msg.Op == message.Command {
		err := e.runCommand(msg)
		if err != nil {
			e.pipe.Err <- NewError(ERROR, e.path, fmt.Sprintf("elasticsearch error (%s)", err), msg.Data)
		}
		return msg, nil
	}

	// TODO there might be some inconsistency here.  elasticsearch uses the _id field for an primary index,
	//  and we're just mapping it to a string here.
	id, err := msg.IDString("_id")
	if err != nil {
		id = ""
	}

	// Possible fix for #151 issue: code added to make it working with elastic 2 and mongo (now it's possible to use _id)
	if len(id) > 0 {
		var message = msg.Map()
		delete(message, "_id")
		msg.Data = message
	}

	_, _type, err := msg.SplitNamespace()
	if err != nil {
		e.pipe.Err <- NewError(ERROR, e.path, fmt.Sprintf("unable to determine type from msg.Namespace (%s)", msg.Namespace), msg)
		return msg, nil
	}
	switch msg.Op {
	case message.Delete:
		e.indexer.Delete(e.index, _type, id)
		err = nil
	default:
		err = e.indexer.Index(e.index, _type, id, "", "", nil, msg.Data)
	}
	if err != nil {
		e.pipe.Err <- NewError(ERROR, e.path, fmt.Sprintf("elasticsearch error (%s)", err), msg.Data)
	}
	return msg, nil
}

// writeMessage writes one message to the destination mongo, or sends an error down the pipe
// TODO this can be cleaned up.  I'm not sure whether this should pipe the error, or whether the
//   caller should pipe the error
func (m *Mongodb) writeMessage(msg *message.Msg) (*message.Msg, error) {
	_, msgColl, err := msg.SplitNamespace()
	if err != nil {
		m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (msg namespace improperly formatted, must be database.collection, got %s)", msg.Namespace), msg.Data)
		return msg, nil
	}

	collection := m.mongoSession.DB(m.database).C(msgColl)

	if !msg.IsMap() {
		m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (document must be a bson document, got %T instead)", msg.Data), msg.Data)
		return msg, nil
	}

	doc := &SyncDoc{
		Doc:        msg.Map(),
		Collection: msgColl,
	}

	if m.bulk {
		m.bulkWriteChannel <- doc
	} else if msg.Op == message.Delete {
		err := collection.Remove(doc.Doc)
		if err != nil {
			m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error removing (%s)", err.Error()), msg.Data)
		}
	} else {
		err := collection.Insert(doc.Doc)
		if mgo.IsDup(err) {
			err = collection.Update(bson.M{"_id": doc.Doc["_id"]}, doc.Doc)
		}
		if err != nil {
			m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (%s)", err.Error()), msg.Data)
		}
	}

	return msg, nil
}

func (t *Transformer) transformOne(msg *message.Msg) (*message.Msg, error) {

	var (
		doc    interface{}
		value  otto.Value
		outDoc otto.Value
		result interface{}
		err    error
	)

	// short circuit for deletes and commands
	if msg.Op == message.Delete || msg.Op == message.Command {
		return msg, nil
	}

	now := time.Now().Nanosecond()
	if msg.IsMap() {
		if doc, err = mejson.Marshal(msg.Data); err != nil {
			t.pipe.Err <- t.transformerError(ERROR, err, msg)
			return msg, nil
		}
	} else {
		doc = msg.Data
	}

	if value, err = t.vm.ToValue(doc); err != nil {
		t.pipe.Err <- t.transformerError(ERROR, err, msg)
		return msg, nil
	}

	// now that we have finished casting our map to a bunch of different types,
	// lets run our transformer on the document
	beforeVM := time.Now().Nanosecond()
	if outDoc, err = t.vm.Call(`module.exports`, nil, value); err != nil {
		t.pipe.Err <- t.transformerError(ERROR, err, msg)
		return msg, nil
	}

	if result, err = outDoc.Export(); err != nil {
		t.pipe.Err <- t.transformerError(ERROR, err, msg)
		return msg, nil
	}

	afterVM := time.Now().Nanosecond()

	switch r := result.(type) {
	case map[string]interface{}:
		doc, err := mejson.Unmarshal(r)
		if err != nil {
			t.pipe.Err <- t.transformerError(ERROR, err, msg)
			return msg, nil
		}
		msg.Data = map[string]interface{}(doc)
	default:
		msg.Data = r
	}

	if t.debug {
		then := time.Now().Nanosecond()
		fmt.Printf("document transformed in %dus.  %d to marshal, %d in the vm, %d to unmarshal\n", (then-now)/1000, (beforeVM-now)/1000, (afterVM-beforeVM)/1000, (then-afterVM)/1000)
	}

	return msg, nil
}

func (t *Transformer) transformOne(msg *message.Msg) (*message.Msg, error) {

	var (
		doc    interface{}
		value  otto.Value
		outDoc otto.Value
		result interface{}
		err    error
	)

	// short circuit for deletes and commands
	if msg.Op == message.Command {
		return msg, nil
	}

	now := time.Now().Nanosecond()
	currMsg := map[string]interface{}{
		"data": msg.Data,
		"ts":   msg.Timestamp,
		"op":   msg.Op.String(),
		"ns":   msg.Namespace,
	}
	if msg.IsMap() {
		if doc, err = mejson.Marshal(msg.Data); err != nil {
			t.pipe.Err <- t.transformerError(ERROR, err, msg)
			return msg, nil
		}
		currMsg["data"] = doc
	}

	if value, err = t.vm.ToValue(currMsg); err != nil {
		t.pipe.Err <- t.transformerError(ERROR, err, msg)
		return msg, nil
	}

	// now that we have finished casting our map to a bunch of different types,
	// lets run our transformer on the document
	beforeVM := time.Now().Nanosecond()
	if outDoc, err = t.vm.Call(`module.exports`, nil, value); err != nil {
		t.pipe.Err <- t.transformerError(ERROR, err, msg)
		return msg, nil
	}

	if result, err = outDoc.Export(); err != nil {
		t.pipe.Err <- t.transformerError(ERROR, err, msg)
		return msg, nil
	}

	afterVM := time.Now().Nanosecond()

	if err = t.toMsg(result, msg); err != nil {
		t.pipe.Err <- t.transformerError(ERROR, err, msg)
		return msg, err
	}

	if t.debug {
		then := time.Now().Nanosecond()
		fmt.Printf("document transformed in %dus.  %d to marshal, %d in the vm, %d to unmarshal\n", (then-now)/1000, (beforeVM-now)/1000, (afterVM-beforeVM)/1000, (then-afterVM)/1000)
	}

	return msg, nil
}

// applyOp applies one operation to the database
func (cloudant *Cloudant) applyOp(msg *message.Msg) (*message.Msg, error) {
	var err error

	doc := msg.Map()
	database := cloudant.client.DB(cloudant.database)

	switch msg.Op {
	case message.Delete:
		id, err := msg.IDString("_id")
		if err != nil {
			cloudant.pipe.Err <- NewError(ERROR, cloudant.path, "cloudant error (cannot delete an object with a nil id)", msg.Data)
			return msg, err
		}
		rev, err := msg.IDString("_rev")
		if err != nil {
			cloudant.pipe.Err <- NewError(ERROR, cloudant.path, "cloudant error (cannot delete an object with a nil rev)", msg.Data)
			return msg, err
		}
		tombstone, err := database.Delete(id, rev)
		if err == nil && cloudant.debug {
			fmt.Printf("Deleted document {_id=%s, _rev=%s} => tombstode rev: %s\n", id, rev, tombstone)
		}

	case message.Insert: // If the doc contains _id this will be respected
		_, err := msg.IDString("_rev") // If the doc contains a _rev, it can't be an insert, as this would mean a conflict.
		if err == nil {
			cloudant.pipe.Err <- NewError(ERROR, cloudant.path, "cloudant error (cannot insert doc that already has a rev)", msg.Data)
			return msg, err
		}
		newid, newrev, err := database.Post(doc)
		if err == nil && cloudant.debug {
			fmt.Printf("Inserted new document as {_id=%s, _rev=%s}\n", newid, newrev)
		}

	case message.Update:
		id, err := msg.IDString("_id")
		if err != nil {
			cloudant.pipe.Err <- NewError(ERROR, cloudant.path, "cloudant error (cannot update an object with a nil id)", msg.Data)
			return msg, err
		}
		rev, _ := msg.IDString("_rev")
		newrev, err := database.Put(id, doc, rev)
		if err == nil && cloudant.debug {
			fmt.Printf("Updated document as {_id=%s, _rev=%s}\n", id, newrev)
		}
	}

	if err != nil {
		errmsg := fmt.Sprintf("cloudant error (database '%s') %+v %+v", cloudant.database, err, msg)
		if cloudant.debug {
			fmt.Println(errmsg)
		}
		cloudant.pipe.Err <- NewError(ERROR, cloudant.path, errmsg, err)
	}

	return msg, err
}

func (t *Transformer) transformOne(msg *message.Msg) (*message.Msg, error) {

	var (
		doc    interface{}
		value  otto.Value
		outDoc otto.Value
		result interface{}
		err    error
	)

	// short circuit for deletes and commands
	if msg.Op == message.Command {
		return msg, nil
	}

	now := time.Now().Nanosecond()
	fullDoc := map[string]interface{}{
		"data": msg.Data,
		"ts":   msg.Timestamp,
		"op":   msg.Op.String(),
	}
	if msg.IsMap() {
		if doc, err = mejson.Marshal(msg.Data); err != nil {
			t.pipe.Err <- t.transformerError(ERROR, err, msg)
			return msg, nil
		}
		fullDoc["data"] = doc
	}

	if value, err = t.vm.ToValue(fullDoc); err != nil {
		t.pipe.Err <- t.transformerError(ERROR, err, msg)
		return msg, nil
	}

	// now that we have finished casting our map to a bunch of different types,
	// lets run our transformer on the document
	beforeVM := time.Now().Nanosecond()
	if outDoc, err = t.vm.Call(`module.exports`, nil, value); err != nil {
		t.pipe.Err <- t.transformerError(ERROR, err, msg)
		return msg, nil
	}

	if result, err = outDoc.Export(); err != nil {
		t.pipe.Err <- t.transformerError(ERROR, err, msg)
		return msg, nil
	}

	afterVM := time.Now().Nanosecond()

	fullDoc, ok := result.(map[string]interface{})
	if !ok {
		t.pipe.Err <- t.transformerError(ERROR, fmt.Errorf("returned doc was not a map[string]interface{}"), msg)
		return msg, fmt.Errorf("returned doc was not a map[string]interface{}")
	}

	msg.Op = message.OpTypeFromString(fullDoc["op"].(string))
	msg.Timestamp = fullDoc["ts"].(int64)
	switch data := fullDoc["data"].(type) {
	case otto.Value:
		exported, err := data.Export()
		if err != nil {
			t.pipe.Err <- t.transformerError(ERROR, err, msg)
			return msg, nil
		}
		d, err := mejson.Unmarshal(exported.(map[string]interface{}))
		if err != nil {
			t.pipe.Err <- t.transformerError(ERROR, err, msg)
			return msg, nil
		}
		msg.Data = map[string]interface{}(d)
	case map[string]interface{}:
		d, err := mejson.Unmarshal(data)
		if err != nil {
			t.pipe.Err <- t.transformerError(ERROR, err, msg)
			return msg, nil
		}
		msg.Data = map[string]interface{}(d)
	default:
		msg.Data = data
	}

	if t.debug {
		then := time.Now().Nanosecond()
		fmt.Printf("document transformed in %dus.  %d to marshal, %d in the vm, %d to unmarshal\n", (then-now)/1000, (beforeVM-now)/1000, (afterVM-beforeVM)/1000, (then-afterVM)/1000)
	}

	return msg, nil
}