Golang swarm.Swarm类代码示例

func clearDb(dbName string, swarm *swarm.Swarm) error {
	cmd := []string{}
	cmd = append(cmd, "mysql")
	cmd = append(cmd, "-u"+"cup_dba")
	cmd = append(cmd, "-p"+"111111")
	cmd = append(cmd, "-S"+"/DBAASDAT/upsql.sock")
	cmd = append(cmd, "-e drop database qjrtest1")
	res, err := swarm.Exec(cmd, dbName)
	if err != nil {
		log.WithFields(log.Fields{
			"DbName": dbName,
			"err":    err.Error(),
		}).Error("drop qjrtest1 fail")
		return err
	}
	res.Close()

	cmd = []string{}
	cmd = append(cmd, "mysql")
	cmd = append(cmd, "-u"+"cup_dba")
	cmd = append(cmd, "-p"+"111111")
	cmd = append(cmd, "-S"+"/DBAASDAT/upsql.sock")
	cmd = append(cmd, "-e drop database qjrtest2")
	res1, err := swarm.Exec(cmd, dbName)
	if err != nil {
		log.WithFields(log.Fields{
			"DbName": dbName,
			"err":    err.Error(),
		}).Error("drop qjrtest2 fail")
		return err
	}
	res1.Close()

	return nil
}

func clearMaster(dbName string, swm *swarm.Swarm) error {
	err := swm.MasterReset(dbName)
	if err != nil {
		return err
	}

	return nil
}

func voteSl(swm *swarm.Swarm, topology *structs.Topology) (string, error) {
	maxMasterLogFileNum := 0
	maxReadMasterLogPos := 0
	ch := make(chan []interface{})
	maxDbName := ""
	for k, v := range topology.DataNodeGroup["default"] {
		if v.Type == consts.Slave && v.Status == consts.Normal {
			dbName := k
			go func() {
				masterLogFileNum, readMasterLogPos := swm.PositionGet(dbName)
				log.WithFields(log.Fields{
					"SlaveName":        dbName,
					"MasterLogFileNum": masterLogFileNum,
					"ReadMasterLogPos": readMasterLogPos,
				}).Debug("Sl vote")
				pos := []interface{}{dbName, masterLogFileNum, readMasterLogPos}
				ch <- pos
			}()
		}
	}

	for i := 0; i < topology.DataNodeGroupNormalCount["default"]-2; i++ {
		pos := <-ch
		dbName := pos[0].(string)
		masterLogFileNum := pos[1].(int)
		readMasterLogPos := pos[2].(int)
		if masterLogFileNum > maxMasterLogFileNum {
			if readMasterLogPos > maxReadMasterLogPos {
				maxMasterLogFileNum = masterLogFileNum
				maxReadMasterLogPos = readMasterLogPos
				maxDbName = dbName
			}
		}
	}

	// all sl unavaliable
	if maxDbName == "" {
		return "", errors.New(
			"when isolate M|Sb, all slave are unavilable for vote, isolate failed and return to hope health check enter sl auto isolate")
	}

	maxDbInfo := topology.DataNodeGroup["default"][maxDbName]
	log.WithFields(log.Fields{
		"winer": maxDbName + "(" + maxDbInfo.Ip + ":" + strconv.Itoa(maxDbInfo.Port) + ")",
	}).Debug("Sl vote")

	return maxDbName, nil
}

func createDb(dbName, dbString string, swarm *swarm.Swarm) error {
	cmd := []string{}
	cmd = append(cmd, "mysql")
	cmd = append(cmd, "-u"+"cup_dba")
	cmd = append(cmd, "-p"+"111111")
	cmd = append(cmd, "-S"+"/DBAASDAT/upsql.sock")
	cmd = append(cmd, "-e create database "+dbString)
	res, err := swarm.Exec(cmd, dbName)
	if err != nil {
		log.WithFields(log.Fields{
			"DbName":   dbName,
			"err":      err.Error(),
			"Database": dbString,
		}).Error("create test database fail")
		return err
	}
	res.Close()
	return nil
}

func clearSlave(dbName string, swm *swarm.Swarm) error {
	slaveStatus, err := swm.SlaveShowStatus(dbName)
	if err != nil {
		return err
	}
	if slaveStatus.MasterIp == "" {
		return nil
	}
	err = swm.SlaveStop(dbName)
	if err != nil {
		return err
	}
	err = swm.SlaveReset(dbName)
	if err != nil {
		return err
	}

	return nil
}

func changeMaster(swm *swarm.Swarm, fromName, toIp string, toPort int) error {
	slaveStatus, err := swm.SlaveShowStatus(fromName)
	if err != nil {
		return err
	}
	log.WithFields(log.Fields{
		"DbName":      fromName,
		"SlaveStatus": slaveStatus,
	}).Debug("change master")

	if slaveStatus.MasterIp == toIp && slaveStatus.MasterPort == toPort {
		if slaveStatus.SlaveIoRunning == "No" && slaveStatus.SlaveSqlRunning == "No" {
			// 1.4 start slave
			err = swm.SlaveStart(fromName)
			if err != nil {
				return err
			}
		}
		return nil
	}
	if slaveStatus.SlaveIoRunning == "Yes" || slaveStatus.SlaveSqlRunning == "Yes" {
		// 1.1 stop slave
		err = swm.SlaveStop(fromName)
		if err != nil {
			return err
		}
	}
	if slaveStatus.MasterIp != "" {
		// 1.2 reset slave
		err = swm.SlaveReset(fromName)
		if err != nil {
			return err
		}
	}

	// 1.3 get gtid_executed
	gtidExecuted, err := swm.GetGtidExecuted(fromName)
	if err != nil {
		return err
	}

	if gtidExecuted != "" {
		// 1.4 reset master
		err = swm.MasterReset(fromName)
		if err != nil {
			return err
		}
		// 1.5 set gtid_purged
		err = swm.SetGtidPurged(fromName, gtidExecuted)
		if err != nil {
			return err
		}
	}

	// 1.6 change master
	err = swm.ChangeMaster(fromName, toIp, toPort)
	if err != nil {
		return err
	}
	// 1.7 start slave
	err = swm.SlaveStart(fromName)
	if err != nil {
		return err
	}
	return nil
}

// no need RLock
func RecoverDb(dbName string, rwMutex *sync.RWMutex, topology *structs.Topology, swm *swarm.Swarm) (error, []string) {
	var err error
	var steps []string
	defer func() {
		if err != nil {
			log.WithFields(log.Fields{
				"DbName": dbName,
				"err":    err.Error(),
			}).Error("recover fail")
		} else {
			log.WithFields(log.Fields{
				"DbName": dbName,
			}).Info("recover success")
		}
	}()

	var recoverDbInfo *structs.DatabaseInfo
	for k, v := range topology.DataNodeGroup["default"] {
		if k == dbName {
			recoverDbInfo = v
			break
		}
	}

	if recoverDbInfo == nil {
		return errors.New("recover db node not found"), steps
	}

	// 1. check m slave status
	if recoverDbInfo.Type == consts.StandBy || recoverDbInfo.Type == consts.Slave {
		mName, _ := getMaster(topology)
		var masterSlStatus structs.SlaveStatus
		masterSlStatus, err = swm.SlaveShowStatus(mName)
		if err != nil {
			steps = append(steps, fmt.Sprintf("Step1. Clear M status fail: %s\n", err.Error()))
			return err, steps
		}
		if masterSlStatus.MasterIp != "" {
			// 1.1 stop m slave
			err = swm.SlaveStop(mName)
			if err != nil {
				steps = append(steps, fmt.Sprintf("Step1. Clear M status fail: %s\n", err.Error()))
				return err, steps
			}
			// 1.2 reset m slave
			err = swm.SlaveReset(mName)
			if err != nil {
				steps = append(steps, fmt.Sprintf("Step1. Clear M status fail: %s\n", err.Error()))
				return err, steps
			}
		}
	}

	steps = append(steps, "Step1. Clear M status success\n")

	// 2. recover change master
	if recoverDbInfo.Type == consts.StandBy {
		mName, mDbInfo := getMaster(topology)
		err = changeMaster(swm, dbName, mDbInfo.Ip, mDbInfo.Port)
		if err != nil {
			log.WithFields(log.Fields{
				"Sb":   dbName + "(" + recoverDbInfo.Ip + ":" + strconv.Itoa(recoverDbInfo.Port) + ")",
				"M":    mName + "(" + mDbInfo.Ip + ":" + strconv.Itoa(mDbInfo.Port) + ")",
				"err:": err.Error(),
			}).Error("change master fail, return")
			steps = append(steps, fmt.Sprintf("Step2. Sb change master to M fail: %s\n", err.Error()))
			return err, steps
		}
		log.WithFields(log.Fields{
			"Sb": dbName + "(" + recoverDbInfo.Ip + ":" + strconv.Itoa(recoverDbInfo.Port) + ")",
			"M":  mName + "(" + mDbInfo.Ip + ":" + strconv.Itoa(mDbInfo.Port) + ")",
		}).Debug("change master success")
		steps = append(steps, "Step2. Sb change master to M success\n")
	} else if recoverDbInfo.Type == consts.Slave {
		sbName, sbDbInfo := getStandBy(topology)
		err = changeMaster(swm, dbName, sbDbInfo.Ip, sbDbInfo.Port)
		if err != nil {
			log.WithFields(log.Fields{
				"Sl":   dbName + "(" + recoverDbInfo.Ip + ":" + strconv.Itoa(recoverDbInfo.Port) + ")",
				"Sb":   sbName + "(" + sbDbInfo.Ip + ":" + strconv.Itoa(sbDbInfo.Port) + ")",
				"err:": err.Error(),
			}).Error("change master fail, return")
			steps = append(steps, fmt.Sprintf("Step2. Sl change master to Sb fail: %s\n", err.Error()))
			return err, steps
		}
		log.WithFields(log.Fields{
			"Sl": dbName + "(" + recoverDbInfo.Ip + ":" + strconv.Itoa(recoverDbInfo.Port) + ")",
			"Sb": sbName + "(" + sbDbInfo.Ip + ":" + strconv.Itoa(sbDbInfo.Port) + ")",
		}).Info("change master success")
		steps = append(steps, "Step2. Sl change master to Sb success\n")
	}

	// 3. update topology
	rwMutex.Lock()
	dbInfo := getDbInfo(dbName, topology)
	dbInfo.Status = consts.Normal
	updateVersionAndNormalCount(topology, 1)
	rwMutex.Unlock()

//.........这里部分代码省略.........

// no need RLock
func IsolateDB(dbName string, rwMutex *sync.RWMutex, topology *structs.Topology, swm *swarm.Swarm) (error, []string) {
	var err error
	steps := []string{}
	defer func() {
		if err != nil && err.Error() != "" {
			log.WithFields(log.Fields{
				"DbName": dbName,
				"err":    err.Error(),
			}).Error("isolate fail")
		} else {
			log.WithFields(log.Fields{
				"DbName": dbName,
			}).Info("isolate success")
		}
	}()
	var isolateType string
	for k, v := range topology.DataNodeGroup["default"] {
		if k == dbName {
			isolateType = v.Type
			break
		}
	}

	if isolateType == "" {
		err = errors.New("isolate db node not found")
		return err, steps
	}

	btopology, _ := json.MarshalIndent(topology, "", "  ")
	log.WithFields(log.Fields{
		"IsolateDbType": isolateType,
		"DbName":        dbName,
		"topology":      string(btopology),
	}).Debug("isolate db")

	// only m normal ||
	// isolate sb && only m sb normal ||
	// isolate sl
	// no need swarm, update topology and return
	if topology.DataNodeGroupNormalCount["default"] == 1 ||
		(isolateType == consts.StandBy && topology.DataNodeGroupNormalCount["default"] == 2) ||
		isolateType == consts.Slave {
		rwMutex.Lock()
		topology.DataNodeGroup["default"][dbName].Status = consts.Abnormal
		updateVersionAndNormalCount(topology, -1)
		rwMutex.Unlock()
		steps = append(steps, fmt.Sprintf("Step1. Isolate %s success\n", dbName))
		return nil, steps
	}

	if isolateType == consts.Master {
		// 1.sb checkGtidDiff
		sbName, sbDbInfo := getStandBy(topology)

		err = swm.IsDelay(sbName)
		if err != nil {
			log.WithFields(log.Fields{
				"DbName": sbName,
				"err":    err.Error(),
			}).Warn("Sb delay check fail, continue")
			err = nil
		}
		// 1. end ********************

		// m sb normal
		// update topology and return
		if topology.DataNodeGroupNormalCount["default"] == 2 {
			// 3.update topology
			_, mDbInfo := getMaster(topology)
			rwMutex.Lock()
			sbDbInfo.Type = consts.Master // sb -> m
			mDbInfo.Type = consts.StandBy // m -> sb
			mDbInfo.Status = consts.Abnormal
			updateVersionAndNormalCount(topology, -1)
			steps = append(steps, fmt.Sprintf("Step1. Isolate %s success\n", dbName))
			rwMutex.Unlock()
			return nil, steps
			// ********************
		}

		// has normal sl
		// 2.vote sb
		var maxGtidDbName string
		maxGtidDbName, err = voteSl(swm, topology)
		if err != nil {
			steps = append(steps, fmt.Sprintf("Step2. Vote new Sb fail: %s\n", err.Error()))
			return err, steps
		}
		steps = append(steps, fmt.Sprintf("Step2. Vote new Sb success:%s\n", maxGtidDbName))
		// 2. end ********************

		// 3.update topology
		rwMutex.Lock()
		mName, mDbInfo := getMaster(topology)
		mDbInfo.Type = consts.Slave // m -> sl
		mDbInfo.Status = consts.Abnormal
		sbDbInfo.Type = consts.Master // sb -> master
		slDbInfo := topology.DataNodeGroup["default"][maxGtidDbName]
		slDbInfo.Type = consts.StandBy // maxGtidSl -> sb
//.........这里部分代码省略.........

func checkTopo(swarm *swarm.Swarm, topology *structs.Topology, mCheck bool, t *testing.T) error {
	log.Println("***CheckTopo start***")
	var mName, mIp, sbName, sbIp string
	var mPort, sbPort int
	for k, v := range topology.DataNodeGroup["default"] {
		if v.Type == consts.Master && v.Status == consts.Normal {
			mName = k
			mIp = v.Ip
			mPort = v.Port
		} else if v.Type == consts.StandBy && v.Status == consts.Normal {
			sbName = k
			sbIp = v.Ip
			sbPort = v.Port
		}
	}

	if mCheck {
		// check m
		mSlaveStatus, err := swarm.SlaveShowStatus(mName)
		if err != nil {
			return fmt.Errorf(err.Error())
		}
		if mSlaveStatus.MasterIp != "" ||
			mSlaveStatus.MasterPort != 0 {
			return fmt.Errorf("master %s init/isolate/recover err, slave status:%#v\n", mName, mSlaveStatus)
		}
	}

	if sbName == "" {
		log.Println("***CheckTopo end***")
		return nil
	}

	// check sb
	sbSlaveStatus, err := swarm.SlaveShowStatus(sbName)
	if err != nil {
		return fmt.Errorf(err.Error())
	}
	if sbSlaveStatus.MasterIp != mIp ||
		sbSlaveStatus.MasterPort != mPort ||
		sbSlaveStatus.SlaveIoRunning != "Yes" ||
		sbSlaveStatus.SlaveSqlRunning != "Yes" {
		return fmt.Errorf("standby %s init/isolate/recover err, slave status:%#v\n", sbName, sbSlaveStatus)
	}

	// check sl
	if topology.DataNodeGroupNormalCount["default"] > 2 {
		for k, v := range topology.DataNodeGroup["default"] {
			if v.Type == consts.Slave && v.Status == consts.Normal {
				slSlaveStatus, err := swarm.SlaveShowStatus(k)
				if err != nil {
					return fmt.Errorf(err.Error())
				}
				if slSlaveStatus.MasterIp != sbIp ||
					slSlaveStatus.MasterPort != sbPort ||
					slSlaveStatus.SlaveIoRunning != "Yes" ||
					slSlaveStatus.SlaveSqlRunning != "Yes" {
					return fmt.Errorf("slave %s init/isolate/recover err, slave status:%#v\n", k, sbSlaveStatus)
				}
			}
		}
	}
	log.Println("***CheckTopo end***")
	return nil
}