Golang toolkit.NewLog函数代码示例

func (a *Server) RegisterRPCFunctions(o interface{}) error {
	t := reflect.TypeOf(o)
	v := reflect.ValueOf(o)
	if v.Kind() != reflect.Ptr {
		return errors.New("Invalid object for RPC Register")
	}
	if a.Log == nil {
		a.Log, _ = toolkit.NewLog(true, false, "", "", "")
	}
	//objName := toolkit.TypeName(o)
	methodCount := t.NumMethod()
	for i := 0; i < methodCount; i++ {
		method := t.Method(i)
		mtype := method.Type
		methodName := strings.ToLower(method.Name)
		//fmt.Println("Evaluating " + toolkit.TypeName(o) + "." + methodName)

		//-- now check method signature
		if mtype.NumIn() == 2 && mtype.In(1).String() == "toolkit.M" {
			if mtype.NumOut() == 1 && mtype.Out(0).String() == "*toolkit.Result" {
				//a.Log.Info("Registering RPC Function " + objName + "." + methodName)
				a.AddFn(methodName, v.Method(i).Interface().(func(toolkit.M) *toolkit.Result), true, "session")
			}
		}
	}
	return nil
}

func (d *DataGrabberController) StopTransformation(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson

	dataGrabber := new(colonycore.DataGrabber)
	err := r.GetPayload(dataGrabber)
	if err != nil {
		return helper.CreateResult(false, nil, err.Error())
	}

	logFileName := dataGrabber.ID
	logFilePattern := ""
	logConf, err := toolkit.NewLog(false, true, dgLogPath, logFileName, logFilePattern)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")
	}

	if _, ok := serviceHolder[dataGrabber.ID]; ok {
		serviceHolder[dataGrabber.ID] = false
		delete(serviceHolder, dataGrabber.ID)

		message := fmt.Sprintf("===> Transformation stopped! %s -> %s", dataGrabber.DataSourceOrigin, dataGrabber.DataSourceDestination)
		logConf.AddLog(message, "SUCCESS")
		fmt.Println(message)
	}

	return helper.CreateResult(true, nil, "")
}

func main() {
	var e error
	l, e := toolkit.NewLog(true, true, "/Users/ariefdarmawan/Temp", "log-test-%s.log", "20060102")
	if e != nil {
		fmt.Println("Error: ", e.Error())
	}

	l.Info("Test Log Info")
}

func (p *Page) SendFiles(EC_DATA_PATH string, serverid string) error {
	path := filepath.Join(EC_DATA_PATH, "widget", "log")
	log, _ := toolkit.NewLog(false, true, path, "sendfile-%s", "20060102-1504")

	for _, wValue := range p.Widget {
		appID := wValue.ID
		log.AddLog("Get widget with ID: "+appID, "INFO")
		widget := new(Widget)
		err := Get(widget, appID)
		if err != nil {
			log.AddLog(err.Error(), "ERROR")
			return err
		}

		log.AddLog("Get server with ID: "+serverid, "INFO")
		server := new(Server)
		err = Get(server, serverid)
		if err != nil {
			log.AddLog(err.Error(), "ERROR")
			return err
		}

		serverPathSeparator := p.GetServerPathSeparator(server)
		sourcePath := filepath.Join(EC_DATA_PATH, "widget", appID)
		destPath := strings.Join([]string{server.AppPath, "src", "widget"}, serverPathSeparator)

		if server.OS == "windows" {
			if strings.Contains(server.CmdExtract, "7z") || strings.Contains(server.CmdExtract, "zip") {
				err = p.CopyFileToServer(server, sourcePath, destPath, appID, log)
				if err != nil {
					log.AddLog(err.Error(), "ERROR")
					return err
				}
			} else {
				message := "currently only zip/7z command which is supported"
				log.AddLog(message, "ERROR")
				return err
			}
		} else {
			if strings.Contains(server.CmdExtract, "tar") || strings.Contains(server.CmdExtract, "zip") {
				err = p.CopyFileToServer(server, sourcePath, destPath, appID, log)
				if err != nil {
					log.AddLog(err.Error(), "ERROR")
					return err
				}
			} else {
				message := "currently only zip/tar command which is supported"
				log.AddLog(message, "ERROR")
				return err
			}
		}
	}
	return nil
}

/*
Generate log as condition inputed, if warning and error found will send email too
*/
func (s *Service) AddLog(logtext, logtype string) {
	if s.Log == nil {
		s.Log, _ = toolkit.NewLog(true, false, "", "", "")
	}
	s.Log.AddLog(logtext, logtype)

	subj := fmt.Sprintf("[%s] From Service %s at %s", logtype, s.Name, time.Now().Format("20060102_15:04:05"))
	msg := fmt.Sprintf("Message From Live Service : %s", logtext)

	if logtype == "ERROR" {
		s.Mail.sendEmail(subj, msg, s.EmailError)
	} else if logtype == "WARNING" {
		s.Mail.sendEmail(subj, msg, s.EmailWarning)
	}
}

func (d *DataGrabberController) getLogger(dataGrabber *colonycore.DataGrabber) (*toolkit.LogEngine, error) {
	logAt := time.Now().Format("20060102-150405")
	logFileName := strings.Split(logAt, "-")[0]
	logFileNameParsed := fmt.Sprintf("%s-%s", dataGrabber.ID, logFileName)
	logFilePattern := ""

	logConf, err := toolkit.NewLog(false, true, dgLogPath, logFileNameParsed, logFilePattern)
	if err != nil {
		return nil, err
	}

	currentDataGrabber := new(colonycore.DataGrabber)
	err = colonycore.Get(currentDataGrabber, dataGrabber.ID)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")
		return logConf, err
	}
	if currentDataGrabber.RunAt == nil {
		currentDataGrabber.RunAt = []string{}
	}
	if !toolkit.HasMember(currentDataGrabber.RunAt, logAt) {
		currentDataGrabber.RunAt = append(currentDataGrabber.RunAt, logAt)
	}

	err = colonycore.Delete(currentDataGrabber)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")
		return logConf, err
	}

	err = colonycore.Save(currentDataGrabber)
	if err != nil {
		logConf.AddLog(err.Error(), "ERROR")
		return logConf, err
	}

	return logConf, nil
}

func main() {
	var err error

	flagConfigPath := flag.String("config", "", "config file")
	flagSnapShot := flag.String("snapshot", "", "snapshot filepath")
	flagID := flag.String("id", "", "_id of the config (if array)")
	flagPID := flag.Int("pid", 0, "process id number for identify snapshot active")

	flag.Parse()
	tconfigPath := toolkit.ToString(*flagConfigPath)
	tsnapshot := toolkit.ToString(*flagSnapShot)
	_id = *flagID

	_id = strings.Replace(_id, `"`, "", -1)
	if _id == "" {
		sedotan.CheckError(errors.New("-id cannot be empty"))
	}

	configpath = strings.Replace(tconfigPath, `"`, "", -1)
	if configpath == "" {
		sedotan.CheckError(errors.New("-config cannot be empty"))
	}

	snapshot = strings.Replace(tsnapshot, `"`, "", -1)
	if snapshot == "" {
		sedotan.CheckError(errors.New("-snapshot cannot be empty"))
	}

	err = getsnapshot()
	if err != nil {
		sedotan.CheckError(errors.New(fmt.Sprintf("get snapshot error found : %v", err.Error())))
	}

	pid = *flagPID
	if pid == 0 {
		sedotan.CheckError(errors.New("-pid cannot be empty or zero value"))
	}

	err = getConfig()
	checkfatalerror(err)

	logconf, _ := toolkit.ToM(config.Get("logconf", toolkit.M{}))
	if !logconf.Has("logpath") || !logconf.Has("filename") || !logconf.Has("filepattern") {
		checkfatalerror(errors.New(fmt.Sprintf("config log is not complete")))
	}

	logpath := logconf["logpath"].(string)
	if EC_DATA_PATH != "" {
		logpath = filepath.Join(EC_DATA_PATH, "webgrabber", "log")
	}

	Log, err = toolkit.NewLog(false, true, logpath, (logconf["filename"].(string) + "-%s"), logconf["filepattern"].(string))
	checkfatalerror(err)
	Log.AddLog("Starting web grab data", "INFO")

	Log.AddLog("Start fetch the config", "INFO")
	err = fetchConfig()
	checkexiterror(err)
	Log.AddLog(fmt.Sprintf("Data grabber created : %v", toolkit.JsonString(wGrabber)), "INFO")
	Log.AddLog("Fetch the config success", "INFO")

	Log.AddLog("Get the data", "INFO")
	err = wGrabber.Grab(nil)
	if err != nil {
		checkexiterror(errors.New(fmt.Sprintf("Grab Failed : %v", err.Error())))
	}
	Log.AddLog("Get data grab success", "INFO")

	Log.AddLog("Start save data grab", "INFO")
	err = savedatagrab()
	if err != nil {
		checkexiterror(errors.New(fmt.Sprintf("Save data finish with error : %v", err.Error())))
	}
	Log.AddLog("Save data grab success", "INFO")

	err = getsnapshot()
	if err != nil {
		note := fmt.Sprintf("Unable to get last snapshot :%s", err.Error())
		Log.AddLog(note, "ERROR")
	}

	snapshotdata.Note = ""
	snapshotdata.Lastgrabstatus = "success"
	snapshotdata.Grabstatus = "done"

	if pid == snapshotdata.Pid {
		err = savesnapshot()
		if err != nil {
			checkexiterror(errors.New(fmt.Sprintf("Save snapshot error : %v", err.Error())))
		}
	}

	Log.AddLog("Finish grab data", "INFO")
}

func GrabHtmlConfig(data toolkit.M) (*sdt.GrabService, error) {
	var e error
	var gi, ti time.Duration
	xGrabService := sdt.NewGrabService()
	xGrabService.Name = data["nameid"].(string) //"irondcecom"
	xGrabService.Url = data["url"].(string)     //"http://www.dce.com.cn/PublicWeb/MainServlet"

	xGrabService.SourceType = sdt.SourceType_HttpHtml

	grabintervalToInt := toolkit.ToInt(data["grabinterval"], toolkit.RoundingAuto)
	timeintervalToInt := toolkit.ToInt(data["timeoutinterval"], toolkit.RoundingAuto)
	if data["intervaltype"].(string) == "seconds" {
		gi = time.Duration(grabintervalToInt) * time.Second
		ti = time.Duration(timeintervalToInt) * time.Second
	} else if data["intervaltype"].(string) == "minutes" {
		gi = time.Duration(grabintervalToInt) * time.Minute
		ti = time.Duration(timeintervalToInt) * time.Minute
	} else if data["intervaltype"].(string) == "hours" {
		gi = time.Duration(grabintervalToInt) * time.Hour
		ti = time.Duration(timeintervalToInt) * time.Hour
	}

	xGrabService.GrabInterval = gi    //* time.Minute
	xGrabService.TimeOutInterval = ti //* time.Minute //time.Hour, time.Minute, time.Second

	xGrabService.TimeOutIntervalInfo = fmt.Sprintf("%v %s", timeintervalToInt, data["intervaltype"] /*"seconds"*/)

	grabConfig := sdt.Config{}

	if data["calltype"].(string) == "POST" {
		dataurl := toolkit.M{}
		for _, grabconf := range data["grabconf"].(map[string]interface{}) {
			grabDataConf, e := toolkit.ToM(grabconf)
			if e != nil {
				return nil, e
			}
			for key, subGrabDataConf := range grabDataConf {
				if reflect.ValueOf(subGrabDataConf).Kind() == reflect.Float64 {
					i := toolkit.ToInt(subGrabDataConf, toolkit.RoundingAuto)
					toString := strconv.Itoa(i)
					dataurl[key] = toString
				} else {
					dataurl[key] = subGrabDataConf
				}
			}
		}

		grabConfig.SetFormValues(dataurl)
	}

	grabDataConf, e := toolkit.ToM(data["grabconf"])
	if e != nil {
		return nil, e
	}

	isAuthType := grabDataConf.Has("authtype")
	if isAuthType {
		grabConfig.AuthType = grabDataConf["authtype"].(string)
		grabConfig.LoginUrl = grabDataConf["loginurl"].(string)   //"http://localhost:8000/login"
		grabConfig.LogoutUrl = grabDataConf["logouturl"].(string) //"http://localhost:8000/logout"

		grabConfig.LoginValues = toolkit.M{}.
			Set("name", grabDataConf["loginvalues"].(map[string]interface{})["name"].(string)).
			Set("password", grabDataConf["loginvalues"].(map[string]interface{})["password"].(string))

	}

	xGrabService.ServGrabber = sdt.NewGrabber(xGrabService.Url, data["calltype"].(string), &grabConfig)

	logconfToMap, e := toolkit.ToM(data["logconf"])
	if e != nil {
		return nil, e
	}
	logpath := logconfToMap["logpath"].(string)           //"E:\\data\\vale\\log"
	filename := logconfToMap["filename"].(string) + "-%s" //"LOG-GRABDCETEST"
	filepattern := logconfToMap["filepattern"].(string)   //"20060102"

	logconf, e := toolkit.NewLog(false, true, logpath, filename, filepattern)
	if e != nil {
		return nil, e
	}

	xGrabService.Log = logconf

	xGrabService.ServGrabber.DataSettings = make(map[string]*sdt.DataSetting)
	xGrabService.DestDbox = make(map[string]*sdt.DestInfo)

	tempDataSetting := sdt.DataSetting{}
	tempDestInfo := sdt.DestInfo{}
	// isCondition := []interface{}{}
	tempFilterCond := toolkit.M{}
	// var condition string

	for _, dataSet := range data["datasettings"].([]interface{}) {
		dataToMap, _ := toolkit.ToM(dataSet)
		tempDataSetting.RowSelector = dataToMap["rowselector"].(string)

		for _, columnSet := range dataToMap["columnsettings"].([]interface{}) {
			columnToMap, e := toolkit.ToM(columnSet)
			if e != nil {
//.........这里部分代码省略.........

func (s *ServerController) SaveServers(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson
	r.Request.ParseMultipartForm(32 << 20)
	r.Request.ParseForm()

	path := filepath.Join(EC_DATA_PATH, "server", "log")
	log, _ := toolkit.NewLog(false, true, path, "log-%s", "20060102-1504")

	data := new(colonycore.Server)
	if r.Request.FormValue("sshtype") == "File" {
		log.AddLog("Get forms", "INFO")
		dataRaw := map[string]interface{}{}
		err := r.GetForms(&dataRaw)
		if err != nil {
			log.AddLog(err.Error(), "ERROR")
			return helper.CreateResult(false, nil, err.Error())
		}

		log.AddLog("Serding data", "INFO")
		err = toolkit.Serde(dataRaw, &data, "json")
		if err != nil {
			log.AddLog(err.Error(), "ERROR")
			return helper.CreateResult(false, nil, err.Error())
		}
	} else {
		log.AddLog("Get payload", "INFO")
		err := r.GetPayload(&data)
		if err != nil {
			log.AddLog(err.Error(), "ERROR")
			return helper.CreateResult(false, nil, err.Error())
		}
	}

	if data.SSHType == "File" {
		log.AddLog("Fetching public key", "INFO")
		reqFileName := "privatekey"
		file, _, err := r.Request.FormFile(reqFileName)
		if err != nil {
			log.AddLog(err.Error(), "ERROR")
			return helper.CreateResult(false, nil, err.Error())
		}

		if file != nil {
			log.AddLog("Saving public key", "INFO")
			data.SSHFile = filepath.Join(EC_DATA_PATH, "server", "privatekeys", data.ID)
			_, _, err = helper.FetchThenSaveFile(r.Request, reqFileName, data.SSHFile)
			if err != nil {
				log.AddLog(err.Error(), "ERROR")
				return helper.CreateResult(false, nil, err.Error())
			}
		}
	}
	oldData := new(colonycore.Server)

	log.AddLog(fmt.Sprintf("Find server ID: %s", data.ID), "INFO")
	cursor, err := colonycore.Find(new(colonycore.Server), dbox.Eq("_id", data.ID))
	if err != nil {
		log.AddLog(err.Error(), "ERROR")
		return helper.CreateResult(false, nil, err.Error())
	}
	oldDataAll := []colonycore.Server{}
	err = cursor.Fetch(&oldDataAll, 0, false)
	if err != nil {
		log.AddLog(err.Error(), "ERROR")
		return helper.CreateResult(false, nil, err.Error())
	}
	defer cursor.Close()

	if len(oldDataAll) > 0 {
		oldData = &oldDataAll[0]
	}

	if data.ServerType == "hdfs" {
		log.AddLog(fmt.Sprintf("SSH Connect %v", data), "INFO")
		hadeepes, err := hdfs.NewWebHdfs(hdfs.NewHdfsConfig(data.Host, data.SSHUser))
		if err != nil {
			log.AddLog(err.Error(), "ERROR")
			return helper.CreateResult(false, nil, err.Error())
		}

		_, err = hadeepes.List("/")
		if err != nil {
			return helper.CreateResult(false, nil, err.Error())
		}

		hadeepes.Config.TimeOut = 5 * time.Millisecond
		hadeepes.Config.PoolSize = 100

		return helper.CreateResult(true, nil, "")
	}

	log.AddLog(fmt.Sprintf("SSH Connect %v", data), "INFO")
	sshSetting, client, err := s.SSHConnect(data)
	if err != nil {
		log.AddLog(err.Error(), "ERROR")
		return helper.CreateResult(false, nil, err.Error())
	}
	defer client.Close()

	if data.OS == "linux" {
//.........这里部分代码省略.........

//.........这里部分代码省略.........
				SshClient: &lv.SshSetting{
					SSHHost:     data.ExedCommandStop.SshHost + ":" + data.ExedCommandStop.SshPort,
					SSHUser:     data.ExedCommandStop.SshUser,
					SSHPassword: data.ExedCommandStop.SshPassword,
					SSHAuthType: lv.SSHAuthType_Password,
				},
				CommandText:     data.ExedCommandStop.CommandText,
				ValidationType:  valtypestop,
				ValidationValue: "running",
			}
		} else {
			svc.CommandStop = &lv.Command{
				Type: lv.CommandType_SSH,
				SshClient: &lv.SshSetting{
					SSHHost:        data.ExedCommandStop.SshHost + ":" + data.ExedCommandStop.SshPort,
					SSHUser:        data.ExedCommandStop.SshUser,
					SSHKeyLocation: data.ExedCommandStop.SshKeyLocation,
					SSHAuthType:    lv.SSHAuthType_Certificate,
				},
				CommandText:     data.ExedCommandStop.CommandText,
				ValidationType:  valtypestop,
				ValidationValue: "running",
			}
		}
	} else {
		var valrestaunth lv.RESTAuthTypeEnum
		if data.ExedCommandStop.RestAuthType == "RESTAuthType_None" {
			valrestaunth = lv.RESTAuthType_None
		} else {
			valrestaunth = lv.RESTAuthType_Basic
		}
		svc.CommandStop = &lv.Command{
			Type:            lv.CommandType_REST,
			RESTUrl:         data.ExedCommandStop.RestUrl,
			RESTMethod:      data.ExedCommandStop.RestMenthod, //POST,GET
			RESTUser:        data.ExedCommandStop.RestUser,
			RESTPassword:    data.ExedCommandStop.RestPassword,
			RESTAuthType:    valrestaunth,
			ValidationType:  valtypestop,
			ValidationValue: "SUCCESS",
		}
	}

	svc.EmailError = data.Service.EmailWarning
	svc.EmailWarning = data.Service.EmailError

	svc.Mail = &lv.EmailSetting{
		SenderEmail:   "[email protected]",
		HostEmail:     "smtp.office365.com",
		PortEmail:     587,
		UserEmail:     "[email protected]",
		PasswordEmail: "B920Support",
	}

	svc.Log, err = toolkit.NewLog(false, true, logservicepath+"/", "LogService"+strconv.Itoa(data.Service.ID), "20060102")
	if err != nil {
		fmt.Println("Error Start Log : %s", err.Error())
	}
	datasvc := modelsvc{}
	datasvc.ID = data.Service.ID
	datasvc.svc = svc

	if statuslive == "Start" && data.Service.StatusService == "Start" {
		if len(arrsvc) == 0 && data.Service.StatusService == "Start" {
			svc.KeepAlive()
			arrsvc = append(arrsvc, datasvc)
			return "Preparing"
		}
		for j := 0; j < len(arrsvc); j++ {
			if arrsvc[j].ID != data.Service.ID && data.Service.StatusService == "Start" {
				if statuslive == "Start" {
					svc.KeepAlive()
					arrsvc = append(arrsvc, datasvc)
					return "Preparing"
				}
			} else if data.Service.StatusService == "Start" {
				if statuslive == "Start" {
					svc.KeepAlive()
					arrsvc[j] = datasvc
					return "Preparing"
				}
			}
		}
	} else if statuslive == "Live" && data.Service.StatusService == "Start" {
		for j := 0; j < len(arrsvc); j++ {
			if arrsvc[j].ID == data.Service.ID {
				if arrsvc[j].svc.MonitorStatus == "Running" {
					if arrsvc[j].svc.PingStatus == "OK" {
						return arrsvc[j].svc.PingStatus
					} else if arrsvc[j].svc.PingStatus == "Fail" {
						return arrsvc[j].svc.PingStatus
					} else {
						return arrsvc[j].svc.PingStatus
					}
				}
			}
		}
	}
	return "Fail"
}

func GrabDocConfig(data toolkit.M) (*sdt.GrabService, error) {
	var e error
	var gi, ti time.Duration

	GrabService := sdt.NewGrabService()
	GrabService.Name = data["nameid"].(string) //"iopriceindices"
	GrabService.SourceType = sdt.SourceType_DocExcel

	grabintervalToInt := toolkit.ToInt(data["grabinterval"], toolkit.RoundingAuto)
	timeintervalToInt := toolkit.ToInt(data["timeoutinterval"], toolkit.RoundingAuto)
	if data["intervaltype"].(string) == "seconds" {
		gi = time.Duration(grabintervalToInt) * time.Second
		ti = time.Duration(timeintervalToInt) * time.Second
	} else if data["intervaltype"].(string) == "minutes" {
		gi = time.Duration(grabintervalToInt) * time.Minute
		ti = time.Duration(timeintervalToInt) * time.Minute
	} else if data["intervaltype"].(string) == "hours" {
		gi = time.Duration(grabintervalToInt) * time.Hour
		ti = time.Duration(timeintervalToInt) * time.Hour
	}
	GrabService.GrabInterval = gi
	GrabService.TimeOutInterval = ti //time.Hour, time.Minute, time.Second
	GrabService.TimeOutIntervalInfo = fmt.Sprintf("%v %s", timeintervalToInt, data["intervaltype"])

	ci := dbox.ConnectionInfo{}

	grabDataConf, e := toolkit.ToM(data["grabconf"])
	if e != nil {
		return nil, e
	}
	isDoctype := grabDataConf.Has("doctype")
	if isDoctype {
		connToMap, e := toolkit.ToM(grabDataConf["connectioninfo"])
		if e != nil {
			return nil, e
		}
		ci.Host = connToMap["host"].(string) //"E:\\data\\sample\\IO Price Indices.xlsm"
		if hasSettings := connToMap.Has("settings"); !hasSettings {
			ci.Settings = nil
		} else {
			settingToMap, e := toolkit.ToM(connToMap["settings"])
			if e != nil {
				return nil, e
			}
			ci.Settings = settingToMap //toolkit.M{}.Set("useheader", settingToMap["useheader"].(bool)).Set("delimiter", settingToMap["delimiter"])
		}
		GrabService.ServGetData, e = sdt.NewGetDatabase(ci.Host, grabDataConf["doctype"].(string), &ci)
	}

	logconfToMap, e := toolkit.ToM(data["logconf"])
	if e != nil {
		return nil, e
	}
	logpath := logconfToMap["logpath"].(string)           //"E:\\data\\vale\\log"
	filename := logconfToMap["filename"].(string) + "-%s" //"LOG-LOCALXLSX-%s"
	filepattern := logconfToMap["filepattern"].(string)   //"20060102"

	logconf, e := toolkit.NewLog(false, true, logpath, filename, filepattern)
	if e != nil {
		return nil, e
	}

	GrabService.Log = logconf

	GrabService.ServGetData.CollectionSettings = make(map[string]*sdt.CollectionSetting)
	GrabService.DestDbox = make(map[string]*sdt.DestInfo)

	tempDataSetting := sdt.CollectionSetting{}
	tempDestInfo := sdt.DestInfo{}

	for _, dataSet := range data["datasettings"].([]interface{}) {
		dataToMap, e := toolkit.ToM(dataSet)
		if e != nil {
			return nil, e
		}
		tempDataSetting.Collection = dataToMap["rowselector"].(string) //"HIST"
		for _, columnSet := range dataToMap["columnsettings"].([]interface{}) {
			columnToMap, e := toolkit.ToM(columnSet)
			if e != nil {
				return nil, e
			}
			tempDataSetting.SelectColumn = append(tempDataSetting.SelectColumn, &sdt.GrabColumn{Alias: columnToMap["alias"].(string), Selector: columnToMap["selector"].(string)})
		}
		GrabService.ServGetData.CollectionSettings[dataToMap["name"].(string)] = &tempDataSetting //DATA01 use name in datasettings

		// fmt.Println("doctype>", grabDataConf["doctype"])
		connToMap, e := toolkit.ToM(dataToMap["connectioninfo"])
		if e != nil {
			return nil, e
		}
		var db, usr, pwd string

		if hasDb := connToMap.Has("database"); !hasDb {
			db = ""
		} else {
			db = connToMap["database"].(string)
		}
		if hasUser := connToMap.Has("username"); !hasUser {
			usr = ""
		} else {
//.........这里部分代码省略.........

func (s *ServerController) SaveServers(r *knot.WebContext) interface{} {
	r.Config.OutputType = knot.OutputJson
	r.Request.ParseMultipartForm(32 << 20)
	r.Request.ParseForm()

	path := filepath.Join(EC_DATA_PATH, "server", "log")
	log, _ := toolkit.NewLog(false, true, path, "log-%s", "20060102-1504")

	data := new(colonycore.Server)
	if r.Request.FormValue("serviceSSH[type]") == "File" {
		log.AddLog("Get forms", "INFO")
		dataRaw := map[string]interface{}{}
		err := r.GetForms(&dataRaw)
		if err != nil {
			log.AddLog(err.Error(), "ERROR")
			return helper.CreateResult(false, nil, err.Error())
		}

		log.AddLog("Serding data", "INFO")
		err = toolkit.Serde(dataRaw, &data, "json")
		if err != nil {
			log.AddLog(err.Error(), "ERROR")
			return helper.CreateResult(false, nil, err.Error())
		}
	} else {
		log.AddLog("Get payload", "INFO")
		err := r.GetPayload(&data)
		if err != nil {
			log.AddLog(err.Error(), "ERROR")
			return helper.CreateResult(false, nil, err.Error())
		}
	}

	if data.ServiceSSH.Type == "File" {
		log.AddLog("Fetching public key", "INFO")
		reqFileName := "privatekey"
		file, _, err := r.Request.FormFile(reqFileName)
		if err != nil {
			log.AddLog(err.Error(), "ERROR")
			return helper.CreateResult(false, nil, err.Error())
		}

		if file != nil {
			log.AddLog("Saving public key", "INFO")
			data.ServiceSSH.File = filepath.Join(EC_DATA_PATH, "server", "privatekeys", data.ID)
			_, _, err = helper.FetchThenSaveFile(r.Request, reqFileName, data.ServiceSSH.File)
			if err != nil {
				log.AddLog(err.Error(), "ERROR")
				return helper.CreateResult(false, nil, err.Error())
			}
		}
	}

	if data.IsAccessValid("node") && data.IsColonyServer {
		if data.OS == "linux" {
			if err := data.InstallColonyOnLinux(log); err != nil {
				return helper.CreateResult(false, nil, err.Error())
			}
		} else {
			if err := data.InstallColonyOnWindows(log); err != nil {
				return helper.CreateResult(false, nil, err.Error())
			}
		}

		data.DetectInstalledLang()
		data.DetectService()
		data.UpdateInternalAppDeploymentStatus("add")

		log.AddLog("Restart sedotand", "INFO")
		if _, err := data.ToggleSedotanService("start stop", data.ID); err != nil {
			log.AddLog(err.Error(), "ERROR")
		}
	}

	if data.IsAccessValid("hdfs") {
		log.AddLog(fmt.Sprintf("SSH Connect %v", data), "INFO")
		hdfsConfig := hdfs.NewHdfsConfig(data.ServiceHDFS.Host, data.ServiceHDFS.User)
		hdfsConfig.Password = data.ServiceHDFS.Pass

		hadeepes, err := hdfs.NewWebHdfs(hdfsConfig)
		if err != nil {
			log.AddLog(err.Error(), "ERROR")
			return helper.CreateResult(false, nil, err.Error())
		}

		_, err = hadeepes.List("/")
		if err != nil {
			return helper.CreateResult(false, nil, err.Error())
		}

		hadeepes.Config.TimeOut = 5 * time.Millisecond
		hadeepes.Config.PoolSize = 100
	}

	runsrv, e := new(colonycore.ServerService).GetAll()
	if e != nil {
		return helper.CreateResult(false, nil, e.Error())
	}

	svr, _, a := data.Connect()
//.........这里部分代码省略.........

func (s *Server) Log() *toolkit.LogEngine {
	if s.log == nil {
		s.log, _ = toolkit.NewLog(true, false, "", "", "")
	}
	return s.log
}

func TestServiceGrabDocument(t *testing.T) {
	var e error

	xGrabService := NewGrabService()
	xGrabService.Name = "iopriceindices"

	xGrabService.SourceType = SourceType_DocExcel

	xGrabService.GrabInterval = 5 * time.Minute
	xGrabService.TimeOutInterval = 10 * time.Second //time.Hour, time.Minute, time.Second

	xGrabService.TimeOutIntervalInfo = fmt.Sprintf("%v %s", 1, "seconds")

	//==must have grabconf and Connection info inside grabconf         ===========================================
	// mapValConfig, e := toolkit.ToM(mapVal.Get("grabconf", nil).(map[string]interface{}))
	// mapConnVal, e := toolkit.ToM(mapValConfig.Get("connectioninfo", nil).(map[string]interface{}))

	ci := dbox.ConnectionInfo{}

	ci.Host = "E:\\data\\sample\\IO Price Indices.xlsm"
	// ci.Database = mapConnVal.Get("database", "").(string)
	// ci.UserName = mapConnVal.Get("userName", "").(string)
	// ci.Password = mapConnVal.Get("password", "").(string)

	// if have setting inside of connection info
	// ci.Settings, e = toolkit.ToM(tempSetting.(map[string]interface{}))
	ci.Settings = nil

	xGrabService.ServGetData, e = NewGetDatabase(ci.Host, "xlsx", &ci)

	//===================================================================

	//==For Data Log          ===========================================

	// 	logpath = tempLogConf.Get("logpath", "").(string)
	// 	filename = tempLogConf.Get("filename", "").(string)
	// 	filepattern = tempLogConf.Get("filepattern", "").(string)

	logpath := "E:\\data\\vale\\log"
	filename := "LOG-LOCALXLSX-%s"
	filepattern := "20060102"

	logconf, e := toolkit.NewLog(false, true, logpath, filename, filepattern)
	if e != nil {
		t.Errorf("Error Found : ", e)
	}

	xGrabService.Log = logconf
	//===================================================================

	//===================================================================
	//==Data Setting and Destination Save =====================

	xGrabService.ServGetData.CollectionSettings = make(map[string]*CollectionSetting)
	xGrabService.DestDbox = make(map[string]*DestInfo)

	// ==For Every Data Setting ===============================
	tempDataSetting := CollectionSetting{}
	tempDestInfo := DestInfo{}

	// .Collection = mapxVal.Get("rowselector", "").(string)
	tempDataSetting.Collection = "HIST"
	tempDataSetting.SelectColumn = append(tempDataSetting.SelectColumn, &GrabColumn{Alias: "Date", Selector: "1"})
	tempDataSetting.SelectColumn = append(tempDataSetting.SelectColumn, &GrabColumn{Alias: "Platts 62% Fe IODEX", Selector: "2"})
	tempDataSetting.SelectColumn = append(tempDataSetting.SelectColumn, &GrabColumn{Alias: "Platts 65% Fe", Selector: "4"})
	tempDataSetting.SelectColumn = append(tempDataSetting.SelectColumn, &GrabColumn{Alias: "TSI 62% Fe", Selector: "15"})
	tempDataSetting.SelectColumn = append(tempDataSetting.SelectColumn, &GrabColumn{Alias: "TSI 65% Fe", Selector: "16"})
	tempDataSetting.SelectColumn = append(tempDataSetting.SelectColumn, &GrabColumn{Alias: "TSI 62% Fe LOW ALUMINA", Selector: "17"})
	tempDataSetting.SelectColumn = append(tempDataSetting.SelectColumn, &GrabColumn{Alias: "MB 62% Fe", Selector: "26"})
	tempDataSetting.SelectColumn = append(tempDataSetting.SelectColumn, &GrabColumn{Alias: "MB 65% Fe", Selector: "29"})

	// tempDataSetting.SetFilterCond(tempFilterCond)
	// -Check "filtercond" in config-
	// tempFilterCond, e = toolkit.ToM(mapxVal.Get("filtercond", nil).(map[string]interface{}))
	// tempDataSetting.SetFilterCond(tempFilterCond)

	xGrabService.ServGetData.CollectionSettings["DATA01"] = &tempDataSetting //DATA01 use name in datasettings

	ci = dbox.ConnectionInfo{}
	ci.Host = "localhost:27017"
	ci.Database = "valegrab"
	ci.UserName = ""
	ci.Password = ""
	// ci.Settings = toolkit.M{}.Set("useheader", true).Set("delimiter", ",")
	// setting will be depend on config file

	tempDestInfo.Collection = "iopriceindices"
	tempDestInfo.Desttype = "mongo"

	tempDestInfo.IConnection, e = dbox.NewConnection(tempDestInfo.Desttype, &ci)
	if e != nil {
		t.Errorf("Error Found : ", e)
	}

	xGrabService.DestDbox["DATA01"] = &tempDestInfo
	//=History===========================================================
	xGrabService.HistoryPath = "E:\\data\\vale\\history\\"
	xGrabService.HistoryRecPath = "E:\\data\\vale\\historyrec\\"
	//===================================================================

//.........这里部分代码省略.........

func (s *Service) AddLog(logtype, logtext string) {
	if s.logEngine == nil {
		s.logEngine = toolkit.NewLog(true, false, "", "", false)
	}
	s.logEngine.AddLog(logtext, logtype)
}