Golang client.Debug函数代码示例

func TestLRU(t *testing.T) {
	gopath := os.Getenv("GOPATH")
	if _, err := os.Stat(gopath + "/src/segments"); err != nil {
		if os.IsNotExist(err) {
			os.Mkdir(gopath+"/src/segments", 0777)
		} else {
			panic(err)
		}
	}
	tuple1 := Tuple{Slice: []string{"Vedha", "Vikas", "Jeffrey", "Zack"}}
	tuple2 := Tuple{Slice: []string{"Vivek", "Anuhya", "Esha"}}
	tuple3 := Tuple{Slice: []string{"Christina", "Keerti"}}
	tuple4 := Tuple{Slice: []string{"Suganya", "Arooshi"}}

	var segment1 Segment
	segment1.Partitions = make([][]Tuple, 2)
	segment1.Partitions[0] = []Tuple{tuple1, tuple2}
	segment1.Partitions[1] = []Tuple{tuple3, tuple4}
	segment1.Id = 1234

	var segment2 Segment
	segment2.Partitions = make([][]Tuple, 2)
	segment2.Partitions[0] = []Tuple{tuple1, tuple3}
	segment2.Partitions[1] = []Tuple{tuple2, tuple4}
	segment2.Id = 1111

	lru := NewLRU(1, 4)
	lru.Insert(1234, &segment1)
	lru.Insert(1111, &segment2)
	s := lru.Get(1234)
	s2 := lru.Get(1111)
	client.Debug("Here's what I got", s)
	client.Debug(s2)
	client.Debug("Length", lru.Length())
}

func main() {
	hd := master.GetDbConnection()

	workflows := master.GetWorkflows(hd)

	for _, w := range workflows {
		client.Debug("--------------------------Begin--------------------------")
		client.Debug("Workflow ID:", w.Id)
		client.Debug(workflow.WorkflowToString(hd, w))
		client.Debug("---------------------------End---------------------------")
	}
}

func main() {
	if len(os.Args) != 3 {
		printUsage()
		return
	}

	workerhost := os.Args[1]
	masterhost := os.Args[2]
	client.Debug("Starting server on", workerhost)
	client.Debug("Press Ctrl-C to stop")
	worker.StartServer(workerhost, masterhost)

	waitForInterrupt()
}

func main() {
	if len(os.Args) != 2 {
		printUsage()
		return
	}

	host := os.Args[1]
	hd := master.GetDbConnection()
	client.Debug("Starting server on", host)
	client.Debug("Press Ctrl-C to stop")
	master.StartServer(host, hd)

	waitForInterrupt()
}

// Execute a UDF command that accepts zero or more input lists of tuples, and
// returns one output list of tuples. This function blocks until the UDF is
// done executing.
func runUDF(command string, inputTuples map[int][]Tuple) []Tuple {
	// spawn the external process
	splits := strings.Split(command, " ")
	client.Debug(preprocessCommand(splits[0]))
	cmd := exec.Command(preprocessCommand(splits[0]), splits[1:]...)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		log.Panic(err)
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Panic(err)
	}
	cmd.Start()

	// write tuples to standard input on a background goroutine
	go func() {
		for index, tupleList := range inputTuples {
			for _, tuple := range tupleList {
				stdin.Write(tuple.SerializeTuple(index))
				stdin.Write([]byte{'\n'})
			}
		}
		stdin.Close()
	}()

	// read from standard output to get the output tuples
	outputTuples := make([]Tuple, 0)
	ReadTupleStream(stdout, func(tuple Tuple, index int) {
		outputTuples = append(outputTuples, tuple)
	})

	return outputTuples
}

func (m *Master) execLaunchJob(rddId int64, data interface{}) {
	client.Debug("execLaunchJob", rddId)

	m.mu.Lock()
	defer m.mu.Unlock()
	tx := m.hd.Begin()

	// TODO: check that all of the input RDDS are available

	rdd := GetRdd(tx, rddId)

	// check whether the rdd is already complete
	if rdd.State != RDD_COMPLETE {
		sourceRdds := rdd.GetSourceRdds(tx)

		readyToContinue := true
		for _, srcRdd := range sourceRdds {
			if srcRdd.State != RDD_COMPLETE {
				// relaunch any dependencies that are not complete
				readyToContinue = false
				e := Event{
					Type: LAUNCH_JOB,
					Id:   int64(srcRdd.Id),
				}
				m.queueEvent(e)
			}
		}

		// If all the dependencies are met, then launch the next
		// Rdd (dependencies are also checked for each individual task)
		if readyToContinue {
			// check whether we already created segments for this rdd
			segments := rdd.GetSegments(tx)
			if len(segments) > 0 {
				// if segments already present, just run the ones that are not complete
				// (this is part of the recovery protocol)
				for _, segment := range segments {
					if segment.Status != SEGMENT_COMPLETE {
						e := Event{
							Type: LAUNCH_TASK,
							Id:   int64(segment.Id),
						}
						m.queueEvent(e)
					}
				}
			} else {
				// Otherwise, create the new segments and run them
				segments, _ := rdd.CreateSegments(tx)
				for _, segment := range segments {
					e := Event{
						Type: LAUNCH_TASK,
						Id:   int64(segment.Id),
					}
					m.queueEvent(e)
				}
			}
		}
	}
	commitOrPanic(tx)
}

func (m *Master) tryLaunchingDependentJobs(tx *hood.Hood, rdd *Rdd, pj *Protojob) {

	destRdds := rdd.GetDestRdds(tx)

	// For each destRdd, check whether all of the srcRdds
	// for that destRdd are complete. If so, launch the job
	// for destRdd
	// TODO: this logic will have to be re-written when fault-tolerance
	// is implemented
	for _, destRdd := range destRdds {
		srcRdds := destRdd.GetSourceRdds(tx)
		isComplete := true
		for _, srcRdd := range srcRdds {
			if (srcRdd.State != RDD_COMPLETE) && (srcRdd.Id != rdd.Id) {
				isComplete = false
			}
		}
		if isComplete {
			client.Debug("launching next job", destRdd)
			e := Event{
				Type: LAUNCH_JOB,
				Id:   int64(destRdd.Id),
			}
			m.queueEvent(e)
		}
	}
}

func (m *Master) execTaskSuccess(segmentId int64, data interface{}) {
	client.Debug("execTaskSuccess", segmentId)

	m.mu.Lock()
	defer m.mu.Unlock()
	tx := m.hd.Begin()

	segment := GetSegment(tx, segmentId)
	rdd := segment.GetRdd(tx)
	pj := rdd.GetProtojob(tx)

	segment.Status = SEGMENT_COMPLETE
	saveOrPanic(tx, segment)

	numComplete := rdd.GetNumSegmentsComplete(tx, segment)

	if numComplete == pj.NumSegments {
		batch := rdd.GetWorkflowBatch(tx)
		workflow := batch.GetWorkflow(tx)
		fmt.Println("Job complete", rdd.Id, pj.Command, time.Now().UnixNano()/1000000-batch.StartTime-workflow.Duration)
		rdd.State = RDD_COMPLETE
		saveOrPanic(tx, rdd)
		m.tryLaunchingDependentJobs(tx, rdd, pj)
	}

	commitOrPanic(tx)
}

//
// server Register RPC handler.
//
func (m *Master) Register(args *client.RegisterArgs, reply *client.RegisterReply) error {
	client.Debug("Registering", args)

	m.mu.Lock()
	defer m.mu.Unlock()

	tx := m.hd.Begin()
	existingWorkers := GetWorkersAtAddress(tx, args.Me)
	for _, w := range existingWorkers {
		w.Status = WORKER_DEAD
		tx.Save(w)
	}
	newWorker := Worker{
		Url: args.Me,
	}
	tx.Save(&newWorker)
	commitOrPanic(tx)

	tx = m.hd.Begin()
	m.getNumAliveWorkers(tx)
	commitOrPanic(tx)

	reply.Err = client.OK
	reply.Id = int64(newWorker.Id)

	return nil
}

func waitForInterrupt() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	for sig := range c {
		client.Debug("\ncaptured signal, stopping and exiting.\n", sig)
		return
	}
}

func (m *Master) HandleFailureData(data *FailureData) {
	client.Debug("HANDLING FAILURE", data)
	tx := m.hd.Begin()
	worker := GetWorker(tx, data.WorkerId)
	worker.Status = WORKER_DEAD
	saveOrPanic(tx, worker)
	m.getNumAliveWorkers(tx)
	commitOrPanic(tx)
}

func (w *Worker) ExecTask(args *client.ExecArgs, reply *client.ExecReply) error {
	inputTuples := make(map[int][]Tuple)
	fmt.Println("executing task", args)
	for _, segment := range args.Segments {
		localSegment := w.LocalGetSegment(segment.SegmentId)
		// fetch the segment if it is not already stored locally
		if localSegment == nil {
			client.Debug("fetching tuples", segment)
			clerk := MakeWorkerInternalClerk(segment.WorkerUrl)
			args2 := GetTuplesArgs{SegmentId: segment.SegmentId, PartitionIndex: segment.PartitionIndex}
			reply2 := clerk.GetTuples(&args2, 3)
			if reply2 != nil {
				if reply2.Err == client.OK {
					client.Debug("fetched tuples", len(reply2.Tuples))
					inputTuples[segment.Index] = append(inputTuples[segment.Index], reply2.Tuples...)
				} else {
					reply.Err = reply2.Err
					reply.WorkerId = segment.WorkerId
					client.Debug(reply.Err)
					return nil
				}
			} else {
				reply.Err = client.DEAD_SEGMENT
				reply.WorkerId = segment.WorkerId
				client.Debug(reply.Err)
				return nil
			}
		} else {
			// use the locally stored copy
			inputTuples[segment.Index] = append(inputTuples[segment.Index], localSegment.Partitions[segment.PartitionIndex]...)
		}
	}

	client.Debug("running udf")
	start := time.Now()
	outputTuples := runUDF(args.Command, inputTuples)
	end := time.Now()
	client.Debug("duration:", end.Sub(start))
	client.Debug("got output tuples", len(outputTuples))

	client.Debug("writing segment")
	segment := MakeSegment(outputTuples, args.Indices, args.Parts)
	w.LocalPutSegment(args.OutputSegmentId, segment)

	client.Debug("success")
	reply.Err = client.OK
	return nil
}

func (m *Master) eventLoop() {
	to := 1
	iteration := 0
	for {
		if iteration%10 == 0 {
			fmt.Println("on iteration", iteration)
		}
		iteration += 1
		if atomic.LoadInt64(&m.numAliveWorkers) >= atomic.LoadInt64(&m.minWorkers) {
			start := time.Now()
			to = 1
			e := <-m.events
			atomic.AddInt64(&m.numQueuedEvents, -1)
			switch e.Type {
			case NEW_BATCH:
				m.execNewBatch(e.Id, e.Data)
			case LAUNCH_TASK:
				m.execLaunchTask(e.Id, e.Data)
			case TASK_SUCCESS:
				m.execTaskSuccess(e.Id, e.Data)
			case TASK_FAILURE:
				m.execTaskFailure(e.Id, e.Data)
			case LAUNCH_JOB:
				m.execLaunchJob(e.Id, e.Data)
			case COPY_SUCCESS:
				m.execCopySuccess(e.Id, e.Data)
			case COPY_FAILURE:
				m.execCopyFailure(e.Id, e.Data)
			case LAUNCH_COPY:
				m.execLaunchCopy(e.Id, e.Data)
			}
			diff := time.Now().Sub(start)
			client.Debug("duration", diff)
		} else {
			client.Debug("sleeping", to)
			time.Sleep(time.Duration(to) * time.Millisecond)
			if to < 1000 {
				to *= 2
			}
		}
	}
}

func (w *Worker) CopySegment(args *client.CopySegmentArgs, reply *client.CopySegmentReply) error {
	client.Debug("copying segment", args)
	if w.LocalGetSegment(args.SegmentId) != nil {
		// this should never happen during normal operaiton (though it might
		// happen during the master recovery procedure)
		client.Debug("already have segment, overwriting...")
	}
	client.Debug("fetching segment", args.SegmentId)
	clerk := MakeWorkerInternalClerk(args.WorkerUrl)
	args2 := GetSegmentArgs{SegmentId: args.SegmentId}
	reply2 := clerk.GetSegment(&args2, 3)
	if reply2 != nil {
		if reply2.Err == client.OK {
			client.Debug("fetched segment", args.SegmentId)
			w.LocalPutSegment(args.SegmentId, reply2.Segment)
			reply.Err = client.OK
		} else {
			reply.Err = reply2.Err
			reply.WorkerId = args.WorkerId
			client.Debug(reply.Err)
			return nil
		}
	} else {
		reply.Err = client.DEAD_SEGMENT
		reply.WorkerId = args.WorkerId
		client.Debug(reply.Err)
		return nil
	}
	return nil
}

func (m *Master) execTaskFailure(segmentId int64, data interface{}) {
	client.Debug("execTaskFailure", segmentId)

	m.mu.Lock()
	defer m.mu.Unlock()

	m.HandleFailureData(data.(*FailureData))
	e := Event{
		Type: LAUNCH_TASK,
		Id:   int64(segmentId),
	}
	m.queueEvent(e)
}