Golang persist.JobInfo类代码示例

func (j *jobRunner) runJobInfo(persistJobInfo *persist.JobInfo) error {
	switch {
	case persistJobInfo.GetTransform() != nil:
		return j.reallyRunJobInfo(
			uuid.NewWithoutDashes(),
			persistJobInfo.JobId,
			persistJobInfo.GetTransform(),
			persistJobInfo.Input,
			persistJobInfo.OutputParent,
			1,
		)
	case persistJobInfo.GetPipelineName() != "":
		persistPipelineInfo, err := j.persistAPIClient.GetPipelineInfo(
			context.Background(),
			&pps.Pipeline{Name: persistJobInfo.GetPipelineName()},
		)
		if err != nil {
			return err
		}
		if persistPipelineInfo.GetTransform() == nil {
			return fmt.Errorf("pachyderm.pps.server: transform not set on pipeline info %v", persistPipelineInfo)
		}
		return j.reallyRunJobInfo(
			persistPipelineInfo.PipelineName,
			persistJobInfo.JobId,
			persistPipelineInfo.GetTransform(),
			persistJobInfo.Input,
			persistJobInfo.OutputParent,
			1,
		)
	default:
		return fmt.Errorf("pachyderm.pps.server: neither transform or pipeline name set on job info %v", persistJobInfo)
	}
}

func (a *apiServer) persistJobInfoToJobInfo(ctx context.Context, persistJobInfo *persist.JobInfo) (*pps.JobInfo, error) {
	job := &pps.Job{Id: persistJobInfo.JobId}
	jobInfo := &pps.JobInfo{
		Job:         job,
		Shards:      persistJobInfo.Shards,
		InputCommit: persistJobInfo.InputCommit,
	}
	if persistJobInfo.GetTransform() != nil {
		jobInfo.Spec = &pps.JobInfo_Transform{
			Transform: persistJobInfo.GetTransform(),
		}
	}
	if persistJobInfo.GetPipelineName() != "" {
		jobInfo.Spec = &pps.JobInfo_Pipeline{
			Pipeline: &pps.Pipeline{
				Name: persistJobInfo.GetPipelineName(),
			},
		}
	}
	persistJobOutput, err := a.persistAPIClient.GetJobOutput(ctx, job)
	if err != nil {
		return nil, err
	}
	if persistJobOutput != nil {
		jobInfo.OutputCommit = persistJobOutput.OutputCommit
	}
	return jobInfo, nil
}

// job_id cannot be set
// timestamp cannot be set
func (a *rethinkAPIServer) CreateJobInfo(ctx context.Context, request *persist.JobInfo) (response *persist.JobInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if request.JobId != "" {
		return nil, ErrIDSet
	}
	if request.CreatedAt != nil {
		return nil, ErrTimestampSet
	}
	request.JobId = uuid.New()
	request.CreatedAt = a.now()
	if err := a.insertMessage(jobInfosTable, request); err != nil {
		return nil, err
	}
	return request, nil
}

// JobId cannot be set
// Timestamp cannot be set
func (a *rethinkAPIServer) CreateJobInfo(ctx context.Context, request *persist.JobInfo) (response *persist.JobInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	if request.JobId != "" {
		return nil, fmt.Errorf("request.JobId should be unset")
	}
	if request.CreatedAt != nil {
		return nil, fmt.Errorf("request.CreatedAt should be unset")
	}
	if request.CommitIndex != "" {
		return nil, fmt.Errorf("request.CommitIndex should be unset")
	}
	request.JobId = uuid.NewWithoutDashes()
	request.CreatedAt = prototime.TimeToTimestamp(time.Now())
	request.CommitIndex = genCommitIndex(request.InputCommit)
	if err := a.insertMessage(jobInfosTable, request); err != nil {
		return nil, err
	}
	return request, nil
}

func job(jobInfo *persist.JobInfo) *extensions.Job {
	app := jobInfo.JobId
	shards := int(jobInfo.Shards)
	return &extensions.Job{
		TypeMeta: unversioned.TypeMeta{
			Kind:       "Job",
			APIVersion: "v1",
		},
		ObjectMeta: api.ObjectMeta{
			Name:   jobInfo.JobId,
			Labels: labels(app),
		},
		Spec: extensions.JobSpec{
			Selector: &extensions.PodSelector{
				MatchLabels: labels(app),
			},
			Parallelism: &shards,
			Completions: &shards,
			Template: api.PodTemplateSpec{
				ObjectMeta: api.ObjectMeta{
					Name:   jobInfo.JobId,
					Labels: labels(app),
				},
				Spec: api.PodSpec{
					Containers: []api.Container{
						{
							Name:    "user",
							Image:   "pachyderm/job-shim",
							Command: append([]string{"/job-shim", jobInfo.JobId}, jobInfo.GetTransform().Cmd...),
							SecurityContext: &api.SecurityContext{
								Privileged: &trueVal, // god is this dumb
							},
						},
					},
					RestartPolicy: "Never",
				},
			},
		},
	}
}

func job(jobInfo *persist.JobInfo) *extensions.Job {
	app := jobInfo.JobId
	return &extensions.Job{
		TypeMeta: unversioned.TypeMeta{
			Kind:       "Job",
			APIVersion: "v1",
		},
		ObjectMeta: api.ObjectMeta{
			Name: jobInfo.JobId,
			Labels: map[string]string{
				"app": app,
			},
		},
		Spec: extensions.JobSpec{
			Selector: &extensions.PodSelector{
				MatchLabels: map[string]string{
					"app": app,
				},
			},
			Template: api.PodTemplateSpec{
				ObjectMeta: api.ObjectMeta{
					Name: jobInfo.JobId,
					Labels: map[string]string{
						"app": app,
					},
				},
				Spec: api.PodSpec{
					Containers: []api.Container{
						{
							Name:    "user",
							Image:   jobInfo.GetTransform().Image,
							Command: jobInfo.GetTransform().Cmd,
						},
					},
				},
			},
		},
	}
}

// TODO: bulk get
func (a *apiServer) persistJobInfoToJobInfo(ctx context.Context, persistJobInfo *persist.JobInfo) (*pps.JobInfo, error) {
	job := &pps.Job{Id: persistJobInfo.JobId}
	persistJobStatuses, err := a.persistAPIClient.GetJobStatuses(ctx, job)
	if err != nil {
		return nil, err
	}
	persistJobOutput, err := a.persistAPIClient.GetJobOutput(ctx, job)
	if err != nil {
		return nil, err
	}
	jobInfo := &pps.JobInfo{
		Job:   job,
		Input: persistJobInfo.Input,
	}
	if persistJobInfo.GetTransform() != nil {
		jobInfo.Spec = &pps.JobInfo_Transform{
			Transform: persistJobInfo.GetTransform(),
		}
	}
	if persistJobInfo.GetPipelineName() != "" {
		jobInfo.Spec = &pps.JobInfo_Pipeline{
			Pipeline: &pps.Pipeline{
				Name: persistJobInfo.GetPipelineName(),
			},
		}
	}
	jobInfo.JobStatus = make([]*pps.JobStatus, len(persistJobStatuses.JobStatus))
	for i, persistJobStatus := range persistJobStatuses.JobStatus {
		jobInfo.JobStatus[i] = &pps.JobStatus{
			Type:      persistJobStatus.Type,
			Timestamp: persistJobStatus.Timestamp,
			Message:   persistJobStatus.Message,
		}
	}
	if persistJobOutput != nil {
		jobInfo.Output = persistJobOutput.Output
	}
	return jobInfo, nil
}