Golang model.LoadProjectInto函數代碼示例

// validateProjectConfig returns a slice containing a list of any errors
// found in validating the given project configuration
func (as *APIServer) validateProjectConfig(w http.ResponseWriter, r *http.Request) {
	defer r.Body.Close()
	yamlBytes, err := ioutil.ReadAll(r.Body)
	if err != nil {
		as.WriteJSON(w, http.StatusBadRequest, fmt.Sprintf("Error reading request body: %v", err))
		return
	}

	project := &model.Project{}
	validationErr := validator.ValidationError{}
	if err := model.LoadProjectInto(yamlBytes, "", project); err != nil {
		validationErr.Message = err.Error()
		as.WriteJSON(w, http.StatusBadRequest, []validator.ValidationError{validationErr})
		return
	}
	syntaxErrs, err := validator.CheckProjectSyntax(project)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	semanticErrs := validator.CheckProjectSemantics(project)
	if len(syntaxErrs)+len(semanticErrs) != 0 {
		as.WriteJSON(w, http.StatusBadRequest, append(syntaxErrs, semanticErrs...))
		return
	}
	as.WriteJSON(w, http.StatusOK, []validator.ValidationError{})
}

// GetRemoteConfig fetches the contents of a remote github repository's
// configuration data as at a given revision
func (gRepoPoller *GithubRepositoryPoller) GetRemoteConfig(
	projectFileRevision string) (projectConfig *model.Project, err error) {
	// find the project configuration file for the given repository revision
	projectRef := gRepoPoller.ProjectRef
	projectFileURL := thirdparty.GetGithubFileURL(
		projectRef.Owner,
		projectRef.Repo,
		projectRef.RemotePath,
		projectFileRevision,
	)

	githubFile, err := thirdparty.GetGithubFile(
		gRepoPoller.OauthToken,
		projectFileURL,
	)
	if err != nil {
		return nil, err
	}

	projectFileBytes, err := base64.StdEncoding.DecodeString(githubFile.Content)
	if err != nil {
		return nil, thirdparty.FileDecodeError{err.Error()}
	}

	projectConfig = &model.Project{}
	err = model.LoadProjectInto(projectFileBytes, projectRef.Identifier, projectConfig)
	if err != nil {
		return nil, thirdparty.YAMLFormatError{err.Error()}
	}

	return projectConfig, nil
}

// GetPatchedProject creates and validates a project created by fetching latest commit information from GitHub
// and applying the patch to the latest remote configuration. The error returned can be a validation error.
func GetPatchedProject(p *patch.Patch, settings *evergreen.Settings) (*model.Project, error) {
	if p.Version != "" {
		return nil, fmt.Errorf("Patch %v already finalized", p.Version)
	}
	projectRef, err := model.FindOneProjectRef(p.Project)
	if err != nil {
		return nil, err
	}

	// try to get the remote project file data at the requested revision
	var projectFileBytes []byte
	projectFileURL := thirdparty.GetGithubFileURL(
		projectRef.Owner,
		projectRef.Repo,
		projectRef.RemotePath,
		p.Githash,
	)
	githubFile, err := thirdparty.GetGithubFile(settings.Credentials["github"], projectFileURL)
	if err != nil {
		// if the project file doesn't exist, but our patch includes a project file,
		// we try to apply the diff and proceed.
		if !(p.ConfigChanged(projectRef.RemotePath) && thirdparty.IsFileNotFound(err)) {
			// return an error if the github error is network/auth-related or we aren't patching the config
			return nil, fmt.Errorf("Could not get github file at %v: %v", projectFileURL, err)
		}
	} else {
		// we successfully got the project file in base64, so we decode it
		projectFileBytes, err = base64.StdEncoding.DecodeString(githubFile.Content)
		if err != nil {
			return nil, fmt.Errorf("Could not decode github file at %v: %v", projectFileURL, err)
		}
	}

	project := &model.Project{}
	if err = model.LoadProjectInto(projectFileBytes, projectRef.Identifier, project); err != nil {
		return nil, err
	}
	// apply remote configuration patch if needed
	if p.ConfigChanged(projectRef.RemotePath) {
		project, err = model.MakePatchedConfig(p, projectRef.RemotePath, string(projectFileBytes))
		if err != nil {
			return nil, fmt.Errorf("Could not patch remote configuration file: %v", err)
		}
		// overwrite project fields with the project ref to disallow tracking a
		// different project or doing other crazy things via config patches
		errs := CheckProjectSyntax(project)
		if len(errs) != 0 {
			var message string
			for _, err := range errs {
				message += fmt.Sprintf("\n\t=> %v", err)
			}
			return nil, fmt.Errorf(message)
		}
	}
	return project, nil
}

// GetPatchedProject creates and validates a project created by fetching latest commit information from GitHub
// and applying the patch to the latest remote configuration. The error returned can be a validation error.
func GetPatchedProject(p *patch.Patch, settings *evergreen.Settings) (*model.Project, error) {
	if p.Version != "" {
		return nil, fmt.Errorf("Patch %v already finalized", p.Version)
	}
	projectRef, err := model.FindOneProjectRef(p.Project)
	if err != nil {
		return nil, err
	}

	// get the remote file at the requested revision
	projectFileURL := thirdparty.GetGithubFileURL(
		projectRef.Owner,
		projectRef.Repo,
		projectRef.RemotePath,
		p.Githash,
	)

	githubFile, err := thirdparty.GetGithubFile(
		settings.Credentials["github"],
		projectFileURL,
	)
	if err != nil {
		return nil, fmt.Errorf("Could not get github file at %v: %v", projectFileURL, err)
	}

	projectFileBytes, err := base64.StdEncoding.DecodeString(githubFile.Content)
	if err != nil {
		return nil, fmt.Errorf("Could not decode github file at %v: %v", projectFileURL, err)
	}

	project := &model.Project{}

	if err = model.LoadProjectInto(projectFileBytes, projectRef.Identifier, project); err != nil {
		return nil, err
	}
	// apply remote configuration patch if needed
	if p.ConfigChanged(projectRef.RemotePath) {
		project, err = model.MakePatchedConfig(p, projectRef.RemotePath, string(projectFileBytes))
		if err != nil {
			return nil, fmt.Errorf("Could not patch remote configuration file: %v", err)
		}
		// overwrite project fields with the project ref to disallow tracking a
		// different project or doing other crazy things via config patches
		errs := CheckProjectSyntax(project)
		if len(errs) != 0 {
			var message string
			for _, err := range errs {
				message += fmt.Sprintf("\n\t=> %v", err)
			}
			return nil, fmt.Errorf(message)
		}
	}
	return project, nil
}

// GetProjectConfig loads the communicator's task's project from the API server.
func (h *HTTPCommunicator) GetProjectConfig() (*model.Project, error) {
	projectConfig := &model.Project{}
	retriableGet := util.RetriableFunc(
		func() error {
			resp, err := h.tryGet("version")
			if resp != nil {
				defer resp.Body.Close()
			}
			if resp != nil && resp.StatusCode == http.StatusConflict {
				// Something very wrong, fail now with no retry.
				return fmt.Errorf("conflict - wrong secret!")
			}
			if err != nil {
				// Some generic error trying to connect - try again
				return util.RetriableError{err}
			}
			if resp == nil {
				return util.RetriableError{fmt.Errorf("empty response")}
			} else {
				v := &version.Version{}
				err = util.ReadJSONInto(resp.Body, v)
				if err != nil {
					h.Logger.Errorf(slogger.ERROR,
						"unable to read project version response: %v\n", err)
					return util.RetriableError{fmt.Errorf("unable to read "+
						"project version response: %v\n", err)}
				}
				err = model.LoadProjectInto([]byte(v.Config), v.Project, projectConfig)
				if err != nil {
					h.Logger.Errorf(slogger.ERROR,
						"unable to unmarshal project config: %v\n", err)
					return util.RetriableError{fmt.Errorf("unable to "+
						"unmarshall project config: %v\n", err)}
				}
				return nil
			}
		},
	)

	retryFail, err := util.Retry(retriableGet, h.MaxAttempts, h.RetrySleep)
	if retryFail {
		return nil, fmt.Errorf("getting project configuration failed after %v "+
			"tries: %v", h.MaxAttempts, err)
	}
	return projectConfig, nil
}

// Construct a map of project names to build variants for that project
func findProjectBuildVariants(configName string) (map[string][]string, error) {
	projectNameToBuildVariants := make(map[string][]string)

	allProjects, err := model.FindAllTrackedProjectRefs()
	if err != nil {
		return nil, err
	}

	for _, projectRef := range allProjects {
		if !projectRef.Enabled {
			continue
		}
		var buildVariants []string
		var proj *model.Project
		var err error
		if projectRef.LocalConfig != "" {
			proj, err = model.FindProject("", &projectRef)
			if err != nil {
				return nil, fmt.Errorf("unable to find project file: %v", err)
			}
		} else {
			lastGood, err := version.FindOne(version.ByLastKnownGoodConfig(projectRef.Identifier))
			if err != nil {
				return nil, fmt.Errorf("unable to find last valid config: %v", err)
			}
			if lastGood == nil { // brand new project + no valid config yet, just return an empty map
				return projectNameToBuildVariants, nil
			}

			proj = &model.Project{}
			err = model.LoadProjectInto([]byte(lastGood.Config), projectRef.Identifier, proj)
			if err != nil {
				return nil, fmt.Errorf("error loading project '%v' from version: %v", projectRef.Identifier, err)
			}
		}

		for _, buildVariant := range proj.BuildVariants {
			buildVariants = append(buildVariants, buildVariant.Name)
		}

		projectNameToBuildVariants[projectRef.Identifier] = buildVariants
	}

	return projectNameToBuildVariants, nil
}

func (ec *EvaluateCommand) Execute(args []string) error {
	if len(args) != 1 {
		return fmt.Errorf("the evaluate command takes one project config path as an argument")
	}
	configBytes, err := ioutil.ReadFile(args[0])
	if err != nil {
		return fmt.Errorf("error reading project config: %v", err)
	}

	p := &model.Project{}
	err = model.LoadProjectInto(configBytes, "", p)
	if err != nil {
		return fmt.Errorf("error loading project: %v", err)
	}

	var out interface{}
	if ec.Tasks || ec.Variants {
		tmp := struct {
			Functions interface{} `yaml:"functions,omitempty"`
			Tasks     interface{} `yaml:"tasks,omitempty"`
			Variants  interface{} `yaml:"buildvariants,omitempty`
		}{}
		if ec.Tasks {
			tmp.Functions = p.Functions
			tmp.Tasks = p.Tasks
		}
		if ec.Variants {
			tmp.Variants = p.BuildVariants
		}
		out = tmp
	} else {
		out = p
	}

	outYAML, err := yaml.Marshal(out)
	if err != nil {
		return fmt.Errorf("error marshalling evaluated project YAML: %v", err)
	}
	fmt.Println(string(outYAML))

	return nil
}

// GetTaskConfig fetches task configuration data required to run the task from the API server.
func (agt *Agent) GetTaskConfig() (*model.TaskConfig, error) {
	agt.logger.LogExecution(slogger.INFO, "Fetching distro configuration.")
	confDistro, err := agt.GetDistro()
	if err != nil {
		return nil, err
	}

	agt.logger.LogExecution(slogger.INFO, "Fetching version.")
	confVersion, err := agt.GetVersion()
	if err != nil {
		return nil, err
	}

	confProject := &model.Project{}
	err = model.LoadProjectInto([]byte(confVersion.Config), confVersion.Identifier, confProject)
	if err != nil {
		return nil, fmt.Errorf("reading project config: %v", err)
	}

	agt.logger.LogExecution(slogger.INFO, "Fetching task configuration.")
	confTask, err := agt.GetTask()
	if err != nil {
		return nil, err
	}

	agt.logger.LogExecution(slogger.INFO, "Fetching project ref.")
	confRef, err := agt.GetProjectRef()
	if err != nil {
		return nil, err
	}
	if confRef == nil {
		return nil, fmt.Errorf("agent retrieved an empty project ref")
	}

	agt.logger.LogExecution(slogger.INFO, "Constructing TaskConfig.")
	return model.NewTaskConfig(confDistro, confVersion, confProject, confTask, confRef)
}

// Takes in a version id and a map of "key -> buildvariant" (where "key" is of
// type "versionBuildVariant") and updates the map with an entry for the
// buildvariants associated with "versionStr"
func (self *Scheduler) updateVersionBuildVarMap(versionStr string,
	versionBuildVarMap map[versionBuildVariant]model.BuildVariant) (err error) {
	version, err := version.FindOne(version.ById(versionStr))
	if err != nil {
		return
	}
	if version == nil {
		return fmt.Errorf("nil version returned for version id '%v'", versionStr)
	}
	project := &model.Project{}

	err = model.LoadProjectInto([]byte(version.Config), version.Identifier, project)
	if err != nil {
		return fmt.Errorf("unable to load project config for version %v: "+
			"%v", versionStr, err)
	}

	// create buildvariant map (for accessing purposes)
	for _, buildVariant := range project.BuildVariants {
		key := versionBuildVariant{versionStr, buildVariant.Name}
		versionBuildVarMap[key] = buildVariant
	}
	return
}

func setupAPITestData(testConfig *evergreen.Settings, taskDisplayName string,
	variant string, projectFile string, patchMode patchTestMode, t *testing.T) (*task.Task, *build.Build, error) {
	// Ignore errs here because the ns might just not exist.
	clearDataMsg := "Failed to clear test data collection"
	testCollections := []string{
		task.Collection, build.Collection, host.Collection,
		distro.Collection, version.Collection, patch.Collection,
		model.PushlogCollection, model.ProjectVarsCollection, model.TaskQueuesCollection,
		manifest.Collection, model.ProjectRefCollection}
	testutil.HandleTestingErr(dbutil.ClearCollections(testCollections...), t, clearDataMsg)

	// Read in the project configuration
	projectConfig, err := ioutil.ReadFile(projectFile)
	testutil.HandleTestingErr(err, t, "failed to read project config")

	// Unmarshall the project configuration into a struct
	project := &model.Project{}
	testutil.HandleTestingErr(model.LoadProjectInto(projectConfig, "test", project),
		t, "failed to unmarshal project config")

	// Marshall the project YAML for storage
	projectYamlBytes, err := yaml.Marshal(project)
	testutil.HandleTestingErr(err, t, "failed to marshall project config")

	// Create the ref for the project
	projectRef := &model.ProjectRef{
		Identifier:  project.DisplayName,
		Owner:       project.Owner,
		Repo:        project.Repo,
		RepoKind:    project.RepoKind,
		Branch:      project.Branch,
		Enabled:     project.Enabled,
		BatchTime:   project.BatchTime,
		LocalConfig: string(projectConfig),
	}
	testutil.HandleTestingErr(projectRef.Insert(), t, "failed to insert projectRef")

	// Save the project variables
	projectVars := &model.ProjectVars{
		Id: project.DisplayName,
		Vars: map[string]string{
			"aws_key":    testConfig.Providers.AWS.Id,
			"aws_secret": testConfig.Providers.AWS.Secret,
			"fetch_key":  "fetch_expansion_value",
		},
	}
	_, err = projectVars.Upsert()
	testutil.HandleTestingErr(err, t, clearDataMsg)

	// Create and insert two tasks
	taskOne := &task.Task{
		Id:           "testTaskId",
		BuildId:      "testBuildId",
		DistroId:     "test-distro-one",
		BuildVariant: variant,
		Project:      project.DisplayName,
		DisplayName:  taskDisplayName,
		HostId:       "testHost",
		Secret:       "testTaskSecret",
		Version:      "testVersionId",
		Status:       evergreen.TaskDispatched,
		Requester:    evergreen.RepotrackerVersionRequester,
	}
	if patchMode != NoPatch {
		taskOne.Requester = evergreen.PatchVersionRequester
	}
	testutil.HandleTestingErr(taskOne.Insert(), t, "failed to insert taskOne")

	taskTwo := &task.Task{
		Id:           "testTaskIdTwo",
		BuildId:      "testBuildId",
		DistroId:     "test-distro-one",
		BuildVariant: variant,
		Project:      project.DisplayName,
		DisplayName:  taskDisplayName,
		HostId:       "",
		Secret:       "testTaskSecret",
		Version:      "testVersionId",
		Status:       evergreen.TaskUndispatched,
		Requester:    evergreen.RepotrackerVersionRequester,
		Activated:    true,
	}
	testutil.HandleTestingErr(taskTwo.Insert(), t, "failed to insert taskTwo")

	// Set up a task queue for task end tests
	taskQueue := &model.TaskQueue{
		Distro: "test-distro-one",
		Queue: []model.TaskQueueItem{
			{
				Id:          "testTaskIdTwo",
				DisplayName: taskDisplayName,
			},
		},
	}
	testutil.HandleTestingErr(taskQueue.Save(), t, "failed to insert taskqueue")

	// Insert the version document
	v := &version.Version{
		Id:       "testVersionId",
		BuildIds: []string{taskOne.BuildId},
//.........這裏部分代碼省略.........