Golang keepclient.MakeKeepClient函数代码示例

func main() {
	flag.Parse()

	arv, err := arvadosclient.MakeArvadosClient()
	if err != nil {
		log.Fatal(err)
	}
	kc, err := keepclient.MakeKeepClient(&arv)
	if err != nil {
		log.Fatal(err)
	}
	kc.Want_replicas = *Replicas
	kc.Client.Timeout = 10 * time.Minute

	nextBuf := make(chan []byte, *WriteThreads)
	nextLocator := make(chan string, *ReadThreads+*WriteThreads)

	go countBeans(nextLocator)
	for i := 0; i < *WriteThreads; i++ {
		go makeBufs(nextBuf, i)
		go doWrites(kc, nextBuf, nextLocator)
	}
	for i := 0; i < *ReadThreads; i++ {
		go doReads(kc, nextLocator)
	}
	<-make(chan struct{})
}

func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
	testdata := make([]byte, blocksize)
	for i := 0; i < blocksize; i++ {
		testdata[i] = byte(' ')
	}
	arv, err := arvadosclient.MakeArvadosClient()
	c.Assert(err, check.Equals, nil)
	arv.ApiToken = arvadostest.ActiveToken
	kc, err := keepclient.MakeKeepClient(&arv)
	c.Assert(err, check.Equals, nil)
	loc, _, err := kc.PutB(testdata[:])
	c.Assert(err, check.Equals, nil)
	mtext := "."
	for i := 0; i < 100; i++ {
		mtext = mtext + " " + loc
	}
	mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
	coll := map[string]interface{}{}
	err = arv.Create("collections",
		map[string]interface{}{
			"collection": map[string]interface{}{
				"name":          fmt.Sprintf("testdata blocksize=%d", blocksize),
				"manifest_text": mtext,
			},
		}, &coll)
	c.Assert(err, check.Equals, nil)
	uuid := coll["uuid"].(string)

	hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
	c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
	c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
	c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
	c.Check(size, check.Equals, int64(blocksize)*100)
}

// Since keepstore does not come into picture in tests,
// we need to explicitly start the goroutine in tests.
func RunTestPullWorker(c *C) {
	arv, err := arvadosclient.MakeArvadosClient()
	c.Assert(err, Equals, nil)
	keepClient, err := keepclient.MakeKeepClient(&arv)
	c.Assert(err, Equals, nil)

	pullq = NewWorkQueue()
	go RunPullWorker(pullq, keepClient)
}

func (s *IntegrationSuite) SetUpSuite(c *check.C) {
	arvadostest.StartAPI()
	arvadostest.StartKeep(2, true)

	arv, err := arvadosclient.MakeArvadosClient()
	c.Assert(err, check.Equals, nil)
	arv.ApiToken = arvadostest.ActiveToken
	kc, err := keepclient.MakeKeepClient(&arv)
	c.Assert(err, check.Equals, nil)
	kc.PutB([]byte("Hello world\n"))
	kc.PutB([]byte("foo"))
	kc.PutB([]byte("foobar"))
}

// setup keepclient using the config provided
func setupKeepClient(config apiConfig, keepServicesJSON string, isDst bool, replications int, srcBlobSignatureTTL time.Duration) (kc *keepclient.KeepClient, blobSignatureTTL time.Duration, err error) {
	arv := arvadosclient.ArvadosClient{
		ApiToken:    config.APIToken,
		ApiServer:   config.APIHost,
		ApiInsecure: config.APIHostInsecure,
		Client: &http.Client{Transport: &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: config.APIHostInsecure}}},
		External: config.ExternalClient,
	}

	// if keepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
	if keepServicesJSON == "" {
		kc, err = keepclient.MakeKeepClient(&arv)
		if err != nil {
			return nil, 0, err
		}
	} else {
		kc = keepclient.New(&arv)
		err = kc.LoadKeepServicesFromJSON(keepServicesJSON)
		if err != nil {
			return kc, 0, err
		}
	}

	if isDst {
		// Get default replications value from destination, if it is not already provided
		if replications == 0 {
			value, err := arv.Discovery("defaultCollectionReplication")
			if err == nil {
				replications = int(value.(float64))
			} else {
				return nil, 0, err
			}
		}

		kc.Want_replicas = replications
	}

	// If srcBlobSignatureTTL is not provided, get it from API server discovery doc
	blobSignatureTTL = srcBlobSignatureTTL
	if !isDst && srcBlobSignatureTTL == 0 {
		value, err := arv.Discovery("blobSignatureTtl")
		if err == nil {
			blobSignatureTTL = time.Duration(int(value.(float64))) * time.Second
		} else {
			return nil, 0, err
		}
	}

	return kc, blobSignatureTTL, nil
}

func main() {
	api, err := arvadosclient.MakeArvadosClient()
	if err != nil {
		log.Fatal(err)
	}

	jobUuid := os.Getenv("JOB_UUID")
	taskUuid := os.Getenv("TASK_UUID")
	tmpdir := os.Getenv("TASK_WORK")
	keepmount := os.Getenv("TASK_KEEPMOUNT")

	var jobStruct Job
	var taskStruct Task

	err = api.Get("jobs", jobUuid, nil, &jobStruct)
	if err != nil {
		log.Fatal(err)
	}
	err = api.Get("job_tasks", taskUuid, nil, &taskStruct)
	if err != nil {
		log.Fatal(err)
	}

	var kc IKeepClient
	kc, err = keepclient.MakeKeepClient(&api)
	if err != nil {
		log.Fatal(err)
	}

	syscall.Umask(0022)
	err = runner(api, kc, jobUuid, taskUuid, tmpdir, keepmount, jobStruct, taskStruct)

	if err == nil {
		os.Exit(0)
	} else if _, ok := err.(TempFail); ok {
		log.Print(err)
		os.Exit(TASK_TEMPFAIL)
	} else if _, ok := err.(PermFail); ok {
		os.Exit(1)
	} else {
		log.Fatal(err)
	}
}

// setup keepclient using the config provided
func setupKeepClient(config apiConfig, keepServicesJSON string, blobSignatureTTL time.Duration) (kc *keepclient.KeepClient, ttl time.Duration, err error) {
	arv := arvadosclient.ArvadosClient{
		ApiToken:    config.APIToken,
		ApiServer:   config.APIHost,
		ApiInsecure: config.APIHostInsecure,
		Client: &http.Client{Transport: &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: config.APIHostInsecure}}},
		External: config.ExternalClient,
	}

	// if keepServicesJSON is provided, use it to load services; else, use DiscoverKeepServers
	if keepServicesJSON == "" {
		kc, err = keepclient.MakeKeepClient(&arv)
		if err != nil {
			return
		}
	} else {
		kc = keepclient.New(&arv)
		err = kc.LoadKeepServicesFromJSON(keepServicesJSON)
		if err != nil {
			return
		}
	}

	// Get if blobSignatureTTL is not provided
	ttl = blobSignatureTTL
	if blobSignatureTTL == 0 {
		value, err := arv.Discovery("blobSignatureTtl")
		if err == nil {
			ttl = time.Duration(int(value.(float64))) * time.Second
		} else {
			return nil, 0, err
		}
	}

	return
}

func main() {
	api, err := arvadosclient.MakeArvadosClient()
	if err != nil {
		log.Fatal(err)
	}

	// Container may not have certificates installed, so need to look for
	// /etc/arvados/ca-certificates.crt in addition to normal system certs.
	var certFiles = []string{
		"/etc/ssl/certs/ca-certificates.crt", // Debian
		"/etc/pki/tls/certs/ca-bundle.crt",   // Red Hat
		"/etc/arvados/ca-certificates.crt",
	}

	certs := x509.NewCertPool()
	for _, file := range certFiles {
		data, err := ioutil.ReadFile(file)
		if err == nil {
			log.Printf("Using TLS certificates at %v", file)
			certs.AppendCertsFromPEM(data)
		}
	}
	api.Client.Transport.(*http.Transport).TLSClientConfig.RootCAs = certs

	jobUuid := os.Getenv("JOB_UUID")
	taskUuid := os.Getenv("TASK_UUID")
	tmpdir := os.Getenv("TASK_WORK")
	keepmount := os.Getenv("TASK_KEEPMOUNT")

	var jobStruct Job
	var taskStruct Task

	err = api.Get("jobs", jobUuid, nil, &jobStruct)
	if err != nil {
		log.Fatal(err)
	}
	err = api.Get("job_tasks", taskUuid, nil, &taskStruct)
	if err != nil {
		log.Fatal(err)
	}

	var kc IKeepClient
	kc, err = keepclient.MakeKeepClient(&api)
	if err != nil {
		log.Fatal(err)
	}

	syscall.Umask(0022)
	err = runner(api, kc, jobUuid, taskUuid, tmpdir, keepmount, jobStruct, taskStruct)

	if err == nil {
		os.Exit(0)
	} else if _, ok := err.(TempFail); ok {
		log.Print(err)
		os.Exit(TASK_TEMPFAIL)
	} else if _, ok := err.(PermFail); ok {
		os.Exit(1)
	} else {
		log.Fatal(err)
	}
}

func main() {
	var (
		listen           string
		no_get           bool
		no_put           bool
		default_replicas int
		timeout          int64
		pidfile          string
	)

	flagset := flag.NewFlagSet("keepproxy", flag.ExitOnError)

	flagset.StringVar(
		&listen,
		"listen",
		DefaultAddr,
		"Interface on which to listen for requests, in the format "+
			"ipaddr:port. e.g. -listen=10.0.1.24:8000. Use -listen=:port "+
			"to listen on all network interfaces.")

	flagset.BoolVar(
		&no_get,
		"no-get",
		false,
		"If set, disable GET operations")

	flagset.BoolVar(
		&no_put,
		"no-put",
		false,
		"If set, disable PUT operations")

	flagset.IntVar(
		&default_replicas,
		"default-replicas",
		2,
		"Default number of replicas to write if not specified by the client.")

	flagset.Int64Var(
		&timeout,
		"timeout",
		15,
		"Timeout on requests to internal Keep services (default 15 seconds)")

	flagset.StringVar(
		&pidfile,
		"pid",
		"",
		"Path to write pid file")

	flagset.Parse(os.Args[1:])

	arv, err := arvadosclient.MakeArvadosClient()
	if err != nil {
		log.Fatalf("Error setting up arvados client %s", err.Error())
	}

	if os.Getenv("ARVADOS_DEBUG") != "" {
		keepclient.DebugPrintf = log.Printf
	}
	kc, err := keepclient.MakeKeepClient(&arv)
	if err != nil {
		log.Fatalf("Error setting up keep client %s", err.Error())
	}

	if pidfile != "" {
		f, err := os.Create(pidfile)
		if err != nil {
			log.Fatalf("Error writing pid file (%s): %s", pidfile, err.Error())
		}
		fmt.Fprint(f, os.Getpid())
		f.Close()
		defer os.Remove(pidfile)
	}

	kc.Want_replicas = default_replicas
	kc.Client.Timeout = time.Duration(timeout) * time.Second
	go kc.RefreshServices(5*time.Minute, 3*time.Second)

	listener, err = net.Listen("tcp", listen)
	if err != nil {
		log.Fatalf("Could not listen on %v", listen)
	}
	log.Printf("Arvados Keep proxy started listening on %v", listener.Addr())

	// Shut down the server gracefully (by closing the listener)
	// if SIGTERM is received.
	term := make(chan os.Signal, 1)
	go func(sig <-chan os.Signal) {
		s := <-sig
		log.Println("caught signal:", s)
		listener.Close()
	}(term)
	signal.Notify(term, syscall.SIGTERM)
	signal.Notify(term, syscall.SIGINT)

	// Start serving requests.
	http.Serve(listener, MakeRESTRouter(!no_get, !no_put, kc))

	log.Println("shutting down")
//.........这里部分代码省略.........

func singlerun(arv arvadosclient.ArvadosClient) error {
	var err error
	if isAdmin, err := util.UserIsAdmin(arv); err != nil {
		return errors.New("Error verifying admin token: " + err.Error())
	} else if !isAdmin {
		return errors.New("Current user is not an admin. Datamanager requires a privileged token.")
	}

	if logEventTypePrefix != "" {
		arvLogger, err = logger.NewLogger(logger.LoggerParams{
			Client:          arv,
			EventTypePrefix: logEventTypePrefix,
			WriteInterval:   time.Second * time.Duration(logFrequencySeconds)})
	}

	loggerutil.LogRunInfo(arvLogger)
	if arvLogger != nil {
		arvLogger.AddWriteHook(loggerutil.LogMemoryAlloc)
	}

	var (
		dataFetcher     summary.DataFetcher
		readCollections collection.ReadCollections
		keepServerInfo  keep.ReadServers
	)

	if summary.ShouldReadData() {
		dataFetcher = summary.ReadData
	} else {
		dataFetcher = BuildDataFetcher(arv)
	}

	err = dataFetcher(arvLogger, &readCollections, &keepServerInfo)
	if err != nil {
		return err
	}

	err = summary.MaybeWriteData(arvLogger, readCollections, keepServerInfo)
	if err != nil {
		return err
	}

	buckets := summary.BucketReplication(readCollections, keepServerInfo)
	bucketCounts := buckets.Counts()

	replicationSummary := buckets.SummarizeBuckets(readCollections)
	replicationCounts := replicationSummary.ComputeCounts()

	log.Printf("Blocks In Collections: %d, "+
		"\nBlocks In Keep: %d.",
		len(readCollections.BlockToDesiredReplication),
		len(keepServerInfo.BlockToServers))
	log.Println(replicationCounts.PrettyPrint())

	log.Printf("Blocks Histogram:")
	for _, rlbss := range bucketCounts {
		log.Printf("%+v: %10d",
			rlbss.Levels,
			rlbss.Count)
	}

	kc, err := keepclient.MakeKeepClient(&arv)
	if err != nil {
		return fmt.Errorf("Error setting up keep client %v", err.Error())
	}

	// Log that we're finished. We force the recording, since go will
	// not wait for the write timer before exiting.
	if arvLogger != nil {
		defer arvLogger.FinalUpdate(func(p map[string]interface{}, e map[string]interface{}) {
			summaryInfo := logger.GetOrCreateMap(p, "summary_info")
			summaryInfo["block_replication_counts"] = bucketCounts
			summaryInfo["replication_summary"] = replicationCounts
			p["summary_info"] = summaryInfo

			p["run_info"].(map[string]interface{})["finished_at"] = time.Now()
		})
	}

	pullServers := summary.ComputePullServers(kc,
		&keepServerInfo,
		readCollections.BlockToDesiredReplication,
		replicationSummary.UnderReplicatedBlocks)

	pullLists := summary.BuildPullLists(pullServers)

	trashLists, trashErr := summary.BuildTrashLists(kc,
		&keepServerInfo,
		replicationSummary.KeepBlocksNotInCollections)

	err = summary.WritePullLists(arvLogger, pullLists, dryRun)
	if err != nil {
		return err
	}

	if trashErr != nil {
		return err
	}
	keep.SendTrashLists(arvLogger, kc, trashLists, dryRun)

//.........这里部分代码省略.........

//.........这里部分代码省略.........
	collection := make(map[string]interface{})
	found := false
	for _, arv.ApiToken = range tokens {
		err := arv.Get("collections", targetID, nil, &collection)
		if err == nil {
			// Success
			found = true
			break
		}
		if srvErr, ok := err.(arvadosclient.APIServerError); ok {
			switch srvErr.HttpStatusCode {
			case 404, 401:
				// Token broken or insufficient to
				// retrieve collection
				tokenResult[arv.ApiToken] = srvErr.HttpStatusCode
				continue
			}
		}
		// Something more serious is wrong
		statusCode, statusText = http.StatusInternalServerError, err.Error()
		return
	}
	if !found {
		if pathToken || !credentialsOK {
			// Either the URL is a "secret sharing link"
			// that didn't work out (and asking the client
			// for additional credentials would just be
			// confusing), or we don't even accept
			// credentials at this path.
			statusCode = http.StatusNotFound
			return
		}
		for _, t := range reqTokens {
			if tokenResult[t] == 404 {
				// The client provided valid token(s), but the
				// collection was not found.
				statusCode = http.StatusNotFound
				return
			}
		}
		// The client's token was invalid (e.g., expired), or
		// the client didn't even provide one.  Propagate the
		// 401 to encourage the client to use a [different]
		// token.
		//
		// TODO(TC): This response would be confusing to
		// someone trying (anonymously) to download public
		// data that has been deleted.  Allow a referrer to
		// provide this context somehow?
		w.Header().Add("WWW-Authenticate", "Basic realm=\"collections\"")
		statusCode = http.StatusUnauthorized
		return
	}

	filename := strings.Join(targetPath, "/")
	kc, err := keepclient.MakeKeepClient(arv)
	if err != nil {
		statusCode, statusText = http.StatusInternalServerError, err.Error()
		return
	}
	if kc.Client != nil && kc.Client.Transport != nil {
		// Workaround for https://dev.arvados.org/issues/9005
		if t, ok := kc.Client.Transport.(*http.Transport); ok {
			defer t.CloseIdleConnections()
		}
	}
	rdr, err := kc.CollectionFileReader(collection, filename)
	if os.IsNotExist(err) {
		statusCode = http.StatusNotFound
		return
	} else if err != nil {
		statusCode, statusText = http.StatusBadGateway, err.Error()
		return
	}
	defer rdr.Close()

	basenamePos := strings.LastIndex(filename, "/")
	if basenamePos < 0 {
		basenamePos = 0
	}
	extPos := strings.LastIndex(filename, ".")
	if extPos > basenamePos {
		// Now extPos is safely >= 0.
		if t := mime.TypeByExtension(filename[extPos:]); t != "" {
			w.Header().Set("Content-Type", t)
		}
	}
	if rdr, ok := rdr.(keepclient.ReadCloserWithLen); ok {
		w.Header().Set("Content-Length", fmt.Sprintf("%d", rdr.Len()))
	}

	applyContentDispositionHdr(w, r, filename[basenamePos:], attachment)
	rangeRdr, statusCode := applyRangeHdr(w, r, rdr)

	w.WriteHeader(statusCode)
	_, err = io.Copy(w, rangeRdr)
	if err != nil {
		statusCode, statusText = http.StatusBadGateway, err.Error()
	}
}

func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
	log.Print("TestPutAndGet start")

	os.Args = []string{"keepproxy", "-listen=:29950"}
	listener = nil
	go main()
	time.Sleep(100 * time.Millisecond)

	setupProxyService()

	os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
	arv, err := arvadosclient.MakeArvadosClient()
	c.Assert(err, Equals, nil)
	kc, err := keepclient.MakeKeepClient(&arv)
	c.Assert(err, Equals, nil)
	c.Check(kc.Arvados.External, Equals, true)
	c.Check(kc.Using_proxy, Equals, true)
	c.Check(len(kc.LocalRoots()), Equals, 1)
	for _, root := range kc.LocalRoots() {
		c.Check(root, Equals, "http://localhost:29950")
	}
	os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")

	waitForListener()
	defer closeListener()

	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
	var hash2 string

	{
		_, _, err := kc.Ask(hash)
		c.Check(err, Equals, keepclient.BlockNotFound)
		log.Print("Finished Ask (expected BlockNotFound)")
	}

	{
		reader, _, _, err := kc.Get(hash)
		c.Check(reader, Equals, nil)
		c.Check(err, Equals, keepclient.BlockNotFound)
		log.Print("Finished Get (expected BlockNotFound)")
	}

	// Note in bug #5309 among other errors keepproxy would set
	// Content-Length incorrectly on the 404 BlockNotFound response, this
	// would result in a protocol violation that would prevent reuse of the
	// connection, which would manifest by the next attempt to use the
	// connection (in this case the PutB below) failing.  So to test for
	// that bug it's necessary to trigger an error response (such as
	// BlockNotFound) and then do something else with the same httpClient
	// connection.

	{
		var rep int
		var err error
		hash2, rep, err = kc.PutB([]byte("foo"))
		c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
		c.Check(rep, Equals, 2)
		c.Check(err, Equals, nil)
		log.Print("Finished PutB (expected success)")
	}

	{
		blocklen, _, err := kc.Ask(hash2)
		c.Assert(err, Equals, nil)
		c.Check(blocklen, Equals, int64(3))
		log.Print("Finished Ask (expected success)")
	}

	{
		reader, blocklen, _, err := kc.Get(hash2)
		c.Assert(err, Equals, nil)
		all, err := ioutil.ReadAll(reader)
		c.Check(all, DeepEquals, []byte("foo"))
		c.Check(blocklen, Equals, int64(3))
		log.Print("Finished Get (expected success)")
	}

	{
		var rep int
		var err error
		hash2, rep, err = kc.PutB([]byte(""))
		c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
		c.Check(rep, Equals, 2)
		c.Check(err, Equals, nil)
		log.Print("Finished PutB zero block")
	}

	{
		reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
		c.Assert(err, Equals, nil)
		all, err := ioutil.ReadAll(reader)
		c.Check(all, DeepEquals, []byte(""))
		c.Check(blocklen, Equals, int64(0))
		log.Print("Finished Get zero block")
	}

	log.Print("TestPutAndGet done")
}