Golang iodine.New函數代碼示例

// getBucketMetadataReaders - readers are returned in map rather than slice
func (donut API) getBucketMetadataReaders() (map[int]io.ReadCloser, error) {
	readers := make(map[int]io.ReadCloser)
	disks := make(map[int]disk.Disk)
	var err error
	for _, node := range donut.nodes {
		nDisks, err := node.ListDisks()
		if err != nil {
			return nil, iodine.New(err, nil)
		}
		for k, v := range nDisks {
			disks[k] = v
		}
	}
	var bucketMetaDataReader io.ReadCloser
	for order, disk := range disks {
		bucketMetaDataReader, err = disk.OpenFile(filepath.Join(donut.config.DonutName, bucketMetadataConfig))
		if err != nil {
			continue
		}
		readers[order] = bucketMetaDataReader
	}
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	return readers, nil
}

// writeEncodedData -
func (b bucket) writeEncodedData(k, m uint8, writers []io.WriteCloser, objectData io.Reader, sumMD5, sum512 hash.Hash) (int, int, error) {
	chunks := split.Stream(objectData, 10*1024*1024)
	encoder, err := newEncoder(k, m, "Cauchy")
	if err != nil {
		return 0, 0, iodine.New(err, nil)
	}
	chunkCount := 0
	totalLength := 0
	for chunk := range chunks {
		if chunk.Err == nil {
			totalLength = totalLength + len(chunk.Data)
			encodedBlocks, _ := encoder.Encode(chunk.Data)
			sumMD5.Write(chunk.Data)
			sum512.Write(chunk.Data)
			for blockIndex, block := range encodedBlocks {
				_, err := io.Copy(writers[blockIndex], bytes.NewBuffer(block))
				if err != nil {
					return 0, 0, iodine.New(err, nil)
				}
			}
		}
		chunkCount = chunkCount + 1
	}
	return chunkCount, totalLength, nil
}

// GetObjectMetadata - get object metadata from cache
func (donut API) GetObjectMetadata(bucket, key string) (ObjectMetadata, error) {
	donut.lock.Lock()
	defer donut.lock.Unlock()

	// check if bucket exists
	if !IsValidBucket(bucket) {
		return ObjectMetadata{}, iodine.New(BucketNameInvalid{Bucket: bucket}, nil)
	}
	if !IsValidObjectName(key) {
		return ObjectMetadata{}, iodine.New(ObjectNameInvalid{Object: key}, nil)
	}
	if !donut.storedBuckets.Exists(bucket) {
		return ObjectMetadata{}, iodine.New(BucketNotFound{Bucket: bucket}, nil)
	}
	storedBucket := donut.storedBuckets.Get(bucket).(storedBucket)
	objectKey := bucket + "/" + key
	if objMetadata, ok := storedBucket.objectMetadata[objectKey]; ok == true {
		return objMetadata, nil
	}
	if len(donut.config.NodeDiskMap) > 0 {
		objMetadata, err := donut.getObjectMetadata(bucket, key)
		if err != nil {
			return ObjectMetadata{}, iodine.New(err, nil)
		}
		// update
		storedBucket.objectMetadata[objectKey] = objMetadata
		donut.storedBuckets.Set(bucket, storedBucket)
		return objMetadata, nil
	}
	return ObjectMetadata{}, iodine.New(ObjectNotFound{Object: key}, nil)
}

// listDonutBuckets -
func (donut API) listDonutBuckets() error {
	for _, node := range donut.nodes {
		disks, err := node.ListDisks()
		if err != nil {
			return iodine.New(err, nil)
		}
		for _, disk := range disks {
			dirs, err := disk.ListDir(donut.config.DonutName)
			if err != nil {
				return iodine.New(err, nil)
			}
			for _, dir := range dirs {
				splitDir := strings.Split(dir.Name(), "$")
				if len(splitDir) < 3 {
					return iodine.New(CorruptedBackend{Backend: dir.Name()}, nil)
				}
				bucketName := splitDir[0]
				// we dont need this once we cache from makeDonutBucket()
				bucket, _, err := newBucket(bucketName, "private", donut.config.DonutName, donut.nodes)
				if err != nil {
					return iodine.New(err, nil)
				}
				donut.buckets[bucketName] = bucket
			}
		}
	}
	return nil
}

// trapSignal wait on listed signals for pre-defined behaviors
func (a *app) trapSignal(wg *sync.WaitGroup) {
	ch := make(chan os.Signal, 10)
	signal.Notify(ch, syscall.SIGTERM, syscall.SIGUSR2, syscall.SIGHUP)
	for {
		sig := <-ch
		switch sig {
		case syscall.SIGTERM:
			// this ensures a subsequent TERM will trigger standard go behaviour of terminating
			signal.Stop(ch)
			// roll through all initialized http servers and stop them
			for _, s := range a.sds {
				go func(s httpdown.Server) {
					defer wg.Done()
					if err := s.Stop(); err != nil {
						a.errors <- iodine.New(err, nil)
					}
				}(s)
			}
			return
		case syscall.SIGUSR2:
			fallthrough
		case syscall.SIGHUP:
			// we only return here if there's an error, otherwise the new process
			// will send us a TERM when it's ready to trigger the actual shutdown.
			if _, err := a.net.StartProcess(); err != nil {
				a.errors <- iodine.New(err, nil)
			}
		}
	}
}

// writeObjectData -
func (b bucket) writeObjectData(k, m uint8, writers []io.WriteCloser, objectData io.Reader, sumMD5, sum256, sum512 hash.Hash) (int, int, error) {
	encoder, err := newEncoder(k, m, "Cauchy")
	if err != nil {
		return 0, 0, iodine.New(err, nil)
	}
	chunkCount := 0
	totalLength := 0
	for chunk := range split.Stream(objectData, 10*1024*1024) {
		if chunk.Err != nil {
			return 0, 0, iodine.New(err, nil)
		}
		totalLength = totalLength + len(chunk.Data)
		encodedBlocks, err := encoder.Encode(chunk.Data)
		if err != nil {
			return 0, 0, iodine.New(err, nil)
		}

		sumMD5.Write(chunk.Data)
		sum256.Write(chunk.Data)
		sum512.Write(chunk.Data)
		for blockIndex, block := range encodedBlocks {
			errCh := make(chan error, 1)
			go func(writer io.Writer, reader io.Reader) {
				defer close(errCh)
				_, err := io.Copy(writers[blockIndex], bytes.NewReader(block))
				errCh <- err
			}(writers[blockIndex], bytes.NewReader(block))
			if err := <-errCh; err != nil {
				return 0, 0, iodine.New(err, nil)
			}
		}
		chunkCount = chunkCount + 1
	}
	return chunkCount, totalLength, nil
}

// ListenUnix announces on the local network address laddr. The network net
// must be a: "unix" or "unixpacket". It returns an inherited net.Listener for
// the matching network and address, or creates a new one using net.ListenUnix.
func (n *nimbleNet) ListenUnix(nett string, laddr *net.UnixAddr) (*net.UnixListener, error) {
	if err := n.getInheritedListeners(); err != nil {
		return nil, iodine.New(err, nil)
	}

	n.mutex.Lock()
	defer n.mutex.Unlock()

	// look for an inherited listener
	for i, l := range n.inheritedListeners {
		if l == nil { // we nil used inherited listeners
			continue
		}
		equal := nimbleAddr{l.Addr()}.IsEqual(laddr)
		if equal {
			n.inheritedListeners[i] = nil
			n.activeListeners = append(n.activeListeners, l)
			return l.(*net.UnixListener), nil
		}
	}

	// make a fresh listener
	l, err := net.ListenUnix(nett, laddr)
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	n.activeListeners = append(n.activeListeners, l)
	return l, nil
}

// SINGLE SOURCE - Type B: copy(f, d) -> copy(f, d/f) -> A
// prepareCopyURLsTypeB - prepares target and source URLs for copying.
func prepareCopyURLsTypeB(sourceURL string, targetURL string) copyURLs {
	_, sourceContent, err := url2Stat(sourceURL)
	if err != nil {
		// Source does not exist or insufficient privileges.
		return copyURLs{Error: NewIodine(iodine.New(err, nil))}
	}
	if !sourceContent.Type.IsRegular() {
		// Source is not a regular file.
		return copyURLs{Error: NewIodine(iodine.New(errInvalidSource{URL: sourceURL}, nil))}
	}

	// All OK.. We can proceed. Type B: source is a file, target is a folder and exists.
	sourceURLParse, err := client.Parse(sourceURL)
	if err != nil {
		return copyURLs{Error: NewIodine(iodine.New(errInvalidSource{URL: sourceURL}, nil))}
	}

	targetURLParse, err := client.Parse(targetURL)
	if err != nil {
		return copyURLs{Error: NewIodine(iodine.New(errInvalidTarget{URL: targetURL}, nil))}
	}

	targetURLParse.Path = filepath.Join(targetURLParse.Path, filepath.Base(sourceURLParse.Path))
	return prepareCopyURLsTypeA(sourceURL, targetURLParse.String())
}

// GetAuthKeys get access key id and secret access key
func GetAuthKeys(url string) ([]byte, error) {
	op := RPCOps{
		Method:  "Auth.Get",
		Request: rpc.Args{Request: ""},
	}
	req, err := NewRequest(url, op, http.DefaultTransport)
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	resp, err := req.Do()
	defer closeResp(resp)
	if err != nil {
		return nil, iodine.New(err, nil)
	}
	var reply rpc.AuthReply
	if err := jsonrpc.DecodeClientResponse(resp.Body, &reply); err != nil {
		return nil, iodine.New(err, nil)
	}
	authConfig := &auth.Config{}
	authConfig.Version = "0.0.1"
	authConfig.Users = make(map[string]*auth.User)
	user := &auth.User{}
	user.Name = "testuser"
	user.AccessKeyID = reply.AccessKeyID
	user.SecretAccessKey = reply.SecretAccessKey
	authConfig.Users[reply.AccessKeyID] = user
	if err := auth.SaveConfig(authConfig); err != nil {
		return nil, iodine.New(err, nil)
	}
	return json.MarshalIndent(reply, "", "\t")
}

// checkCopySyntaxTypeB verifies if the source is a valid file and target is a valid dir.
func checkCopySyntaxTypeB(srcURLs []string, tgtURL string) {
	if len(srcURLs) != 1 {
		console.Fatalf("Invalid number of source arguments to copy command. %s\n", NewIodine(iodine.New(errInvalidArgument{}, nil)))
	}
	srcURL := srcURLs[0]
	_, srcContent, err := url2Stat(srcURL)
	// Source exist?.
	if err != nil {
		console.Fatalf("Unable to stat source ‘%s’. %s\n", srcURL, NewIodine(iodine.New(err, nil)))
	}
	if srcContent.Type.IsDir() {
		console.Fatalf("Source ‘%s’ is a folder. Use ‘%s...’ argument to copy this folder and its contents recursively. %s\n", srcURL, srcURL, NewIodine(iodine.New(errInvalidArgument{}, nil)))
	}
	if !srcContent.Type.IsRegular() {
		console.Fatalf("Source ‘%s’ is not a file. %s\n", srcURL, NewIodine(iodine.New(errInvalidArgument{}, nil)))
	}

	_, tgtContent, err := url2Stat(tgtURL)
	// Target exist?.
	if err == nil {
		if !tgtContent.Type.IsDir() {
			console.Fatalf("Target ‘%s’ is not a folder. %s\n", tgtURL, NewIodine(iodine.New(errInvalidArgument{}, nil)))
		}
	}
}

// checkCopySyntaxTypeD verifies if the source is a valid list of file or valid recursive dir and target is a valid dir.
func checkCopySyntaxTypeD(srcURLs []string, tgtURL string) {
	for _, srcURL := range srcURLs {
		if isURLRecursive(srcURL) {
			srcURL = stripRecursiveURL(srcURL)
			_, srcContent, err := url2Stat(srcURL)
			// Source exist?.
			if err != nil {
				console.Fatalf("Unable to stat source ‘%s’. %s\n", srcURL, NewIodine(iodine.New(err, nil)))
			}
			if !srcContent.Type.IsDir() { // Ellipses is supported only for folders.
				console.Fatalf("Source ‘%s’ is not a folder. %s\n", stripRecursiveURL(srcURL), NewIodine(iodine.New(errInvalidArgument{}, nil)))
			}
		} else { // Regular URL.
			_, srcContent, err := url2Stat(srcURL)
			// Source exist?.
			if err != nil {
				console.Fatalf("Unable to stat source ‘%s’. %s\n", srcURL, NewIodine(iodine.New(err, nil)))
			}
			if srcContent.Type.IsDir() {
				console.Fatalf("Source ‘%s’ is a folder. Use ‘%s...’ argument to copy this folder and its contents recursively. %s\n", srcURL, srcURL, NewIodine(iodine.New(errInvalidArgument{}, nil)))
			}
			if !srcContent.Type.IsRegular() {
				console.Fatalf("Source ‘%s’ is not a file. %s\n", srcURL, NewIodine(iodine.New(errInvalidArgument{}, nil)))
			}
		}
	}
	_, tgtContent, err := url2Stat(tgtURL)
	// Target exist?.
	if err != nil {
		console.Fatalf("Unable to stat target ‘%s’. %s\n", tgtURL, NewIodine(iodine.New(err, nil)))
	}
	if !tgtContent.Type.IsDir() {
		console.Fatalf("Target ‘%s’ is not a folder. %s\n", tgtURL, NewIodine(iodine.New(errInvalidArgument{}, nil)))
	}
}

// ListObjects - list all objects
func (b bucket) ListObjects() (map[string]Object, error) {
	nodeSlice := 0
	for _, node := range b.nodes {
		disks, err := node.ListDisks()
		if err != nil {
			return nil, iodine.New(err, nil)
		}
		for _, disk := range disks {
			bucketSlice := fmt.Sprintf("%s$%d$%d", b.name, nodeSlice, disk.GetOrder())
			bucketPath := filepath.Join(b.donutName, bucketSlice)
			objects, err := disk.ListDir(bucketPath)
			if err != nil {
				return nil, iodine.New(err, nil)
			}
			for _, object := range objects {
				newObject, err := NewObject(object.Name(), filepath.Join(disk.GetPath(), bucketPath))
				if err != nil {
					return nil, iodine.New(err, nil)
				}
				newObjectMetadata, err := newObject.GetObjectMetadata()
				if err != nil {
					return nil, iodine.New(err, nil)
				}
				objectName, ok := newObjectMetadata["object"]
				if !ok {
					return nil, iodine.New(ObjectCorrupted{Object: object.Name()}, nil)
				}
				b.objects[objectName] = newObject
			}
		}
		nodeSlice = nodeSlice + 1
	}
	return b.objects, nil
}

// Rebalance -
func (d donut) Rebalance() error {
	var totalOffSetLength int
	var newDisks []disk.Disk
	var existingDirs []os.FileInfo
	for _, node := range d.nodes {
		disks, err := node.ListDisks()
		if err != nil {
			return iodine.New(err, nil)
		}
		totalOffSetLength = len(disks)
		fmt.Println(totalOffSetLength)
		for _, disk := range disks {
			dirs, err := disk.ListDir(d.name)
			if err != nil {
				return iodine.New(err, nil)
			}
			if len(dirs) == 0 {
				newDisks = append(newDisks, disk)
			}
			existingDirs = append(existingDirs, dirs...)
		}
	}
	for _, dir := range existingDirs {
		splits := strings.Split(dir.Name(), "$")
		bucketName, segment, offset := splits[0], splits[1], splits[2]
		fmt.Println(bucketName, segment, offset)
	}
	return nil
}

// listDonutBuckets -
func (donut API) listDonutBuckets() error {
	var disks map[int]disk.Disk
	var err error
	for _, node := range donut.nodes {
		disks, err = node.ListDisks()
		if err != nil {
			return iodine.New(err, nil)
		}
	}
	var dirs []os.FileInfo
	for _, disk := range disks {
		dirs, err = disk.ListDir(donut.config.DonutName)
		if err == nil {
			break
		}
	}
	// if all disks are missing then return error
	if err != nil {
		return iodine.New(err, nil)
	}
	for _, dir := range dirs {
		splitDir := strings.Split(dir.Name(), "$")
		if len(splitDir) < 3 {
			return iodine.New(CorruptedBackend{Backend: dir.Name()}, nil)
		}
		bucketName := splitDir[0]
		// we dont need this once we cache from makeDonutBucket()
		bucket, _, err := newBucket(bucketName, "private", donut.config.DonutName, donut.nodes)
		if err != nil {
			return iodine.New(err, nil)
		}
		donut.buckets[bucketName] = bucket
	}
	return nil
}

// Start a single disk subsystem
func Start(paths []string) (chan<- string, <-chan error, drivers.Driver) {
	ctrlChannel := make(chan string)
	errorChannel := make(chan error)

	// Soon to be user configurable, when Management API is available
	// we should remove "default" to something which is passed down
	// from configuration paramters
	var d donut.Donut
	var err error
	if len(paths) == 1 {
		d, err = donut.NewDonut("default", createNodeDiskMap(paths[0]))
		if err != nil {
			err = iodine.New(err, nil)
			log.Error.Println(err)
		}
	} else {
		d, err = donut.NewDonut("default", createNodeDiskMapFromSlice(paths))
		if err != nil {
			err = iodine.New(err, nil)
			log.Error.Println(err)
		}
	}
	s := new(donutDriver)
	s.donut = d
	s.paths = paths

	go start(ctrlChannel, errorChannel, s)
	return ctrlChannel, errorChannel, s
}