Golang needle.Needle类代码示例

func compareTestNeedle(t *testing.T, key int64, cookie int32, flag byte, n *needle.Needle, data, buf []byte) (err error) {
	if err = n.Parse(); err != nil {
		t.Error(err)
		return
	}
	if !bytes.Equal(n.Data, data) {
		err = fmt.Errorf("data: %s not match", n.Data)
		t.Error(err)
		return
	}
	if n.Cookie != cookie {
		err = fmt.Errorf("cookie: %d not match", n.Cookie)
		t.Error(err)
		return
	}
	if n.Key != key {
		err = fmt.Errorf("key: %d not match", n.Key)
		t.Error(err)
		return
	}
	if n.Flag != flag {
		err = fmt.Errorf("flag: %d not match", n.Flag)
		t.Error(err)
		return
	}
	if n.Size != int32(len(data)) {
		err = fmt.Errorf("size: %d not match", n.Size)
		t.Error(err)
		return
	}
	return
}

func (h httpGetHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
	var (
		now              = time.Now()
		v                *Volume
		n                *needle.Needle
		err              error
		vid, key, cookie int64
		ret              = http.StatusOK
		params           = r.URL.Query()
	)
	if r.Method != "GET" && r.Method != "HEAD" {
		ret = http.StatusMethodNotAllowed
		http.Error(wr, "method not allowed", ret)
		return
	}
	defer HttpGetWriter(r, wr, now, &err, &ret)
	if vid, err = strconv.ParseInt(params.Get("vid"), 10, 32); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("vid"), err)
		ret = http.StatusBadRequest
		return
	}
	if key, err = strconv.ParseInt(params.Get("key"), 10, 64); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("key"), err)
		ret = http.StatusBadRequest
		return
	}
	if cookie, err = strconv.ParseInt(params.Get("cookie"), 10, 32); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", params.Get("cookie"), err)
		ret = http.StatusBadRequest
		return
	}
	n = h.s.Needle()
	n.Key = key
	n.Cookie = int32(cookie)
	if v = h.s.Volumes[int32(vid)]; v != nil {
		if err = v.Get(n); err != nil {
			if err == errors.ErrNeedleDeleted || err == errors.ErrNeedleNotExist {
				ret = http.StatusNotFound
			} else {
				ret = http.StatusInternalServerError
			}
		}
	} else {
		ret = http.StatusNotFound
		err = errors.ErrVolumeNotExist
	}
	if err == nil {
		if r.Method == "GET" {
			if _, err = wr.Write(n.Data); err != nil {
				log.Errorf("wr.Write() error(%v)", err)
				ret = http.StatusInternalServerError
			}
		}
		if log.V(1) {
			log.Infof("get a needle: %v", n)
		}
	}
	h.s.FreeNeedle(n)
	return
}

// Get get a needle by key and cookie.
func (v *Volume) Get(n *needle.Needle) (err error) {
	var (
		ok     bool
		nc     int64
		size   int32
		offset uint32
		key    = n.Key
		cookie = n.Cookie
		now    = time.Now().UnixNano()
	)
	// WARN pread syscall is atomic, so use rlock
	v.lock.RLock()
	if nc, ok = v.needles[n.Key]; ok {
		offset, size = needle.Cache(nc)
		if offset != needle.CacheDelOffset {
			err = v.Block.Get(offset, n.Buffer[:size])
		} else {
			err = errors.ErrNeedleDeleted
		}
	} else {
		err = errors.ErrNeedleNotExist
	}
	v.lock.RUnlock()
	if err != nil {
		return
	}
	if log.V(1) {
		log.Infof("get needle key: %d, cookie: %d, offset: %d, size: %d", n.Key, n.Cookie, offset, size)
	}
	if err = n.Parse(); err != nil {
		return
	}
	if log.V(1) {
		log.Infof("%v\n", n)
	}
	if n.Key != key {
		err = errors.ErrNeedleKey
		return
	}
	if n.Cookie != cookie {
		err = errors.ErrNeedleCookie
		return
	}
	// needles map may be out-dated, recheck
	if n.Flag == needle.FlagDel {
		v.lock.Lock()
		v.needles[key] = needle.NewCache(needle.CacheDelOffset, size)
		v.lock.Unlock()
		err = errors.ErrNeedleDeleted
	} else {
		atomic.AddUint64(&v.Stats.TotalGetProcessed, 1)
		atomic.AddUint64(&v.Stats.TotalReadBytes, uint64(size))
		atomic.AddUint64(&v.Stats.TotalGetDelay, uint64(time.Now().UnixNano()-now))
	}
	return
}

// Write start add needles to the block, must called after start a transaction.
func (b *SuperBlock) Write(n *needle.Needle) (err error) {
	if b.LastErr != nil {
		return b.LastErr
	}
	var incrOffset = needle.NeedleOffset(int64(n.TotalSize))
	if err = b.available(incrOffset); err != nil {
		return
	}
	if err = n.Write(b.bw); err != nil {
		b.LastErr = err
		return
	}
	b.Offset += incrOffset
	return
}

func BenchmarkVolumeWrite(b *testing.B) {
	var (
		v     *Volume
		err   error
		file  = "./test/testb2"
		ifile = "./test/testb2.idx"
		data  = make([]byte, _16kb) // 16kb
	)
	os.Remove(file)
	os.Remove(ifile)
	defer os.Remove(file)
	defer os.Remove(ifile)
	if _, err = rand.Read(data); err != nil {
		b.Errorf("rand.Read() error(%v)", err)
		b.FailNow()
	}
	if v, err = NewVolume(1, file, ifile, testConf); err != nil {
		b.Errorf("NewVolume() error(%v)", err)
		b.FailNow()
	}
	defer v.Close()
	b.SetParallelism(8)
	b.ResetTimer()
	b.RunParallel(func(pb *testing.PB) {
		var (
			i, j int
			ts   int32
			t    int64
			err1 error
			n    *needle.Needle
			ns   = make([]needle.Needle, 9)
			buf  = make([]byte, 163840) // 16kb
		)
		for i = 0; i < 9; i++ {
			t = mrand.Int63()
			n = &ns[i]
			n.Init(t, 1, data)
			n.Write(buf[ts:])
			ts += n.TotalSize
		}
		for pb.Next() {
			for j = 0; j < 9; j++ {
				t = mrand.Int63()
				n = &ns[j]
				n.Key = t
				binary.BigEndian.PutInt64(buf[n.TotalSize+needle.KeyOffset:], n.Key)
			}
			if err1 = v.Write(ns, buf[:ts]); err1 != nil {
				b.Errorf("Add() error(%v)", err1)
				v.Unlock()
				b.FailNow()
			}
			b.SetBytes(int64(ts))
		}
	})
	os.Remove(file)
	os.Remove(ifile)
}

func (h httpUploadsHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
	var (
		i, rn, tn, nn, nb int
		ok                bool
		buf               []byte
		err               error
		vid               int64
		key               int64
		cookie            int64
		size              int64
		str               string
		keys              []string
		cookies           []string
		sr                sizer
		fr                *os.File
		fi                os.FileInfo
		v                 *Volume
		n                 *needle.Needle
		ns                []needle.Needle
		uerr              errors.Error
		file              multipart.File
		fh                *multipart.FileHeader
		fhs               []*multipart.FileHeader
		res               = map[string]interface{}{"ret": errors.RetOK}
	)
	if r.Method != "POST" {
		http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
		return
	}
	defer HttpPostWriter(r, wr, time.Now(), res)
	// check total content-length
	if size, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil {
		res["ret"] = errors.RetInternalErr
		return
	}
	if size > int64(h.c.NeedleMaxSize*h.c.BatchMaxNum) {
		res["ret"] = errors.RetNeedleTooLarge
		return
	}
	str = r.FormValue("vid")
	if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
		res["ret"] = errors.RetParamErr
		return
	}
	keys = r.MultipartForm.Value["keys"]
	cookies = r.MultipartForm.Value["cookies"]
	if len(keys) != len(cookies) {
		log.Errorf("param length not match, keys: %d, cookies: %d", len(keys), len(cookies))
		res["ret"] = errors.RetParamErr
		return
	}
	fhs = r.MultipartForm.File["file"]
	nn = len(fhs)
	if len(keys) != nn {
		log.Errorf("param length not match, keys: %d, cookies: %d, files: %d", len(keys), len(cookies), len(fhs))
		res["ret"] = errors.RetParamErr
		return
	}
	nb = int(size-1)/(h.c.NeedleMaxSize) + 1
	buf = h.s.Buffer(nb)
	ns = h.s.Needle(nn)
	for i, fh = range fhs {
		if key, err = strconv.ParseInt(keys[i], 10, 64); err != nil {
			log.Errorf("strconv.ParseInt(\"%s\") error(%v)", keys[i], err)
			err = errors.ErrParam
			break
		}
		if cookie, err = strconv.ParseInt(cookies[i], 10, 32); err != nil {
			log.Errorf("strconv.ParseInt(\"%s\") error(%v)", cookies[i], err)
			err = errors.ErrParam
			break
		}
		file, err = fh.Open()
		file.Close()
		if err != nil {
			log.Errorf("fh.Open() error(%v)", err)
			break
		}
		// check size
		if sr, ok = file.(sizer); ok {
			size = sr.Size()
		} else if fr, ok = file.(*os.File); ok {
			if fi, err = fr.Stat(); err != nil {
				break
			}
			size = fi.Size()
		}
		if size > int64(h.c.NeedleMaxSize) {
			err = errors.ErrNeedleTooLarge
			break
		}
		if rn, err = file.Read(buf[tn:]); err != nil {
			log.Errorf("file.Read() error(%v)", err)
			break
		}
		n = &(ns[i])
		n.Parse(key, int32(cookie), buf[tn:tn+rn])
		tn += rn
	}
//.........这里部分代码省略.........

func (h httpUploadHandler) ServeHTTP(wr http.ResponseWriter, r *http.Request) {
	var (
		vid    int64
		key    int64
		cookie int64
		size   int64
		err    error
		str    string
		v      *volume.Volume
		n      *needle.Needle
		file   multipart.File
		res    = map[string]interface{}{}
	)
	if r.Method != "POST" {
		http.Error(wr, "method not allowed", http.StatusMethodNotAllowed)
		return
	}
	defer HttpPostWriter(r, wr, time.Now(), &err, res)
	// check total content-length
	if size, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil {
		err = errors.ErrInternal
		return
	}
	if size > int64(h.c.NeedleMaxSize) {
		err = errors.ErrNeedleTooLarge
		return
	}
	str = r.FormValue("vid")
	if vid, err = strconv.ParseInt(str, 10, 32); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
		err = errors.ErrParam
		return
	}
	str = r.FormValue("key")
	if key, err = strconv.ParseInt(str, 10, 64); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
		err = errors.ErrParam
		return
	}
	str = r.FormValue("cookie")
	if cookie, err = strconv.ParseInt(str, 10, 32); err != nil {
		log.Errorf("strconv.ParseInt(\"%s\") error(%v)", str, err)
		err = errors.ErrParam
		return
	}
	if file, _, err = r.FormFile("file"); err != nil {
		log.Errorf("r.FormFile() error(%v)", err)
		err = errors.ErrInternal
		return
	}
	defer file.Close()
	if size, err = fileSize(file); err != nil {
		log.Errorf("fileSize() error(%v)", err)
		err = errors.ErrInternal
		return
	}
	if size > int64(h.c.NeedleMaxSize) {
		err = errors.ErrNeedleTooLarge
		return
	}
	n = h.s.Needle()
	if err = n.WriteFrom(key, int32(cookie), int32(size), file); err == nil {
		if v = h.s.Volumes[int32(vid)]; v != nil {
			err = v.Add(n)
		} else {
			err = errors.ErrVolumeNotExist
		}
	}
	h.s.FreeNeedle(n)
	return
}